From 37f294fec20e5b27532c156cd6956d65d804a5df Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Tue, 27 Dec 2016 03:21:13 +0200 Subject: [PATCH 01/74] Disable the test for valgrind builds Test is very slow with valgrind, and pointless because it is initially about a race condition which is hardly achievable with valgrind --- mysql-test/t/mdev-504.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/t/mdev-504.test b/mysql-test/t/mdev-504.test index fb5c7666d3347..b96c8779c683f 100644 --- a/mysql-test/t/mdev-504.test +++ b/mysql-test/t/mdev-504.test @@ -1,3 +1,4 @@ +--source include/not_valgrind.inc --disable_ps_protocol SET GLOBAL net_write_timeout = 900; From f493e395b0a6679b55d9859f372f6ccd4475f12d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 29 Dec 2016 15:03:12 +0200 Subject: [PATCH 02/74] Make the test work with any innodb_page_size. --- mysql-test/suite/innodb/t/doublewrite.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/innodb/t/doublewrite.test b/mysql-test/suite/innodb/t/doublewrite.test index 5bd4551aa1edb..56cc88f5590fc 100644 --- a/mysql-test/suite/innodb/t/doublewrite.test +++ b/mysql-test/suite/innodb/t/doublewrite.test @@ -13,10 +13,10 @@ SET GLOBAL innodb_fast_shutdown = 0; --source include/restart_mysqld.inc --disable_query_log -call mtr.add_suppression("InnoDB: Database page [0-9]+:1 contained only zeroes."); -call mtr.add_suppression("Header page consists of zero bytes"); -call mtr.add_suppression("Checksum mismatch in tablespace.*table test/t1"); -call mtr.add_suppression("but the innodb_page_size start-up parameter is"); +call mtr.add_suppression("space header page consists of zero bytes.*test.t1"); +call mtr.add_suppression("checksum mismatch in tablespace.*test.t1"); +call mtr.add_suppression("Current page size .* != page size on page"); +call mtr.add_suppression("innodb-page-size mismatch in tablespace.*test.t1"); call mtr.add_suppression("Database page corruption"); --enable_query_log From 8451e09073e8b1a300f177d74a9e3a530776640a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 28 Dec 2016 12:05:43 +0200 Subject: [PATCH 03/74] MDEV-11556 InnoDB redo log apply fails to adjust data file sizes fil_space_t::recv_size: New member: recovered tablespace size in pages; 0 if no size change was read from the redo log, or if the size change was implemented. fil_space_set_recv_size(): New function for setting space->recv_size. innodb_data_file_size_debug: A debug parameter for setting the system tablespace size in recovery even when the redo log does not contain any size changes. It is hard to write a small test case that would cause the system tablespace to be extended at the critical moment. recv_parse_log_rec(): Note those tablespaces whose size is being changed by the redo log, by invoking fil_space_set_recv_size(). innobase_init(): Correct an error message, and do not require a larger innodb_buffer_pool_size when starting up with a smaller innodb_page_size. innobase_start_or_create_for_mysql(): Allow startup with any initial size of the ibdata1 file if the autoextend attribute is set. Require the minimum size of fixed-size system tablespaces to be 640 pages, not 10 megabytes. Implement innodb_data_file_size_debug. open_or_create_data_files(): Round the system tablespace size down to pages, not to full megabytes, (Our test truncates the system tablespace to more than 800 pages with innodb_page_size=4k. InnoDB should not imagine that it was truncated to 768 pages and then overwrite good pages in the tablespace.) fil_flush_low(): Refactored from fil_flush(). fil_space_extend_must_retry(): Refactored from fil_extend_space_to_desired_size(). fil_mutex_enter_and_prepare_for_io(): Extend the tablespace if fil_space_set_recv_size() was called. The test case has been successfully run with all the innodb_page_size values 4k, 8k, 16k, 32k, 64k. --- .../suite/innodb/r/log_data_file_size.result | 9 + .../suite/innodb/t/log_data_file_size.opt | 2 + .../suite/innodb/t/log_data_file_size.test | 66 ++ .../suite/sys_vars/r/sysvars_innodb.result | 14 + storage/innobase/fil/fil0fil.cc | 790 +++++++++--------- storage/innobase/handler/ha_innodb.cc | 16 +- storage/innobase/include/fil0fil.h | 10 + storage/innobase/include/srv0srv.h | 1 + storage/innobase/log/log0recv.cc | 8 + storage/innobase/srv/srv0start.cc | 28 +- storage/xtradb/fil/fil0fil.cc | 788 ++++++++--------- storage/xtradb/handler/ha_innodb.cc | 16 +- storage/xtradb/include/fil0fil.h | 10 + storage/xtradb/include/srv0srv.h | 1 + storage/xtradb/log/log0recv.cc | 8 + storage/xtradb/srv/srv0start.cc | 28 +- 16 files changed, 1000 insertions(+), 795 deletions(-) create mode 100644 mysql-test/suite/innodb/r/log_data_file_size.result create mode 100644 mysql-test/suite/innodb/t/log_data_file_size.opt create mode 100644 mysql-test/suite/innodb/t/log_data_file_size.test diff --git a/mysql-test/suite/innodb/r/log_data_file_size.result b/mysql-test/suite/innodb/r/log_data_file_size.result new file mode 100644 index 0000000000000..7e994cbe1e2f9 --- /dev/null +++ b/mysql-test/suite/innodb/r/log_data_file_size.result @@ -0,0 +1,9 @@ +SET GLOBAL innodb_file_per_table=0; +SET GLOBAL innodb_file_format=barracuda; +CREATE TABLE t(a INT)ENGINE=InnoDB; +SET GLOBAL innodb_file_per_table=1; +CREATE TABLE ibd4(a INT UNIQUE)ENGINE=InnoDB; +CREATE TABLE ibd4f(a INT UNIQUE)ENGINE=InnoDB; +CREATE TABLE ibd5(a INT UNIQUE, b INT UNIQUE)ENGINE=InnoDB; +# Kill the server +DROP TABLE t,ibd4,ibd4f,ibd5; diff --git a/mysql-test/suite/innodb/t/log_data_file_size.opt b/mysql-test/suite/innodb/t/log_data_file_size.opt new file mode 100644 index 0000000000000..d9a364a32872d --- /dev/null +++ b/mysql-test/suite/innodb/t/log_data_file_size.opt @@ -0,0 +1,2 @@ +--loose-innodb-sys-indexes +--innodb-data-file-path=ibdata1:1M:autoextend diff --git a/mysql-test/suite/innodb/t/log_data_file_size.test b/mysql-test/suite/innodb/t/log_data_file_size.test new file mode 100644 index 0000000000000..23f1ede483f21 --- /dev/null +++ b/mysql-test/suite/innodb/t/log_data_file_size.test @@ -0,0 +1,66 @@ +--source include/have_innodb.inc +--source include/not_embedded.inc + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; +let MYSQLD_IS_DEBUG=`select version() like '%debug%'`; +--source include/no_checkpoint_start.inc +SET GLOBAL innodb_file_per_table=0; +SET GLOBAL innodb_file_format=barracuda; +CREATE TABLE t(a INT)ENGINE=InnoDB; +let INNODB_ROOT_PAGE= `SELECT page_no FROM INFORMATION_SCHEMA.INNODB_SYS_INDEXES WHERE name='GEN_CLUST_INDEX'`; +SET GLOBAL innodb_file_per_table=1; + +CREATE TABLE ibd4(a INT UNIQUE)ENGINE=InnoDB; +CREATE TABLE ibd4f(a INT UNIQUE)ENGINE=InnoDB; +CREATE TABLE ibd5(a INT UNIQUE, b INT UNIQUE)ENGINE=InnoDB; + +let $drop_tables= DROP TABLE t,ibd4,ibd4f,ibd5; +--let CLEANUP_IF_CHECKPOINT= $drop_tables; +--source ../include/no_checkpoint_end.inc + +perl; +use Fcntl 'SEEK_CUR', 'SEEK_END'; + +my $page_size = $ENV{'INNODB_PAGE_SIZE'}; +my $restart = 'restart'; +if ($ENV{'MYSQLD_IS_DEBUG'}) +{ + # It is impractical to ensure that CREATE TABLE t will extend ibdata1. + # We rely on innodb_system_tablespace_extend_debug=1 + # to recover from this fault injection if no size change was redo-logged. + my $root = $ENV{'INNODB_ROOT_PAGE'}; + open(FILE, "+<", "$ENV{'MYSQLD_DATADIR'}ibdata1") or die; + my $size = sysseek(FILE, 0, SEEK_END) / $page_size; + seek(FILE, $page_size * ($root + 1), SEEK_SET) or die; + my $empty_tail= 1; + while() { unless (/\0*/gso) { $empty_tail= 0; last } } + if ($empty_tail) + { + $restart = 'restart: --innodb-data-file-size-debug=' . $size; + truncate(FILE, $page_size * $root); + } + close FILE; +} +open(FILE, ">$ENV{MYSQLTEST_VARDIR}/log/start_mysqld.txt") || die; +print FILE '--exec echo "', $restart, '" > $_expect_file_name +--enable_reconnect +--source include/wait_until_connected_again.inc +--disable_reconnect +'; +close FILE; +open(FILE, "+<", "$ENV{'MYSQLD_DATADIR'}test/ibd4.ibd") or die; +truncate(FILE, $page_size * 4); +close FILE; +open(FILE, "+<", "$ENV{'MYSQLD_DATADIR'}test/ibd4f.ibd") or die; +truncate(FILE, $page_size * 4 + 1234); +close FILE; +open(FILE, "+<", "$ENV{'MYSQLD_DATADIR'}test/ibd5.ibd") or die; +truncate(FILE, $page_size * 5); +close FILE; +EOF + +--source $MYSQLTEST_VARDIR/log/start_mysqld.txt +--remove_file $MYSQLTEST_VARDIR/log/start_mysqld.txt + +eval $drop_tables; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index f8eacfb3c6cb9..5274523e3a41b 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -579,6 +579,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST NULL READ_ONLY YES COMMAND_LINE_ARGUMENT REQUIRED +VARIABLE_NAME INNODB_DATA_FILE_SIZE_DEBUG +SESSION_VALUE NULL +GLOBAL_VALUE 0 +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE 0 +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_COMMENT InnoDB system tablespace size to be set in recovery. +NUMERIC_MIN_VALUE 0 +NUMERIC_MAX_VALUE 4294967295 +NUMERIC_BLOCK_SIZE 0 +ENUM_VALUE_LIST NULL +READ_ONLY YES +COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_DATA_HOME_DIR SESSION_VALUE NULL GLOBAL_VALUE diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index ce5c62a8c8b73..1e8b2be6f018f 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -921,6 +921,313 @@ fil_try_to_close_file_in_LRU( return(FALSE); } +/** Flush any writes cached by the file system. +@param[in,out] space tablespace */ +static +void +fil_flush_low(fil_space_t* space) +{ + ut_ad(mutex_own(&fil_system->mutex)); + ut_ad(space); + ut_ad(!space->stop_new_ops); + + if (fil_buffering_disabled(space)) { + + /* No need to flush. User has explicitly disabled + buffering. */ + ut_ad(!space->is_in_unflushed_spaces); + ut_ad(fil_space_is_flushed(space)); + ut_ad(space->n_pending_flushes == 0); + +#ifdef UNIV_DEBUG + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + ut_ad(node->modification_counter + == node->flush_counter); + ut_ad(node->n_pending_flushes == 0); + } +#endif /* UNIV_DEBUG */ + + return; + } + + /* Prevent dropping of the space while we are flushing */ + space->n_pending_flushes++; + + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + ib_int64_t old_mod_counter = node->modification_counter; + + if (old_mod_counter <= node->flush_counter) { + continue; + } + + ut_a(node->open); + + if (space->purpose == FIL_TABLESPACE) { + fil_n_pending_tablespace_flushes++; + } else { + fil_n_pending_log_flushes++; + fil_n_log_flushes++; + } +#ifdef __WIN__ + if (node->is_raw_disk) { + + goto skip_flush; + } +#endif /* __WIN__ */ +retry: + if (node->n_pending_flushes > 0) { + /* We want to avoid calling os_file_flush() on + the file twice at the same time, because we do + not know what bugs OS's may contain in file + i/o */ + + ib_int64_t sig_count = + os_event_reset(node->sync_event); + + mutex_exit(&fil_system->mutex); + + os_event_wait_low(node->sync_event, sig_count); + + mutex_enter(&fil_system->mutex); + + if (node->flush_counter >= old_mod_counter) { + + goto skip_flush; + } + + goto retry; + } + + ut_a(node->open); + node->n_pending_flushes++; + + mutex_exit(&fil_system->mutex); + + os_file_flush(node->handle); + + mutex_enter(&fil_system->mutex); + + os_event_set(node->sync_event); + + node->n_pending_flushes--; +skip_flush: + if (node->flush_counter < old_mod_counter) { + node->flush_counter = old_mod_counter; + + if (space->is_in_unflushed_spaces + && fil_space_is_flushed(space)) { + + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE( + unflushed_spaces, + fil_system->unflushed_spaces, + space); + } + } + + if (space->purpose == FIL_TABLESPACE) { + fil_n_pending_tablespace_flushes--; + } else { + fil_n_pending_log_flushes--; + } + } + + space->n_pending_flushes--; +} + +/** Try to extend a tablespace. +@param[in,out] space tablespace to be extended +@param[in,out] node last file of the tablespace +@param[in] size desired size in number of pages +@param[out] success whether the operation succeeded +@return whether the operation should be retried */ +static UNIV_COLD __attribute__((warn_unused_result, nonnull)) +bool +fil_space_extend_must_retry( + fil_space_t* space, + fil_node_t* node, + ulint size, + ibool* success) +{ + ut_ad(mutex_own(&fil_system->mutex)); + ut_ad(UT_LIST_GET_LAST(space->chain) == node); + ut_ad(size >= FIL_IBD_FILE_INITIAL_SIZE); + + *success = space->size >= size; + + if (*success) { + /* Space already big enough */ + return(false); + } + + if (node->being_extended) { + /* Another thread is currently extending the file. Wait + for it to finish. + It'd have been better to use event driven mechanism but + the entire module is peppered with polling stuff. */ + mutex_exit(&fil_system->mutex); + os_thread_sleep(100000); + return(true); + } + + node->being_extended = true; + + if (!fil_node_prepare_for_io(node, fil_system, space)) { + /* The tablespace data file, such as .ibd file, is missing */ + node->being_extended = false; + return(false); + } + + /* At this point it is safe to release fil_system mutex. No + other thread can rename, delete or close the file because + we have set the node->being_extended flag. */ + mutex_exit(&fil_system->mutex); + + ulint start_page_no = space->size; + ulint file_start_page_no = start_page_no - node->size; + + /* Determine correct file block size */ + if (node->file_block_size == 0) { + node->file_block_size = os_file_get_block_size( + node->handle, node->name); + space->file_block_size = node->file_block_size; + } + + ulint page_size = fsp_flags_get_zip_size(space->flags); + ulint pages_added = 0; + + if (!page_size) { + page_size = UNIV_PAGE_SIZE; + } + +#ifdef HAVE_POSIX_FALLOCATE + /* We must complete the I/O request after invoking + posix_fallocate() to avoid an assertion failure at shutdown. + Because no actual writes were dispatched, a read operation + will suffice. */ + const ulint io_completion_type = srv_use_posix_fallocate + ? OS_FILE_READ : OS_FILE_WRITE; + + if (srv_use_posix_fallocate) { + const os_offset_t start_offset = static_cast( + start_page_no) * page_size; + const os_offset_t len = static_cast( + pages_added) * page_size; + + *success = !posix_fallocate(node->handle, start_offset, len); + if (!*success) { + ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " + "space for file \'%s\' failed. Current size " + INT64PF ", desired size " INT64PF, + node->name, start_offset, len+start_offset); + os_file_handle_error_no_exit( + node->name, "posix_fallocate", + FALSE, __FILE__, __LINE__); + } + + DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", + *success = FALSE; errno = 28; + os_has_said_disk_full = TRUE;); + + if (*success) { + os_has_said_disk_full = FALSE; + } else { + pages_added = 0; + } + } else +#else + const ulint io_completion_type = OS_FILE_WRITE; +#endif + { + byte* buf2; + byte* buf; + ulint buf_size; + + /* Extend at most 64 pages at a time */ + buf_size = ut_min(64, size - start_page_no) + * page_size; + buf2 = static_cast(mem_alloc(buf_size + page_size)); + buf = static_cast(ut_align(buf2, page_size)); + + memset(buf, 0, buf_size); + + while (start_page_no < size) { + ulint n_pages + = ut_min(buf_size / page_size, + size - start_page_no); + + os_offset_t offset = static_cast( + start_page_no - file_start_page_no) + * page_size; + + const char* name = node->name == NULL + ? space->name : node->name; + + *success = os_aio(OS_FILE_WRITE, 0, OS_AIO_SYNC, + name, node->handle, buf, + offset, page_size * n_pages, + page_size, node, NULL, 0); + + DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", + *success = FALSE; errno = 28; + os_has_said_disk_full = TRUE;); + + if (*success) { + os_has_said_disk_full = FALSE; + } else { + /* Let us measure the size of the file + to determine how much we were able to + extend it */ + os_offset_t size; + + size = os_file_get_size(node->handle); + ut_a(size != (os_offset_t) -1); + + n_pages = ((ulint) (size / page_size)) + - node->size - pages_added; + + pages_added += n_pages; + break; + } + + start_page_no += n_pages; + pages_added += n_pages; + } + + mem_free(buf2); + } + + mutex_enter(&fil_system->mutex); + + ut_a(node->being_extended); + + space->size += pages_added; + node->size += pages_added; + + fil_node_complete_io(node, fil_system, io_completion_type); + + node->being_extended = FALSE; + + if (space->id == 0) { + ulint pages_per_mb = (1024 * 1024) / page_size; + + /* Keep the last data file size info up to date, rounded to + full megabytes */ + + srv_data_file_sizes[srv_n_data_files - 1] + = (node->size / pages_per_mb) * pages_per_mb; + } + + fil_flush_low(space); + return(false); +} + /*******************************************************************//** Reserves the fil_system mutex and tries to make sure we can open at least one file while holding it. This should be called before calling @@ -932,27 +1239,25 @@ fil_mutex_enter_and_prepare_for_io( ulint space_id) /*!< in: space id */ { fil_space_t* space; - ibool success; - ibool print_info = FALSE; ulint count = 0; ulint count2 = 0; retry: mutex_enter(&fil_system->mutex); - if (space_id == 0 || space_id >= SRV_LOG_SPACE_FIRST_ID) { - /* We keep log files and system tablespace files always open; - this is important in preventing deadlocks in this module, as - a page read completion often performs another read from the - insert buffer. The insert buffer is in tablespace 0, and we - cannot end up waiting in this function. */ - + if (space_id >= SRV_LOG_SPACE_FIRST_ID) { + /* We keep log files always open. */ return; } space = fil_space_get_by_id(space_id); - if (space != NULL && space->stop_ios) { + if (space == NULL) { + return; + } + + if (space->stop_ios) { + ut_ad(space->id != 0); /* We are going to do a rename file and want to stop new i/o's for a while */ @@ -992,76 +1297,81 @@ fil_mutex_enter_and_prepare_for_io( goto retry; } - if (fil_system->n_open < fil_system->max_n_open) { - - return; - } + fil_node_t* node = UT_LIST_GET_LAST(space->chain); - /* If the file is already open, no need to do anything; if the space - does not exist, we handle the situation in the function which called - this function */ + ut_ad(space->id == 0 || node == UT_LIST_GET_FIRST(space->chain)); - if (!space) { - return; - } + if (space->id == 0) { + /* We keep the system tablespace files always open; + this is important in preventing deadlocks in this module, as + a page read completion often performs another read from the + insert buffer. The insert buffer is in tablespace 0, and we + cannot end up waiting in this function. */ + } else if (!node || node->open) { + /* If the file is already open, no need to do + anything; if the space does not exist, we handle the + situation in the function which called this + function */ + } else { + /* Too many files are open, try to close some */ + while (fil_system->n_open >= fil_system->max_n_open) { + if (fil_try_to_close_file_in_LRU(count > 1)) { + /* No problem */ + } else if (count >= 2) { + ib_logf(IB_LOG_LEVEL_WARN, + "innodb_open_files=%lu is exceeded" + " (%lu files stay open)", + fil_system->max_n_open, + fil_system->n_open); + break; + } else { + mutex_exit(&fil_system->mutex); - fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + /* Wake the i/o-handler threads to + make sure pending i/o's are + performed */ + os_aio_simulated_wake_handler_threads(); + os_thread_sleep(20000); - if (!node || node->open) { - return; - } + /* Flush tablespaces so that we can + close modified files in the LRU list */ + fil_flush_file_spaces(FIL_TABLESPACE); - if (count > 1) { - print_info = TRUE; + count++; + goto retry; + } + } } - /* Too many files are open, try to close some */ -close_more: - success = fil_try_to_close_file_in_LRU(print_info); - - if (success && fil_system->n_open >= fil_system->max_n_open) { + if (ulint size = UNIV_UNLIKELY(space->recv_size)) { + ut_ad(node); + ibool success; + if (fil_space_extend_must_retry(space, node, size, &success)) { + goto retry; + } - goto close_more; - } + ut_ad(mutex_own(&fil_system->mutex)); + /* Crash recovery requires the file extension to succeed. */ + ut_a(success); + /* InnoDB data files cannot shrink. */ + ut_a(space->size >= size); - if (fil_system->n_open < fil_system->max_n_open) { - /* Ok */ + /* There could be multiple concurrent I/O requests for + this tablespace (multiple threads trying to extend + this tablespace). - return; - } + Also, fil_space_set_recv_size() may have been invoked + again during the file extension while fil_system->mutex + was not being held by us. - if (count >= 2) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: too many (%lu) files stay open" - " while the maximum\n" - "InnoDB: allowed value would be %lu.\n" - "InnoDB: You may need to raise the value of" - " innodb_open_files in\n" - "InnoDB: my.cnf.\n", - (ulong) fil_system->n_open, - (ulong) fil_system->max_n_open); + Only if space->recv_size matches what we read originally, + reset the field. In this way, a subsequent I/O request + will handle any pending fil_space_set_recv_size(). */ - return; + if (size == space->recv_size) { + space->recv_size = 0; + } } - - mutex_exit(&fil_system->mutex); - -#ifndef UNIV_HOTBACKUP - /* Wake the i/o-handler threads to make sure pending i/o's are - performed */ - os_aio_simulated_wake_handler_threads(); - - os_thread_sleep(20000); -#endif - /* Flush tablespaces so that we can close modified files in the LRU - list */ - - fil_flush_file_spaces(FIL_TABLESPACE); - - count++; - - goto retry; } /*******************************************************************//** @@ -1544,6 +1854,24 @@ fil_space_get_first_path( return(path); } +/** Set the recovered size of a tablespace in pages. +@param id tablespace ID +@param size recovered size in pages */ +UNIV_INTERN +void +fil_space_set_recv_size(ulint id, ulint size) +{ + mutex_enter(&fil_system->mutex); + ut_ad(size); + ut_ad(id < SRV_LOG_SPACE_FIRST_ID); + + if (fil_space_t* space = fil_space_get_space(id)) { + space->recv_size = size; + } + + mutex_exit(&fil_system->mutex); +} + /*******************************************************************//** Returns the size of the space in pages. The tablespace must be cached in the memory cache. @@ -5226,212 +5554,23 @@ fil_extend_space_to_desired_size( extension; if the current space size is bigger than this already, the function does nothing */ { - fil_node_t* node; - fil_space_t* space; - byte* buf2; - byte* buf; - ulint buf_size; - ulint start_page_no; - ulint file_start_page_no; - ulint page_size; - ulint pages_added; - ibool success; - ut_ad(!srv_read_only_mode); -retry: - pages_added = 0; - success = TRUE; - - fil_mutex_enter_and_prepare_for_io(space_id); - - space = fil_space_get_by_id(space_id); - ut_a(space); - - if (space->size >= size_after_extend) { - /* Space already big enough */ - - *actual_size = space->size; - - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - page_size = fsp_flags_get_zip_size(space->flags); - - if (!page_size) { - page_size = UNIV_PAGE_SIZE; - } - - node = UT_LIST_GET_LAST(space->chain); - - if (!node->being_extended) { - /* Mark this node as undergoing extension. This flag - is used by other threads to wait for the extension - opereation to finish. */ - node->being_extended = TRUE; - } else { - /* Another thread is currently extending the file. Wait - for it to finish. - It'd have been better to use event driven mechanism but - the entire module is peppered with polling stuff. */ - mutex_exit(&fil_system->mutex); - os_thread_sleep(100000); - goto retry; - } - - if (!fil_node_prepare_for_io(node, fil_system, space)) { - /* The tablespace data file, such as .ibd file, is missing */ - node->being_extended = false; - mutex_exit(&fil_system->mutex); - - return(false); - } - - /* At this point it is safe to release fil_system mutex. No - other thread can rename, delete or close the file because - we have set the node->being_extended flag. */ - mutex_exit(&fil_system->mutex); - - start_page_no = space->size; - file_start_page_no = space->size - node->size; - - /* Determine correct file block size */ - if (node->file_block_size == 0) { - node->file_block_size = os_file_get_block_size(node->handle, node->name); - space->file_block_size = node->file_block_size; - } - -#ifdef HAVE_POSIX_FALLOCATE - if (srv_use_posix_fallocate) { - os_offset_t start_offset = start_page_no * page_size; - os_offset_t n_pages = (size_after_extend - start_page_no); - os_offset_t len = n_pages * page_size; - - if (posix_fallocate(node->handle, start_offset, len) == -1) { - ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " - "space for file \'%s\' failed. Current size " - INT64PF ", desired size " INT64PF "\n", - node->name, start_offset, len+start_offset); - os_file_handle_error_no_exit(node->name, "posix_fallocate", FALSE, __FILE__, __LINE__); - success = FALSE; - } else { - success = TRUE; - } - - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - success = FALSE; errno = 28; os_has_said_disk_full = TRUE;); - - mutex_enter(&fil_system->mutex); - - if (success) { - node->size += (size_after_extend - start_page_no); - space->size += (size_after_extend - start_page_no); - - os_has_said_disk_full = FALSE; - } - - /* If posix_fallocate was used to extent the file space - we need to complete the io. Because no actual writes were - dispatched read operation is enough here. Without this - there will be assertion at shutdown indicating that - all IO is not completed. */ - fil_node_complete_io(node, fil_system, OS_FILE_READ); - goto file_extended; - } -#endif - - /* Extend at most 64 pages at a time */ - buf_size = ut_min(64, size_after_extend - start_page_no) * page_size; - buf2 = static_cast(mem_alloc(buf_size + page_size)); - buf = static_cast(ut_align(buf2, page_size)); - - memset(buf, 0, buf_size); - - while (start_page_no < size_after_extend) { - ulint n_pages - = ut_min(buf_size / page_size, - size_after_extend - start_page_no); - - os_offset_t offset - = ((os_offset_t) (start_page_no - file_start_page_no)) - * page_size; - - const char* name = node->name == NULL ? space->name : node->name; - -#ifdef UNIV_HOTBACKUP - success = os_file_write(name, node->handle, buf, - offset, page_size * n_pages); -#else - success = os_aio(OS_FILE_WRITE, 0, OS_AIO_SYNC, - name, node->handle, buf, - offset, page_size * n_pages, page_size, - node, NULL, 0); -#endif /* UNIV_HOTBACKUP */ - - - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - success = FALSE; errno = 28; os_has_said_disk_full = TRUE;); - - if (success) { - os_has_said_disk_full = FALSE; - } else { - /* Let us measure the size of the file to determine - how much we were able to extend it */ - os_offset_t size; - - size = os_file_get_size(node->handle); - ut_a(size != (os_offset_t) -1); + for (;;) { + fil_mutex_enter_and_prepare_for_io(space_id); - n_pages = ((ulint) (size / page_size)) - - node->size - pages_added; + fil_space_t* space = fil_space_get_by_id(space_id); + ut_a(space); + ibool success; - pages_added += n_pages; - break; + if (!fil_space_extend_must_retry( + space, UT_LIST_GET_LAST(space->chain), + size_after_extend, &success)) { + *actual_size = space->size; + mutex_exit(&fil_system->mutex); + return(success); } - - start_page_no += n_pages; - pages_added += n_pages; } - - mem_free(buf2); - - mutex_enter(&fil_system->mutex); - - ut_a(node->being_extended); - - space->size += pages_added; - node->size += pages_added; - - fil_node_complete_io(node, fil_system, OS_FILE_WRITE); - - /* At this point file has been extended */ -file_extended: - - node->being_extended = FALSE; - *actual_size = space->size; - -#ifndef UNIV_HOTBACKUP - if (space_id == 0) { - ulint pages_per_mb = (1024 * 1024) / page_size; - - /* Keep the last data file size info up to date, rounded to - full megabytes */ - - srv_data_file_sizes[srv_n_data_files - 1] - = (node->size / pages_per_mb) * pages_per_mb; - } -#endif /* !UNIV_HOTBACKUP */ - - /* - printf("Extended %s to %lu, actual size %lu pages\n", space->name, - size_after_extend, *actual_size); */ - mutex_exit(&fil_system->mutex); - - fil_flush(space_id); - - return(success); } #ifdef UNIV_HOTBACKUP @@ -6123,14 +6262,9 @@ fil_flush( ulint space_id) /*!< in: file space id (this can be a group of log files or a tablespace of the database) */ { - fil_space_t* space; - fil_node_t* node; - os_file_t file; - - mutex_enter(&fil_system->mutex); - space = fil_space_get_by_id(space_id); + fil_space_t* space = fil_space_get_by_id(space_id); if (!space || space->stop_new_ops) { mutex_exit(&fil_system->mutex); @@ -6138,115 +6272,7 @@ fil_flush( return; } - if (fil_buffering_disabled(space)) { - - /* No need to flush. User has explicitly disabled - buffering. */ - ut_ad(!space->is_in_unflushed_spaces); - ut_ad(fil_space_is_flushed(space)); - ut_ad(space->n_pending_flushes == 0); - -#ifdef UNIV_DEBUG - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { - ut_ad(node->modification_counter - == node->flush_counter); - ut_ad(node->n_pending_flushes == 0); - } -#endif /* UNIV_DEBUG */ - - mutex_exit(&fil_system->mutex); - return; - } - - space->n_pending_flushes++; /*!< prevent dropping of the space while - we are flushing */ - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { - - ib_int64_t old_mod_counter = node->modification_counter; - - if (old_mod_counter <= node->flush_counter) { - continue; - } - - ut_a(node->open); - - if (space->purpose == FIL_TABLESPACE) { - fil_n_pending_tablespace_flushes++; - } else { - fil_n_pending_log_flushes++; - fil_n_log_flushes++; - } -#ifdef __WIN__ - if (node->is_raw_disk) { - - goto skip_flush; - } -#endif /* __WIN__ */ -retry: - if (node->n_pending_flushes > 0) { - /* We want to avoid calling os_file_flush() on - the file twice at the same time, because we do - not know what bugs OS's may contain in file - i/o */ - - ib_int64_t sig_count = - os_event_reset(node->sync_event); - - mutex_exit(&fil_system->mutex); - - os_event_wait_low(node->sync_event, sig_count); - - mutex_enter(&fil_system->mutex); - - if (node->flush_counter >= old_mod_counter) { - - goto skip_flush; - } - - goto retry; - } - - ut_a(node->open); - file = node->handle; - node->n_pending_flushes++; - - mutex_exit(&fil_system->mutex); - - os_file_flush(file); - - mutex_enter(&fil_system->mutex); - - os_event_set(node->sync_event); - - node->n_pending_flushes--; -skip_flush: - if (node->flush_counter < old_mod_counter) { - node->flush_counter = old_mod_counter; - - if (space->is_in_unflushed_spaces - && fil_space_is_flushed(space)) { - - space->is_in_unflushed_spaces = false; - - UT_LIST_REMOVE( - unflushed_spaces, - fil_system->unflushed_spaces, - space); - } - } - - if (space->purpose == FIL_TABLESPACE) { - fil_n_pending_tablespace_flushes--; - } else { - fil_n_pending_log_flushes--; - } - } - - space->n_pending_flushes--; + fil_flush_low(space); mutex_exit(&fil_system->mutex); } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 35d7ef7ee6fbc..3713dd959d45d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -3453,14 +3453,15 @@ innobase_init( if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_DEF) { ib_logf(IB_LOG_LEVEL_INFO, "innodb_page_size has been " - "changed from default value %d to %ldd.", + "changed from default value %d to %ld.", UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE); /* There is hang on buffer pool when trying to get a new page if buffer pool size is too small for large page sizes */ - if (innobase_buffer_pool_size < (24 * 1024 * 1024)) { - ib_logf(IB_LOG_LEVEL_INFO, - "innobase_page_size %lu requires " + if (UNIV_PAGE_SIZE > UNIV_PAGE_SIZE_DEF + && innobase_buffer_pool_size < (24 * 1024 * 1024)) { + ib_logf(IB_LOG_LEVEL_ERROR, + "innodb_page_size=%lu requires " "innodb_buffer_pool_size > 24M current %lld", UNIV_PAGE_SIZE, innobase_buffer_pool_size); goto error; @@ -19692,6 +19693,12 @@ static MYSQL_SYSVAR_BOOL(trx_purge_view_update_only_debug, "but the each purges were not done yet.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_ULONG(data_file_size_debug, + srv_sys_space_size_debug, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "InnoDB system tablespace size to be set in recovery.", + NULL, NULL, 0, 0, UINT_MAX32, 0); + static MYSQL_SYSVAR_ULONG(fil_make_page_dirty_debug, srv_fil_make_page_dirty_debug, PLUGIN_VAR_OPCMDARG, "Make the first page of the given tablespace dirty.", @@ -20043,6 +20050,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(trx_rseg_n_slots_debug), MYSQL_SYSVAR(limit_optimistic_insert_debug), MYSQL_SYSVAR(trx_purge_view_update_only_debug), + MYSQL_SYSVAR(data_file_size_debug), MYSQL_SYSVAR(fil_make_page_dirty_debug), MYSQL_SYSVAR(saved_page_number_debug), #endif /* UNIV_DEBUG */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index ae8224d77bb25..cc67d918d5f20 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -299,6 +299,10 @@ struct fil_space_t { tablespace whose size we do not know yet; last incomplete megabytes in data files may be ignored if space == 0 */ + ulint recv_size; + /*!< recovered tablespace size in pages; + 0 if no size change was read from the redo log, + or if the size change was implemented */ ulint flags; /*!< tablespace flags; see fsp_flags_is_valid(), fsp_flags_get_zip_size() */ @@ -500,6 +504,12 @@ char* fil_space_get_first_path( /*=====================*/ ulint id); /*!< in: space id */ +/** Set the recovered size of a tablespace in pages. +@param id tablespace ID +@param size recovered size in pages */ +UNIV_INTERN +void +fil_space_set_recv_size(ulint id, ulint size); /*******************************************************************//** Returns the size of the space in pages. The tablespace must be cached in the memory cache. diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index a5713afbd4934..905cc80f0dfa3 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -524,6 +524,7 @@ extern my_bool srv_ibuf_disable_background_merge; #ifdef UNIV_DEBUG extern my_bool srv_purge_view_update_only_debug; +extern ulong srv_sys_space_size_debug; #endif /* UNIV_DEBUG */ #define SRV_SEMAPHORE_WAIT_EXTENSION 7200 diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 9fde18757c54c..926f5f5ff3426 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2170,6 +2170,7 @@ recv_parse_log_rec( } #endif /* UNIV_LOG_LSN_DEBUG */ + byte* old_ptr = new_ptr; new_ptr = recv_parse_or_apply_log_rec_body(*type, new_ptr, end_ptr, NULL, NULL, *space); if (UNIV_UNLIKELY(new_ptr == NULL)) { @@ -2177,6 +2178,13 @@ recv_parse_log_rec( return(0); } + if (*page_no == 0 && *type == MLOG_4BYTES + && mach_read_from_2(old_ptr) == FSP_HEADER_OFFSET + FSP_SIZE) { + ulint size; + mach_parse_compressed(old_ptr + 2, end_ptr, &size); + fil_space_set_recv_size(*space, size); + } + if (*page_no > recv_max_parsed_page_no) { recv_max_parsed_page_no = *page_no; } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index bbb9dc0205ed8..acfce27499267 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -127,6 +127,10 @@ UNIV_INTERN ibool srv_is_being_started = FALSE; UNIV_INTERN ibool srv_was_started = FALSE; /** TRUE if innobase_start_or_create_for_mysql() has been called */ static ibool srv_start_has_been_called = FALSE; +#ifdef UNIV_DEBUG +/** InnoDB system tablespace to set during recovery */ +UNIV_INTERN ulong srv_sys_space_size_debug; +#endif /* UNIV_DEBUG */ /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ @@ -171,9 +175,6 @@ static const ulint SRV_UNDO_TABLESPACE_SIZE_IN_PAGES = #define SRV_N_PENDING_IOS_PER_THREAD OS_AIO_N_PENDING_IOS_PER_THREAD #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/** The round off to MB is similar as done in srv_parse_megabytes() */ -#define CALC_NUMBER_OF_PAGES(size) ((size) / (1024 * 1024)) * \ - ((1024 * 1024) / (UNIV_PAGE_SIZE)) #ifdef UNIV_PFS_THREAD /* Keys to register InnoDB threads with performance schema */ UNIV_INTERN mysql_pfs_key_t io_handler_thread_key; @@ -988,15 +989,12 @@ open_or_create_data_files( size = os_file_get_size(files[i]); ut_a(size != (os_offset_t) -1); - /* Under some error conditions like disk full - narios or file size reaching filesystem - limit the data file could contain an incomplete - extent at the end. When we extend a data file - and if some failure happens, then also the data - file could contain an incomplete extent. So we - need to round the size downward to a megabyte.*/ + /* If InnoDB encountered an error or was killed + while extending the data file, the last page + could be incomplete. */ - rounded_size_pages = (ulint) CALC_NUMBER_OF_PAGES(size); + rounded_size_pages = static_cast( + size >> UNIV_PAGE_SIZE_SHIFT); if (i == srv_n_data_files - 1 && srv_auto_extend_last_data_file) { @@ -2112,9 +2110,11 @@ innobase_start_or_create_for_mysql(void) sum_of_new_sizes += srv_data_file_sizes[i]; } - if (sum_of_new_sizes < 10485760 / UNIV_PAGE_SIZE) { + if (!srv_auto_extend_last_data_file && sum_of_new_sizes < 640) { ib_logf(IB_LOG_LEVEL_ERROR, - "Tablespace size must be at least 10 MB"); + "Combined size in innodb_data_file_path" + " must be at least %u MiB", + 640 >> (20 - UNIV_PAGE_SIZE_SHIFT)); return(DB_ERROR); } @@ -2182,6 +2182,8 @@ innobase_start_or_create_for_mysql(void) return(err); } } else { + ut_d(fil_space_get(0)->recv_size = srv_sys_space_size_debug); + for (i = 0; i < SRV_N_LOG_FILES_MAX; i++) { os_offset_t size; os_file_stat_t stat_info; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index e7da4569f0d1f..133960ae8b4a6 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -925,6 +925,314 @@ fil_try_to_close_file_in_LRU( return(FALSE); } +/** Flush any writes cached by the file system. +@param[in,out] space tablespace */ +static +void +fil_flush_low(fil_space_t* space) +{ + ut_ad(mutex_own(&fil_system->mutex)); + ut_ad(space); + ut_ad(!space->stop_new_ops); + + if (fil_buffering_disabled(space)) { + + /* No need to flush. User has explicitly disabled + buffering. */ + ut_ad(!space->is_in_unflushed_spaces); + ut_ad(fil_space_is_flushed(space)); + ut_ad(space->n_pending_flushes == 0); + +#ifdef UNIV_DEBUG + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + ut_ad(node->modification_counter + == node->flush_counter); + ut_ad(node->n_pending_flushes == 0); + } +#endif /* UNIV_DEBUG */ + + return; + } + + /* Prevent dropping of the space while we are flushing */ + space->n_pending_flushes++; + + for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + ib_int64_t old_mod_counter = node->modification_counter; + + if (old_mod_counter <= node->flush_counter) { + continue; + } + + ut_a(node->open); + + if (space->purpose == FIL_TABLESPACE) { + fil_n_pending_tablespace_flushes++; + } else { + fil_n_pending_log_flushes++; + fil_n_log_flushes++; + } +#ifdef __WIN__ + if (node->is_raw_disk) { + + goto skip_flush; + } +#endif /* __WIN__ */ +retry: + if (node->n_pending_flushes > 0) { + /* We want to avoid calling os_file_flush() on + the file twice at the same time, because we do + not know what bugs OS's may contain in file + i/o */ + + ib_int64_t sig_count = + os_event_reset(node->sync_event); + + mutex_exit(&fil_system->mutex); + + os_event_wait_low(node->sync_event, sig_count); + + mutex_enter(&fil_system->mutex); + + if (node->flush_counter >= old_mod_counter) { + + goto skip_flush; + } + + goto retry; + } + + ut_a(node->open); + node->n_pending_flushes++; + + mutex_exit(&fil_system->mutex); + + os_file_flush(node->handle); + + mutex_enter(&fil_system->mutex); + + os_event_set(node->sync_event); + + node->n_pending_flushes--; +skip_flush: + if (node->flush_counter < old_mod_counter) { + node->flush_counter = old_mod_counter; + + if (space->is_in_unflushed_spaces + && fil_space_is_flushed(space)) { + + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE( + unflushed_spaces, + fil_system->unflushed_spaces, + space); + } + } + + if (space->purpose == FIL_TABLESPACE) { + fil_n_pending_tablespace_flushes--; + } else { + fil_n_pending_log_flushes--; + } + } + + space->n_pending_flushes--; +} + +/** Try to extend a tablespace. +@param[in,out] space tablespace to be extended +@param[in,out] node last file of the tablespace +@param[in] size desired size in number of pages +@param[out] success whether the operation succeeded +@return whether the operation should be retried */ +static UNIV_COLD __attribute__((warn_unused_result, nonnull)) +bool +fil_space_extend_must_retry( + fil_space_t* space, + fil_node_t* node, + ulint size, + ibool* success) +{ + ut_ad(mutex_own(&fil_system->mutex)); + ut_ad(UT_LIST_GET_LAST(space->chain) == node); + ut_ad(size >= FIL_IBD_FILE_INITIAL_SIZE); + + *success = space->size >= size; + + if (*success) { + /* Space already big enough */ + return(false); + } + + if (node->being_extended) { + /* Another thread is currently extending the file. Wait + for it to finish. + It'd have been better to use event driven mechanism but + the entire module is peppered with polling stuff. */ + mutex_exit(&fil_system->mutex); + os_thread_sleep(100000); + return(true); + } + + node->being_extended = true; + + if (!fil_node_prepare_for_io(node, fil_system, space)) { + /* The tablespace data file, such as .ibd file, is missing */ + node->being_extended = false; + return(false); + } + + /* At this point it is safe to release fil_system mutex. No + other thread can rename, delete or close the file because + we have set the node->being_extended flag. */ + mutex_exit(&fil_system->mutex); + + ulint start_page_no = space->size; + ulint file_start_page_no = start_page_no - node->size; + + /* Determine correct file block size */ + if (node->file_block_size == 0) { + node->file_block_size = os_file_get_block_size( + node->handle, node->name); + space->file_block_size = node->file_block_size; + } + + ulint page_size = fsp_flags_get_zip_size(space->flags); + ulint pages_added = 0; + + if (!page_size) { + page_size = UNIV_PAGE_SIZE; + } + +#ifdef HAVE_POSIX_FALLOCATE + /* We must complete the I/O request after invoking + posix_fallocate() to avoid an assertion failure at shutdown. + Because no actual writes were dispatched, a read operation + will suffice. */ + const ulint io_completion_type = srv_use_posix_fallocate + ? OS_FILE_READ : OS_FILE_WRITE; + + if (srv_use_posix_fallocate) { + const os_offset_t start_offset = static_cast( + start_page_no) * page_size; + const os_offset_t len = static_cast( + pages_added) * page_size; + + *success = !posix_fallocate(node->handle, start_offset, len); + if (!*success) { + ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " + "space for file \'%s\' failed. Current size " + INT64PF ", desired size " INT64PF, + node->name, start_offset, len+start_offset); + os_file_handle_error_no_exit( + node->name, "posix_fallocate", + FALSE, __FILE__, __LINE__); + } + + DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", + *success = FALSE; errno = 28; + os_has_said_disk_full = TRUE;); + + if (*success) { + os_has_said_disk_full = FALSE; + } else { + pages_added = 0; + } + } else +#else + const ulint io_completion_type = OS_FILE_WRITE; +#endif + { + byte* buf2; + byte* buf; + ulint buf_size; + + /* Extend at most 64 pages at a time */ + buf_size = ut_min(64, size - start_page_no) + * page_size; + buf2 = static_cast(mem_alloc(buf_size + page_size)); + buf = static_cast(ut_align(buf2, page_size)); + + memset(buf, 0, buf_size); + + while (start_page_no < size) { + ulint n_pages + = ut_min(buf_size / page_size, + size - start_page_no); + + os_offset_t offset = static_cast( + start_page_no - file_start_page_no) + * page_size; + + const char* name = node->name == NULL + ? space->name : node->name; + + *success = os_aio(OS_FILE_WRITE, 0, OS_AIO_SYNC, + name, node->handle, buf, + offset, page_size * n_pages, + page_size, node, NULL, + space->id, NULL, 0); + + DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", + *success = FALSE; errno = 28; + os_has_said_disk_full = TRUE;); + + if (*success) { + os_has_said_disk_full = FALSE; + } else { + /* Let us measure the size of the file + to determine how much we were able to + extend it */ + os_offset_t size; + + size = os_file_get_size(node->handle); + ut_a(size != (os_offset_t) -1); + + n_pages = ((ulint) (size / page_size)) + - node->size - pages_added; + + pages_added += n_pages; + break; + } + + start_page_no += n_pages; + pages_added += n_pages; + } + + mem_free(buf2); + } + + mutex_enter(&fil_system->mutex); + + ut_a(node->being_extended); + + space->size += pages_added; + node->size += pages_added; + + fil_node_complete_io(node, fil_system, io_completion_type); + + node->being_extended = FALSE; + + if (space->id == 0) { + ulint pages_per_mb = (1024 * 1024) / page_size; + + /* Keep the last data file size info up to date, rounded to + full megabytes */ + + srv_data_file_sizes[srv_n_data_files - 1] + = (node->size / pages_per_mb) * pages_per_mb; + } + + fil_flush_low(space); + return(false); +} + /*******************************************************************//** Reserves the fil_system mutex and tries to make sure we can open at least one file while holding it. This should be called before calling @@ -936,27 +1244,25 @@ fil_mutex_enter_and_prepare_for_io( ulint space_id) /*!< in: space id */ { fil_space_t* space; - ibool success; - ibool print_info = FALSE; ulint count = 0; ulint count2 = 0; retry: mutex_enter(&fil_system->mutex); - if (space_id == 0 || space_id >= SRV_LOG_SPACE_FIRST_ID) { - /* We keep log files and system tablespace files always open; - this is important in preventing deadlocks in this module, as - a page read completion often performs another read from the - insert buffer. The insert buffer is in tablespace 0, and we - cannot end up waiting in this function. */ - + if (space_id >= SRV_LOG_SPACE_FIRST_ID) { + /* We keep log files always open. */ return; } space = fil_space_get_by_id(space_id); - if (space != NULL && space->stop_ios) { + if (space == NULL) { + return; + } + + if (space->stop_ios) { + ut_ad(space->id != 0); /* We are going to do a rename file and want to stop new i/o's for a while */ @@ -996,76 +1302,81 @@ fil_mutex_enter_and_prepare_for_io( goto retry; } - if (fil_system->n_open < fil_system->max_n_open) { + fil_node_t* node = UT_LIST_GET_LAST(space->chain); - return; - } + ut_ad(space->id == 0 || node == UT_LIST_GET_FIRST(space->chain)); - /* If the file is already open, no need to do anything; if the space - does not exist, we handle the situation in the function which called - this function */ - if (!space) { - return; - } - - fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + if (space->id == 0) { + /* We keep the system tablespace files always open; + this is important in preventing deadlocks in this module, as + a page read completion often performs another read from the + insert buffer. The insert buffer is in tablespace 0, and we + cannot end up waiting in this function. */ + } else if (!node || node->open) { + /* If the file is already open, no need to do + anything; if the space does not exist, we handle the + situation in the function which called this + function */ + } else { + /* Too many files are open, try to close some */ + while (fil_system->n_open >= fil_system->max_n_open) { + if (fil_try_to_close_file_in_LRU(count > 1)) { + /* No problem */ + } else if (count >= 2) { + ib_logf(IB_LOG_LEVEL_WARN, + "innodb_open_files=%lu is exceeded" + " (%lu files stay open)", + fil_system->max_n_open, + fil_system->n_open); + break; + } else { + mutex_exit(&fil_system->mutex); - if (!node || node->open) { + /* Wake the i/o-handler threads to + make sure pending i/o's are + performed */ + os_aio_simulated_wake_handler_threads(); + os_thread_sleep(20000); - return; - } + /* Flush tablespaces so that we can + close modified files in the LRU list */ + fil_flush_file_spaces(FIL_TABLESPACE); - if (count > 1) { - print_info = TRUE; + count++; + goto retry; + } + } } - /* Too many files are open, try to close some */ -close_more: - success = fil_try_to_close_file_in_LRU(print_info); + if (ulint size = UNIV_UNLIKELY(space->recv_size)) { + ut_ad(node); + ibool success; + if (fil_space_extend_must_retry(space, node, size, &success)) { + goto retry; + } - if (success && fil_system->n_open >= fil_system->max_n_open) { + ut_ad(mutex_own(&fil_system->mutex)); + /* Crash recovery requires the file extension to succeed. */ + ut_a(success); + /* InnoDB data files cannot shrink. */ + ut_a(space->size >= size); - goto close_more; - } + /* There could be multiple concurrent I/O requests for + this tablespace (multiple threads trying to extend + this tablespace). - if (fil_system->n_open < fil_system->max_n_open) { - /* Ok */ + Also, fil_space_set_recv_size() may have been invoked + again during the file extension while fil_system->mutex + was not being held by us. - return; - } + Only if space->recv_size matches what we read originally, + reset the field. In this way, a subsequent I/O request + will handle any pending fil_space_set_recv_size(). */ - if (count >= 2) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: too many (%lu) files stay open" - " while the maximum\n" - "InnoDB: allowed value would be %lu.\n" - "InnoDB: You may need to raise the value of" - " innodb_open_files in\n" - "InnoDB: my.cnf.\n", - (ulong) fil_system->n_open, - (ulong) fil_system->max_n_open); - - return; + if (size == space->recv_size) { + space->recv_size = 0; + } } - - mutex_exit(&fil_system->mutex); - -#ifndef UNIV_HOTBACKUP - /* Wake the i/o-handler threads to make sure pending i/o's are - performed */ - os_aio_simulated_wake_handler_threads(); - - os_thread_sleep(20000); -#endif - /* Flush tablespaces so that we can close modified files in the LRU - list */ - - fil_flush_file_spaces(FIL_TABLESPACE); - - count++; - - goto retry; } /*******************************************************************//** @@ -1582,6 +1893,24 @@ fil_space_get_first_path( return(path); } +/** Set the recovered size of a tablespace in pages. +@param id tablespace ID +@param size recovered size in pages */ +UNIV_INTERN +void +fil_space_set_recv_size(ulint id, ulint size) +{ + mutex_enter(&fil_system->mutex); + ut_ad(size); + ut_ad(id < SRV_LOG_SPACE_FIRST_ID); + + if (fil_space_t* space = fil_space_get_space(id)) { + space->recv_size = size; + } + + mutex_exit(&fil_system->mutex); +} + /*******************************************************************//** Returns the size of the space in pages. The tablespace must be cached in the memory cache. @@ -5263,209 +5592,23 @@ fil_extend_space_to_desired_size( extension; if the current space size is bigger than this already, the function does nothing */ { - fil_node_t* node; - fil_space_t* space; - byte* buf2; - byte* buf; - ulint buf_size; - ulint start_page_no; - ulint file_start_page_no; - ulint page_size; - ulint pages_added; - ibool success; - ut_ad(!srv_read_only_mode); -retry: - pages_added = 0; - success = TRUE; - - fil_mutex_enter_and_prepare_for_io(space_id); - - space = fil_space_get_by_id(space_id); - ut_a(space); - - if (space->size >= size_after_extend) { - /* Space already big enough */ - - *actual_size = space->size; - - mutex_exit(&fil_system->mutex); - - return(TRUE); - } - - page_size = fsp_flags_get_zip_size(space->flags); - if (!page_size) { - page_size = UNIV_PAGE_SIZE; - } - - node = UT_LIST_GET_LAST(space->chain); - - if (!node->being_extended) { - /* Mark this node as undergoing extension. This flag - is used by other threads to wait for the extension - opereation to finish. */ - node->being_extended = TRUE; - } else { - /* Another thread is currently extending the file. Wait - for it to finish. - It'd have been better to use event driven mechanism but - the entire module is peppered with polling stuff. */ - mutex_exit(&fil_system->mutex); - os_thread_sleep(100000); - goto retry; - } - - if (!fil_node_prepare_for_io(node, fil_system, space)) { - /* The tablespace data file, such as .ibd file, is missing */ - node->being_extended = false; - mutex_exit(&fil_system->mutex); - - return(false); - } - - /* At this point it is safe to release fil_system mutex. No - other thread can rename, delete or close the file because - we have set the node->being_extended flag. */ - mutex_exit(&fil_system->mutex); - - start_page_no = space->size; - file_start_page_no = space->size - node->size; - - /* Determine correct file block size */ - if (node->file_block_size == 0) { - node->file_block_size = os_file_get_block_size(node->handle, node->name); - space->file_block_size = node->file_block_size; - } - -#ifdef HAVE_POSIX_FALLOCATE - if (srv_use_posix_fallocate) { - os_offset_t start_offset = start_page_no * page_size; - os_offset_t n_pages = (size_after_extend - start_page_no); - os_offset_t len = n_pages * page_size; - - if (posix_fallocate(node->handle, start_offset, len) == -1) { - ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " - "space for file \'%s\' failed. Current size " - INT64PF ", desired size " INT64PF, - node->name, start_offset, len+start_offset); - os_file_handle_error_no_exit(node->name, "posix_fallocate", FALSE, __FILE__, __LINE__); - success = FALSE; - } else { - success = TRUE; - } - - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - success = FALSE; errno = 28;os_has_said_disk_full = TRUE;); - - mutex_enter(&fil_system->mutex); - - if (success) { - node->size += n_pages; - space->size += n_pages; - os_has_said_disk_full = FALSE; - } - - /* If posix_fallocate was used to extent the file space - we need to complete the io. Because no actual writes were - dispatched read operation is enough here. Without this - there will be assertion at shutdown indicating that - all IO is not completed. */ - fil_node_complete_io(node, fil_system, OS_FILE_READ); - goto file_extended; - } -#endif - - /* Extend at most 64 pages at a time */ - buf_size = ut_min(64, size_after_extend - start_page_no) * page_size; - buf2 = static_cast(mem_alloc(buf_size + page_size)); - buf = static_cast(ut_align(buf2, page_size)); - - memset(buf, 0, buf_size); - - while (start_page_no < size_after_extend) { - ulint n_pages - = ut_min(buf_size / page_size, - size_after_extend - start_page_no); - - os_offset_t offset - = ((os_offset_t) (start_page_no - file_start_page_no)) - * page_size; - - const char* name = node->name == NULL ? space->name : node->name; - -#ifdef UNIV_HOTBACKUP - success = os_file_write(name, node->handle, buf, - offset, page_size * n_pages); -#else - success = os_aio(OS_FILE_WRITE, 0, OS_AIO_SYNC, - name, node->handle, buf, - offset, page_size * n_pages, page_size, - node, NULL, space_id, NULL, 0); -#endif /* UNIV_HOTBACKUP */ - - DBUG_EXECUTE_IF("ib_os_aio_func_io_failure_28", - success = FALSE; errno = 28; os_has_said_disk_full = TRUE;); - - if (success) { - os_has_said_disk_full = FALSE; - } else { - /* Let us measure the size of the file to determine - how much we were able to extend it */ - os_offset_t size; - - size = os_file_get_size(node->handle); - ut_a(size != (os_offset_t) -1); + for (;;) { + fil_mutex_enter_and_prepare_for_io(space_id); - n_pages = ((ulint) (size / page_size)) - - node->size - pages_added; + fil_space_t* space = fil_space_get_by_id(space_id); + ut_a(space); + ibool success; - pages_added += n_pages; - break; + if (!fil_space_extend_must_retry( + space, UT_LIST_GET_LAST(space->chain), + size_after_extend, &success)) { + *actual_size = space->size; + mutex_exit(&fil_system->mutex); + return(success); } - - start_page_no += n_pages; - pages_added += n_pages; - } - - mem_free(buf2); - - mutex_enter(&fil_system->mutex); - - ut_a(node->being_extended); - - space->size += pages_added; - node->size += pages_added; - - fil_node_complete_io(node, fil_system, OS_FILE_WRITE); - - /* At this point file has been extended */ -file_extended: - - node->being_extended = FALSE; - *actual_size = space->size; - -#ifndef UNIV_HOTBACKUP - if (space_id == 0) { - ulint pages_per_mb = (1024 * 1024) / page_size; - - /* Keep the last data file size info up to date, rounded to - full megabytes */ - - srv_data_file_sizes[srv_n_data_files - 1] - = (node->size / pages_per_mb) * pages_per_mb; } -#endif /* !UNIV_HOTBACKUP */ - - /* - printf("Extended %s to %lu, actual size %lu pages\n", space->name, - size_after_extend, *actual_size); */ - mutex_exit(&fil_system->mutex); - - fil_flush(space_id); - - return(success); } #ifdef UNIV_HOTBACKUP @@ -6184,14 +6327,9 @@ fil_flush( ulint space_id) /*!< in: file space id (this can be a group of log files or a tablespace of the database) */ { - fil_space_t* space; - fil_node_t* node; - os_file_t file; - - mutex_enter(&fil_system->mutex); - space = fil_space_get_by_id(space_id); + fil_space_t* space = fil_space_get_by_id(space_id); if (!space || space->stop_new_ops) { mutex_exit(&fil_system->mutex); @@ -6199,115 +6337,7 @@ fil_flush( return; } - if (fil_buffering_disabled(space)) { - - /* No need to flush. User has explicitly disabled - buffering. */ - ut_ad(!space->is_in_unflushed_spaces); - ut_ad(fil_space_is_flushed(space)); - ut_ad(space->n_pending_flushes == 0); - -#ifdef UNIV_DEBUG - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { - ut_ad(node->modification_counter - == node->flush_counter); - ut_ad(node->n_pending_flushes == 0); - } -#endif /* UNIV_DEBUG */ - - mutex_exit(&fil_system->mutex); - return; - } - - space->n_pending_flushes++; /*!< prevent dropping of the space while - we are flushing */ - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { - - ib_int64_t old_mod_counter = node->modification_counter; - - if (old_mod_counter <= node->flush_counter) { - continue; - } - - ut_a(node->open); - - if (space->purpose == FIL_TABLESPACE) { - fil_n_pending_tablespace_flushes++; - } else { - fil_n_pending_log_flushes++; - fil_n_log_flushes++; - } -#ifdef __WIN__ - if (node->is_raw_disk) { - - goto skip_flush; - } -#endif /* __WIN__ */ -retry: - if (node->n_pending_flushes > 0) { - /* We want to avoid calling os_file_flush() on - the file twice at the same time, because we do - not know what bugs OS's may contain in file - i/o */ - - ib_int64_t sig_count = - os_event_reset(node->sync_event); - - mutex_exit(&fil_system->mutex); - - os_event_wait_low(node->sync_event, sig_count); - - mutex_enter(&fil_system->mutex); - - if (node->flush_counter >= old_mod_counter) { - - goto skip_flush; - } - - goto retry; - } - - ut_a(node->open); - file = node->handle; - node->n_pending_flushes++; - - mutex_exit(&fil_system->mutex); - - os_file_flush(file); - - mutex_enter(&fil_system->mutex); - - os_event_set(node->sync_event); - - node->n_pending_flushes--; -skip_flush: - if (node->flush_counter < old_mod_counter) { - node->flush_counter = old_mod_counter; - - if (space->is_in_unflushed_spaces - && fil_space_is_flushed(space)) { - - space->is_in_unflushed_spaces = false; - - UT_LIST_REMOVE( - unflushed_spaces, - fil_system->unflushed_spaces, - space); - } - } - - if (space->purpose == FIL_TABLESPACE) { - fil_n_pending_tablespace_flushes--; - } else { - fil_n_pending_log_flushes--; - } - } - - space->n_pending_flushes--; + fil_flush_low(space); mutex_exit(&fil_system->mutex); } diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 984d508bd0424..8d564df2bb3eb 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3906,14 +3906,15 @@ innobase_init( if (UNIV_PAGE_SIZE != UNIV_PAGE_SIZE_DEF) { ib_logf(IB_LOG_LEVEL_INFO, "innodb_page_size has been " - "changed from default value %d to %ldd.", + "changed from default value %d to %ld.", UNIV_PAGE_SIZE_DEF, UNIV_PAGE_SIZE); /* There is hang on buffer pool when trying to get a new page if buffer pool size is too small for large page sizes */ - if (innobase_buffer_pool_size < (24 * 1024 * 1024)) { - ib_logf(IB_LOG_LEVEL_INFO, - "innobase_page_size %lu requires " + if (UNIV_PAGE_SIZE > UNIV_PAGE_SIZE_DEF + && innobase_buffer_pool_size < (24 * 1024 * 1024)) { + ib_logf(IB_LOG_LEVEL_ERROR, + "innodb_page_size=%lu requires " "innodb_buffer_pool_size > 24M current %lld", UNIV_PAGE_SIZE, innobase_buffer_pool_size); @@ -21560,6 +21561,12 @@ static MYSQL_SYSVAR_BOOL(trx_purge_view_update_only_debug, "but the each purges were not done yet.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_ULONG(data_file_size_debug, + srv_sys_space_size_debug, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "InnoDB system tablespace size to be set in recovery.", + NULL, NULL, 0, 0, UINT_MAX32, 0); + static MYSQL_SYSVAR_ULONG(fil_make_page_dirty_debug, srv_fil_make_page_dirty_debug, PLUGIN_VAR_OPCMDARG, "Make the first page of the given tablespace dirty.", @@ -21998,6 +22005,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(trx_rseg_n_slots_debug), MYSQL_SYSVAR(limit_optimistic_insert_debug), MYSQL_SYSVAR(trx_purge_view_update_only_debug), + MYSQL_SYSVAR(data_file_size_debug), MYSQL_SYSVAR(fil_make_page_dirty_debug), MYSQL_SYSVAR(saved_page_number_debug), #endif /* UNIV_DEBUG */ diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 95011ae61250d..38cc09bced3d6 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -292,6 +292,10 @@ struct fil_space_t { tablespace whose size we do not know yet; last incomplete megabytes in data files may be ignored if space == 0 */ + ulint recv_size; + /*!< recovered tablespace size in pages; + 0 if no size change was read from the redo log, + or if the size change was implemented */ ulint flags; /*!< tablespace flags; see fsp_flags_is_valid(), fsp_flags_get_zip_size() */ @@ -502,6 +506,12 @@ char* fil_space_get_first_path( /*=====================*/ ulint id); /*!< in: space id */ +/** Set the recovered size of a tablespace in pages. +@param id tablespace ID +@param size recovered size in pages */ +UNIV_INTERN +void +fil_space_set_recv_size(ulint id, ulint size); /*******************************************************************//** Returns the size of the space in pages. The tablespace must be cached in the memory cache. diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index aa7e0452792a5..e2d141b414092 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -626,6 +626,7 @@ extern my_bool srv_ibuf_disable_background_merge; #ifdef UNIV_DEBUG extern my_bool srv_purge_view_update_only_debug; +extern ulong srv_sys_space_size_debug; #endif /* UNIV_DEBUG */ #define SRV_SEMAPHORE_WAIT_EXTENSION 7200 diff --git a/storage/xtradb/log/log0recv.cc b/storage/xtradb/log/log0recv.cc index 092c2ed88dc18..1777084e74610 100644 --- a/storage/xtradb/log/log0recv.cc +++ b/storage/xtradb/log/log0recv.cc @@ -2254,6 +2254,7 @@ recv_parse_log_rec( } #endif /* UNIV_LOG_LSN_DEBUG */ + byte* old_ptr = new_ptr; new_ptr = recv_parse_or_apply_log_rec_body(*type, new_ptr, end_ptr, NULL, NULL, *space); if (UNIV_UNLIKELY(new_ptr == NULL)) { @@ -2261,6 +2262,13 @@ recv_parse_log_rec( return(0); } + if (*page_no == 0 && *type == MLOG_4BYTES + && mach_read_from_2(old_ptr) == FSP_HEADER_OFFSET + FSP_SIZE) { + ulint size; + mach_parse_compressed(old_ptr + 2, end_ptr, &size); + fil_space_set_recv_size(*space, size); + } + if (*page_no > recv_max_parsed_page_no) { recv_max_parsed_page_no = *page_no; } diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 2dd0285d03f05..34fb1f87bdf0a 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -130,6 +130,10 @@ UNIV_INTERN ibool srv_is_being_started = FALSE; UNIV_INTERN ibool srv_was_started = FALSE; /** TRUE if innobase_start_or_create_for_mysql() has been called */ static ibool srv_start_has_been_called = FALSE; +#ifdef UNIV_DEBUG +/** InnoDB system tablespace to set during recovery */ +UNIV_INTERN ulong srv_sys_space_size_debug; +#endif /* UNIV_DEBUG */ /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ @@ -188,9 +192,6 @@ static const ulint SRV_UNDO_TABLESPACE_SIZE_IN_PAGES = #define SRV_N_PENDING_IOS_PER_THREAD OS_AIO_N_PENDING_IOS_PER_THREAD #define SRV_MAX_N_PENDING_SYNC_IOS 100 -/** The round off to MB is similar as done in srv_parse_megabytes() */ -#define CALC_NUMBER_OF_PAGES(size) ((size) / (1024 * 1024)) * \ - ((1024 * 1024) / (UNIV_PAGE_SIZE)) #ifdef UNIV_PFS_THREAD /* Keys to register InnoDB threads with performance schema */ UNIV_INTERN mysql_pfs_key_t io_handler_thread_key; @@ -1025,15 +1026,12 @@ open_or_create_data_files( size = os_file_get_size(files[i]); ut_a(size != (os_offset_t) -1); - /* Under some error conditions like disk full - narios or file size reaching filesystem - limit the data file could contain an incomplete - extent at the end. When we extend a data file - and if some failure happens, then also the data - file could contain an incomplete extent. So we - need to round the size downward to a megabyte.*/ + /* If InnoDB encountered an error or was killed + while extending the data file, the last page + could be incomplete. */ - rounded_size_pages = (ulint) CALC_NUMBER_OF_PAGES(size); + rounded_size_pages = static_cast( + size >> UNIV_PAGE_SIZE_SHIFT); if (i == srv_n_data_files - 1 && srv_auto_extend_last_data_file) { @@ -2191,9 +2189,11 @@ innobase_start_or_create_for_mysql(void) sum_of_new_sizes += srv_data_file_sizes[i]; } - if (sum_of_new_sizes < 10485760 / UNIV_PAGE_SIZE) { + if (!srv_auto_extend_last_data_file && sum_of_new_sizes < 640) { ib_logf(IB_LOG_LEVEL_ERROR, - "Tablespace size must be at least 10 MB"); + "Combined size in innodb_data_file_path" + " must be at least %u MiB", + 640 >> (20 - UNIV_PAGE_SIZE_SHIFT)); return(DB_ERROR); } @@ -2260,6 +2260,8 @@ innobase_start_or_create_for_mysql(void) return(err); } } else { + ut_d(fil_space_get(0)->recv_size = srv_sys_space_size_debug); + for (i = 0; i < SRV_N_LOG_FILES_MAX; i++) { os_offset_t size; os_file_stat_t stat_info; From 63574f1275eda0f0ed439f4896b6e7daf6832a8f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 30 Dec 2016 15:04:10 +0200 Subject: [PATCH 04/74] MDEV-11690 Remove UNIV_HOTBACKUP The InnoDB source code contains quite a few references to a closed-source hot backup tool which was originally called InnoDB Hot Backup (ibbackup) and later incorporated in MySQL Enterprise Backup. The open source backup tool XtraBackup uses the full database for recovery. So, the references to UNIV_HOTBACKUP are only cluttering the source code. --- storage/innobase/btr/btr0btr.cc | 31 - storage/innobase/btr/btr0cur.cc | 17 - storage/innobase/btr/btr0defragment.cc | 3 - storage/innobase/buf/buf0buf.cc | 71 +- storage/innobase/buf/buf0dblwr.cc | 4 - storage/innobase/buf/buf0flu.cc | 7 - storage/innobase/buf/buf0lru.cc | 2 - storage/innobase/data/data0data.cc | 7 - storage/innobase/data/data0type.cc | 6 - storage/innobase/dict/dict0dict.cc | 18 - storage/innobase/dict/dict0mem.cc | 26 +- storage/innobase/dict/dict0stats.cc | 4 - storage/innobase/fil/fil0fil.cc | 241 +----- storage/innobase/fil/fil0pagecompress.cc | 11 +- storage/innobase/fsp/fsp0file.cc | 36 - storage/innobase/fsp/fsp0fsp.cc | 21 +- storage/innobase/fsp/fsp0space.cc | 5 - storage/innobase/fsp/fsp0sysspace.cc | 11 +- storage/innobase/gis/gis0rtree.cc | 4 - storage/innobase/gis/gis0sea.cc | 6 - storage/innobase/ha/ha0ha.cc | 4 +- storage/innobase/ha/hash0hash.cc | 12 +- storage/innobase/ibuf/ibuf0ibuf.cc | 12 +- storage/innobase/include/btr0btr.h | 12 +- storage/innobase/include/btr0btr.ic | 5 - storage/innobase/include/btr0cur.h | 5 - storage/innobase/include/btr0cur.ic | 3 - storage/innobase/include/btr0defragment.h | 7 - storage/innobase/include/buf0buf.h | 72 +- storage/innobase/include/buf0buf.ic | 11 - storage/innobase/include/buf0dblwr.h | 5 - storage/innobase/include/buf0flu.h | 6 - storage/innobase/include/buf0flu.ic | 2 - storage/innobase/include/buf0lru.h | 3 - storage/innobase/include/data0data.h | 5 +- storage/innobase/include/data0data.ic | 2 - storage/innobase/include/data0type.h | 17 +- storage/innobase/include/data0type.ic | 20 +- storage/innobase/include/dict0dict.h | 29 +- storage/innobase/include/dict0dict.ic | 13 - storage/innobase/include/dict0mem.h | 16 +- storage/innobase/include/dict0mem.ic | 2 - storage/innobase/include/dict0priv.ic | 2 - storage/innobase/include/fil0fil.h | 72 +- storage/innobase/include/fut0lst.h | 4 +- storage/innobase/include/ha0ha.h | 3 +- storage/innobase/include/ha_prototypes.h | 4 +- storage/innobase/include/hash0hash.h | 35 +- storage/innobase/include/hash0hash.ic | 2 - storage/innobase/include/ibuf0ibuf.h | 11 +- storage/innobase/include/ibuf0ibuf.ic | 2 - storage/innobase/include/log0log.h | 35 +- storage/innobase/include/log0log.ic | 27 - storage/innobase/include/log0recv.h | 117 +-- storage/innobase/include/log0recv.ic | 38 - storage/innobase/include/mach0data.h | 3 +- storage/innobase/include/mach0data.ic | 3 +- storage/innobase/include/mem0mem.h | 3 +- storage/innobase/include/mem0mem.ic | 9 +- storage/innobase/include/mtr0log.h | 9 - storage/innobase/include/mtr0log.ic | 2 - storage/innobase/include/os0file.h | 13 - storage/innobase/include/page0cur.h | 6 +- storage/innobase/include/page0cur.ic | 2 - storage/innobase/include/page0page.h | 12 +- storage/innobase/include/page0page.ic | 15 +- storage/innobase/include/page0zip.h | 5 +- storage/innobase/include/page0zip.ic | 4 - storage/innobase/include/rem0rec.h | 6 - storage/innobase/include/rem0rec.ic | 2 - storage/innobase/include/row0upd.h | 19 +- storage/innobase/include/row0upd.ic | 12 +- storage/innobase/include/srv0mon.h | 7 - storage/innobase/include/srv0srv.h | 21 - storage/innobase/include/srv0start.h | 5 +- storage/innobase/include/sync0rw.h | 7 - storage/innobase/include/trx0rec.h | 9 +- storage/innobase/include/trx0rec.ic | 2 - storage/innobase/include/trx0sys.h | 37 - storage/innobase/include/trx0sys.ic | 9 +- storage/innobase/include/trx0trx.h | 6 - storage/innobase/include/trx0undo.h | 9 +- storage/innobase/include/trx0undo.ic | 5 - storage/innobase/include/univ.i | 29 +- storage/innobase/include/ut0counter.h | 16 +- storage/innobase/include/ut0mem.h | 6 +- storage/innobase/include/ut0ut.h | 83 +- storage/innobase/log/log0log.cc | 66 +- storage/innobase/log/log0recv.cc | 931 +--------------------- storage/innobase/mem/mem0mem.cc | 16 - storage/innobase/mtr/mtr0log.cc | 9 +- storage/innobase/os/os0file.cc | 86 +- storage/innobase/os/os0thread.cc | 3 - storage/innobase/page/page0cur.cc | 10 - storage/innobase/page/page0page.cc | 36 +- storage/innobase/page/page0zip.cc | 61 +- storage/innobase/rem/rem0rec.cc | 8 +- storage/innobase/row/row0upd.cc | 14 - storage/innobase/srv/srv0mon.cc | 3 - storage/innobase/srv/srv0start.cc | 72 +- storage/innobase/trx/trx0rec.cc | 10 - storage/innobase/trx/trx0sys.cc | 209 ----- storage/innobase/trx/trx0undo.cc | 23 - storage/innobase/ut/ut0dbg.cc | 8 - storage/innobase/ut/ut0mem.cc | 11 +- storage/innobase/ut/ut0ut.cc | 93 +-- 106 files changed, 261 insertions(+), 2930 deletions(-) delete mode 100644 storage/innobase/include/log0recv.ic diff --git a/storage/innobase/btr/btr0btr.cc b/storage/innobase/btr/btr0btr.cc index 6bda054503e16..8acb96f1910af 100644 --- a/storage/innobase/btr/btr0btr.cc +++ b/storage/innobase/btr/btr0btr.cc @@ -37,7 +37,6 @@ Created 6/2/1994 Heikki Tuuri #include "page0zip.h" #include "gis0rtree.h" -#ifndef UNIV_HOTBACKUP #include "btr0cur.h" #include "btr0sea.h" #include "btr0pcur.h" @@ -64,8 +63,6 @@ btr_can_merge_with_page( buf_block_t** merge_block, /*!< out: the merge block */ mtr_t* mtr); /*!< in: mini-transaction */ -#endif /* UNIV_HOTBACKUP */ - /**************************************************************//** Report that an index page is corrupted. */ void @@ -80,7 +77,6 @@ btr_corruption_report( << " of table " << index->table->name; } -#ifndef UNIV_HOTBACKUP /* Latching strategy of the InnoDB B-tree -------------------------------------- @@ -174,7 +170,6 @@ btr_root_block_get( buf_block_t* block = btr_block_get(page_id, page_size, mode, index, mtr); - if (!block) { if (index && index->table) { index->table->is_encrypted = TRUE; @@ -1526,7 +1521,6 @@ btr_write_autoinc(dict_index_t* index, ib_uint64_t autoinc, bool reset) fil_space_release(space); } } -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Reorganizes an index page. @@ -1554,9 +1548,7 @@ btr_page_reorganize_low( mtr_t* mtr) /*!< in/out: mini-transaction */ { buf_block_t* block = page_cur_get_block(cursor); -#ifndef UNIV_HOTBACKUP buf_pool_t* buf_pool = buf_pool_from_bpage(&block->page); -#endif /* !UNIV_HOTBACKUP */ page_t* page = buf_block_get_frame(block); page_zip_des_t* page_zip = buf_block_get_page_zip(block); buf_block_t* temp_block; @@ -1581,12 +1573,7 @@ btr_page_reorganize_low( /* Turn logging off */ mtr_log_t log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE); -#ifndef UNIV_HOTBACKUP temp_block = buf_block_alloc(buf_pool); -#else /* !UNIV_HOTBACKUP */ - ut_ad(block == back_block1); - temp_block = back_block2; -#endif /* !UNIV_HOTBACKUP */ temp_page = temp_block->frame; MONITOR_INC(MONITOR_INDEX_REORG_ATTEMPTS); @@ -1599,11 +1586,9 @@ btr_page_reorganize_low( /* Copy the old page to temporary space */ buf_frame_copy(temp_page, page); -#ifndef UNIV_HOTBACKUP if (!recovery) { btr_search_drop_page_hash_index(block); } -#endif /* !UNIV_HOTBACKUP */ /* Save the cursor position. */ pos = page_rec_get_n_recs_before(page_cur_get_rec(cursor)); @@ -1678,12 +1663,10 @@ btr_page_reorganize_low( goto func_exit; } -#ifndef UNIV_HOTBACKUP if (!recovery && !dict_table_is_locking_disabled(index->table)) { /* Update the record lock bitmaps */ lock_move_reorganize_page(block, temp_block); } -#endif /* !UNIV_HOTBACKUP */ data_size2 = page_get_data_size(page); max_ins_size2 = page_get_max_insert_size_after_reorganize(page, 1); @@ -1712,14 +1695,11 @@ btr_page_reorganize_low( #ifdef UNIV_ZIP_DEBUG ut_a(!page_zip || page_zip_validate(page_zip, page, index)); #endif /* UNIV_ZIP_DEBUG */ -#ifndef UNIV_HOTBACKUP buf_block_free(temp_block); -#endif /* !UNIV_HOTBACKUP */ /* Restore logging mode */ mtr_set_log_mode(mtr, log_mode); -#ifndef UNIV_HOTBACKUP if (success) { mlog_id_t type; byte* log_ptr; @@ -1748,7 +1728,6 @@ btr_page_reorganize_low( MONITOR_INC(MONITOR_INDEX_REORG_SUCCESSFUL); } -#endif /* !UNIV_HOTBACKUP */ return(success); } @@ -1784,7 +1763,6 @@ btr_page_reorganize_block( return(btr_page_reorganize_low(recovery, z_level, &cur, index, mtr)); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Reorganizes an index page. @@ -1806,7 +1784,6 @@ btr_page_reorganize( return(btr_page_reorganize_low(false, page_zip_level, cursor, index, mtr)); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of reorganizing a page. @@ -1850,7 +1827,6 @@ btr_parse_page_reorganize( return(ptr); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Empties an index page. @see btr_page_create(). */ static @@ -3345,9 +3321,6 @@ btr_set_min_rec_mark_log( /* Write rec offset as a 2-byte ulint */ mlog_catenate_ulint(mtr, page_offset(rec), MLOG_2BYTES); } -#else /* !UNIV_HOTBACKUP */ -# define btr_set_min_rec_mark_log(rec,comp,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Parses the redo log record for setting an index record as the predefined @@ -3405,7 +3378,6 @@ btr_set_min_rec_mark( } } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Deletes on the upper level the node pointer to a page. */ void @@ -5497,7 +5469,6 @@ btr_can_merge_with_page( goto error; } - max_ins_size = page_get_max_insert_size(mpage, n_recs); if (data_size > max_ins_size) { @@ -5531,5 +5502,3 @@ btr_can_merge_with_page( *merge_block = NULL; DBUG_RETURN(false); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index d8a2e15d1eb24..87d88cea8200b 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -50,7 +50,6 @@ Created 10/16/1994 Heikki Tuuri #endif #include "row0upd.h" -#ifndef UNIV_HOTBACKUP #include "mtr0log.h" #include "page0page.h" #include "page0zip.h" @@ -151,9 +150,7 @@ can be released by page reorganize, then it is reorganized */ + (sample) - 1 + (ext_size) + (not_empty)) / ((sample) + (ext_size))) /* @} */ -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP /*******************************************************************//** Marks all extern fields in a record as owned by the record. This function should be called if the delete mark of a record is removed: a not delete @@ -212,9 +209,7 @@ btr_rec_free_externally_stored_fields( mtr_t* mtr); /*!< in: mini-transaction handle which contains an X-latch to record page and to the index tree */ -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP /*==================== B-TREE SEARCH =========================*/ #if MTR_MEMO_PAGE_S_FIX != RW_S_LATCH @@ -3257,7 +3252,6 @@ btr_cur_pessimistic_insert( ut_ad(page_rec_get_next(btr_cur_get_rec(cursor)) == *rec || dict_index_is_spatial(index)); - if (!(flags & BTR_NO_LOCKING_FLAG)) { ut_ad(!dict_table_is_temporary(index->table)); if (dict_index_is_spatial(index)) { @@ -3425,7 +3419,6 @@ btr_cur_update_in_place_log( row_upd_index_write_log(update, log_ptr, mtr); } -#endif /* UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of updating a record in-place. @@ -3504,7 +3497,6 @@ btr_cur_parse_update_in_place( return(ptr); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** See if there is enough place in the page modification log to log an update-in-place. @@ -4525,7 +4517,6 @@ btr_cur_del_mark_set_clust_rec_log( mlog_close(mtr, log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Parses the redo log record for delete marking or unmarking of a clustered @@ -4607,7 +4598,6 @@ btr_cur_parse_del_mark_set_clust_rec( return(ptr); } -#ifndef UNIV_HOTBACKUP /***********************************************************//** Marks a clustered index record deleted. Writes an undo log record to undo log on this delete marking. Writes in the trx id field the id @@ -4729,7 +4719,6 @@ btr_cur_del_mark_set_sec_rec_log( mlog_close(mtr, log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Parses the redo log record for delete marking or unmarking of a secondary @@ -4774,7 +4763,6 @@ btr_cur_parse_del_mark_set_sec_rec( return(ptr); } -#ifndef UNIV_HOTBACKUP /***********************************************************//** Sets a secondary index record delete mark to TRUE or FALSE. @return DB_SUCCESS, DB_LOCK_WAIT, or error number */ @@ -5367,7 +5355,6 @@ btr_estimate_n_rows_in_range_on_level( goto inexact; } - page = buf_block_get_frame(block); /* It is possible that the tree has been reorganized in the @@ -5603,7 +5590,6 @@ btr_estimate_n_rows_in_range_low( << " index: " << index->name; } - ut_ad(page_rec_is_supremum(btr_cur_get_rec(&cursor))); /* The range specified is wihout a right border, just @@ -6090,7 +6076,6 @@ btr_estimate_number_of_different_key_vals( } } - if (n_cols == dict_index_get_n_unique_in_tree(index)) { /* If there is more than one leaf page in the tree, @@ -6606,7 +6591,6 @@ struct btr_blob_log_check_t { } }; - /*******************************************************************//** Stores the fields in big_rec_vec to the tablespace and puts pointers to them in rec. The extern flags in rec will have to be set beforehand. @@ -7790,4 +7774,3 @@ btr_rec_copy_externally_stored_field( return(btr_copy_externally_stored_field(len, data, page_size, local_len, heap)); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/btr/btr0defragment.cc b/storage/innobase/btr/btr0defragment.cc index 71a93744ef4b3..7f06aed9ff0bd 100644 --- a/storage/innobase/btr/btr0defragment.cc +++ b/storage/innobase/btr/btr0defragment.cc @@ -26,7 +26,6 @@ Modified 30/07/2014 Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "btr0defragment.h" -#ifndef UNIV_HOTBACKUP #include "btr0btr.h" #include "btr0cur.h" #include "btr0sea.h" @@ -847,5 +846,3 @@ DECLARE_THREAD(btr_defragment_thread)( os_thread_exit(); OS_THREAD_DUMMY_RETURN; } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ffd739bc68673..935b7fadafa54 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -49,7 +49,6 @@ Created 11/5/1995 Heikki Tuuri #include "fil0fil.h" #include "fil0crypt.h" #include "fsp0sysspace.h" -#ifndef UNIV_HOTBACKUP #include "buf0buddy.h" #include "lock0lock.h" #include "sync0rw.h" @@ -59,7 +58,6 @@ Created 11/5/1995 Heikki Tuuri #include "trx0purge.h" #include "log0log.h" #include "dict0stats_bg.h" -#endif /* !UNIV_HOTBACKUP */ #include "srv0srv.h" #include "srv0start.h" #include "dict0dict.h" @@ -323,7 +321,7 @@ that the whole area may be needed in the near future, and issue the read requests for the whole area. */ -#if (!(defined(UNIV_HOTBACKUP) || defined(UNIV_INNOCHECKSUM))) +#ifndef UNIV_INNOCHECKSUM /** Value in microseconds */ static const int WAIT_FOR_READ = 100; static const int WAIT_FOR_WRITE = 100; @@ -591,7 +589,7 @@ buf_block_alloc( return(block); } -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ /** Checks if a page contains only zeroes. @param[in] read_buf database page @@ -883,7 +881,7 @@ buf_page_is_corrupted( return(TRUE); } -#if !defined(UNIV_HOTBACKUP) && !defined(UNIV_INNOCHECKSUM) +#ifndef UNIV_INNOCHECKSUM if (check_lsn && recv_lsn_checks_on) { lsn_t current_lsn; const lsn_t page_lsn @@ -913,7 +911,7 @@ buf_page_is_corrupted( } } -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ /* Check whether the checksum fields have correct values */ @@ -1264,9 +1262,7 @@ buf_page_print( const page_size_t& page_size, ulint flags) { -#ifndef UNIV_HOTBACKUP dict_index_t* index; -#endif /* !UNIV_HOTBACKUP */ if (!(flags & BUF_PAGE_PRINT_NO_FULL)) { @@ -1372,7 +1368,6 @@ buf_page_print( read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); } -#ifndef UNIV_HOTBACKUP if (mach_read_from_2(read_buf + TRX_UNDO_PAGE_HDR + TRX_UNDO_PAGE_TYPE) == TRX_UNDO_INSERT) { fprintf(stderr, @@ -1383,7 +1378,6 @@ buf_page_print( fprintf(stderr, "InnoDB: Page may be an update undo log page\n"); } -#endif /* !UNIV_HOTBACKUP */ switch (fil_page_get_type(read_buf)) { index_id_t index_id; @@ -1394,7 +1388,6 @@ buf_page_print( "InnoDB: Page may be an index page where" " index id is " << index_id; -#ifndef UNIV_HOTBACKUP index = dict_index_find_on_id_low(index_id); if (index) { ib::info() @@ -1402,7 +1395,6 @@ buf_page_print( << " is " << index->name << " in table " << index->table->name; } -#endif /* !UNIV_HOTBACKUP */ break; case FIL_PAGE_INODE: fputs("InnoDB: Page may be an 'inode' page\n", stderr); @@ -1449,8 +1441,6 @@ buf_page_print( ut_ad(flags & BUF_PAGE_PRINT_NO_CRASH); } -#ifndef UNIV_HOTBACKUP - # ifdef PFS_GROUP_BUFFER_SYNC extern mysql_pfs_key_t buffer_block_mutex_key; @@ -3936,7 +3926,6 @@ buf_block_init_low( block->n_bytes = 0; block->left_side = TRUE; } -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Decompress a block. @@ -4007,7 +3996,6 @@ buf_zip_decompress( return(FALSE); } -#ifndef UNIV_HOTBACKUP /** Get a buffer block from an adaptive hash index pointer. This function does not return if the block is not identified. @param[in] ptr pointer to within a page frame @@ -7339,57 +7327,6 @@ buf_pool_check_no_pending_io(void) return(pending_io); } -#if 0 -Code currently not used -/*********************************************************************//** -Gets the current length of the free list of buffer blocks. -@return length of the free list */ -ulint -buf_get_free_list_len(void) -/*=======================*/ -{ - ulint len; - - buf_pool_mutex_enter(buf_pool); - - len = UT_LIST_GET_LEN(buf_pool->free); - - buf_pool_mutex_exit(buf_pool); - - return(len); -} -#endif - -#else /* !UNIV_HOTBACKUP */ - -/** Inits a page to the buffer buf_pool, for use in mysqlbackup --restore. -@param[in] page_id page id -@param[in] page_size page size -@param[in,out] block block to init */ -void -buf_page_init_for_backup_restore( - const page_id_t& page_id, - const page_size_t& page_size, - buf_block_t* block) -{ - block->page.state = BUF_BLOCK_FILE_PAGE; - block->page.id = page_id; - block->page.size.copy_from(page_size); - - page_zip_des_init(&block->page.zip); - - /* We assume that block->page.data has been allocated - with page_size == univ_page_size. */ - if (page_size.is_compressed()) { - page_zip_set_size(&block->page.zip, page_size.physical()); - block->page.zip.data = block->frame + page_size.logical(); - } else { - page_zip_set_size(&block->page.zip, 0); - } -} - -#endif /* !UNIV_HOTBACKUP */ - /** Print the given page_id_t object. @param[in,out] out the output stream @param[in] page_id the page_id_t object to be printed diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index ff1d2057e6afb..5f46576ad7e37 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -40,8 +40,6 @@ Created 2011/12/19 #include "fil0crypt.h" #include "fil0pagecompress.h" -#ifndef UNIV_HOTBACKUP - /** The doublewrite buffer */ buf_dblwr_t* buf_dblwr = NULL; @@ -1195,7 +1193,6 @@ buf_dblwr_add_to_batch( } else { ut_a(buf_page_get_state(bpage) == BUF_BLOCK_FILE_PAGE); - UNIV_MEM_ASSERT_RW(frame, bpage->size.logical()); @@ -1360,4 +1357,3 @@ buf_dblwr_write_single_page( blocks. Next do the write to the intended position. */ buf_dblwr_write_block_to_datafile(bpage, sync); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 179430a0d27f6..b82c4db18ad27 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -41,7 +41,6 @@ Created 11/11/1995 Heikki Tuuri #include "srv0start.h" #include "srv0srv.h" #include "page0zip.h" -#ifndef UNIV_HOTBACKUP #include "ut0byte.h" #include "page0page.h" #include "fil0fil.h" @@ -806,7 +805,6 @@ buf_flush_write_complete( buf_dblwr_update(bpage, flush_type); } -#endif /* !UNIV_HOTBACKUP */ /** Calculate the checksum of a page from compressed table and update the page. @@ -995,7 +993,6 @@ buf_flush_init_for_writing( checksum); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Does an asynchronous write of a buffer page. NOTE: in simulated aio and also when the doublewrite buffer is used, we must call @@ -2264,8 +2261,6 @@ buf_flush_single_page_from_LRU( scanned); } - - ut_ad(!buf_pool_mutex_own(buf_pool)); return(freed); } @@ -2521,7 +2516,6 @@ page_cleaner_flush_pages_recommendation( lsn_avg_rate = (lsn_avg_rate + lsn_rate) / 2; - /* aggregate stats of all slots */ mutex_enter(&page_cleaner->mutex); @@ -3679,7 +3673,6 @@ buf_flush_validate( } #endif /* UNIV_DEBUG || UNIV_BUF_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /******************************************************************//** Check if there are any dirty pages that belong to a space id in the flush diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 5d2077763a690..6a57e6746ffeb 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -28,7 +28,6 @@ Created 11/5/1995 Heikki Tuuri #include "buf0lru.ic" #endif /* UNIV_NOINL */ -#ifndef UNIV_HOTBACKUP #include "ut0byte.h" #include "ut0rnd.h" #include "sync0rw.h" @@ -2791,4 +2790,3 @@ buf_LRU_print(void) } } #endif /* UNIV_DEBUG_PRINT || UNIV_DEBUG || UNIV_BUF_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/data/data0data.cc b/storage/innobase/data/data0data.cc index 4b788c8952c42..0ccf5868d149e 100644 --- a/storage/innobase/data/data0data.cc +++ b/storage/innobase/data/data0data.cc @@ -31,7 +31,6 @@ Created 5/30/1994 Heikki Tuuri #include "data0data.ic" #endif -#ifndef UNIV_HOTBACKUP #include "rem0rec.h" #include "rem0cmp.h" #include "page0page.h" @@ -40,8 +39,6 @@ Created 5/30/1994 Heikki Tuuri #include "btr0cur.h" #include "row0upd.h" -#endif /* !UNIV_HOTBACKUP */ - #ifdef UNIV_DEBUG /** Dummy variable to catch access to uninitialized fields. In the debug version, dtuple_create() will make all fields of dtuple_t point @@ -54,7 +51,6 @@ ulint data_dummy; # endif /* !UNIV_DEBUG_VALGRIND */ #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /** Compare two data tuples. @param[in] tuple1 first data tuple @param[in] tuple2 second data tuple @@ -159,7 +155,6 @@ dtuple_check_typed_no_assert( return(TRUE); } -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /**********************************************************//** @@ -255,7 +250,6 @@ dtuple_validate( } #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /*************************************************************//** Pretty prints a dfield value according to its data type. */ void @@ -848,4 +842,3 @@ dfield_t::clone( return(obj); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/data/data0type.cc b/storage/innobase/data/data0type.cc index 8fb3761531c0f..cad165d148947 100644 --- a/storage/innobase/data/data0type.cc +++ b/storage/innobase/data/data0type.cc @@ -31,7 +31,6 @@ Created 1/16/1996 Heikki Tuuri #include "data0type.ic" #endif -#ifndef UNIV_HOTBACKUP /* At the database startup we store the default-charset collation number of this MySQL installation to this global variable. If we have < 4.1.2 format column definitions, or records in the insert buffer, we use this @@ -78,7 +77,6 @@ dtype_get_at_most_n_mbchars( return(data_len); } -#endif /* UNIV_HOTBACKUP */ /*********************************************************************//** Checks if a data main type is a string type. Also a BLOB is considered a @@ -174,14 +172,11 @@ dtype_validate( ut_a((type->prtype & DATA_MYSQL_TYPE_MASK) < DATA_N_SYS_COLS); } -#ifndef UNIV_HOTBACKUP ut_a(dtype_get_mbminlen(type) <= dtype_get_mbmaxlen(type)); -#endif /* !UNIV_HOTBACKUP */ return(TRUE); } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Prints a data type structure. */ void @@ -299,4 +294,3 @@ dtype_print( fprintf(stderr, " len %lu", (ulong) len); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index d7d81661722d3..37c6341a29368 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -55,7 +55,6 @@ Issue a warning that the row is too big. */ void ib_warn_row_too_big(const dict_table_t* table); -#ifndef UNIV_HOTBACKUP #include "btr0btr.h" #include "btr0cur.h" #include "btr0sea.h" @@ -551,7 +550,6 @@ dict_table_close( } } } -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Closes the only open handle to a table and drops a table while assuring @@ -753,7 +751,6 @@ dict_table_get_nth_v_col_mysql( return(dict_table_get_nth_v_col(table, i)); } -#ifndef UNIV_HOTBACKUP /** Allocate and init the autoinc latch of a given table. This function must not be called concurrently on the same table object. @param[in,out] table_void table whose autoinc latch to create */ @@ -782,7 +779,6 @@ dict_index_zip_pad_alloc( mutex_create(LATCH_ID_ZIP_PAD_MUTEX, index->zip_pad.mutex); } - /********************************************************************//** Acquire the autoinc lock. */ void @@ -844,7 +840,6 @@ dict_table_autoinc_unlock( { mutex_exit(table->autoinc_mutex); } -#endif /* !UNIV_HOTBACKUP */ /** Looks for column n in an index. @param[in] index index @@ -903,7 +898,6 @@ dict_index_get_nth_col_or_prefix_pos( return(ULINT_UNDEFINED); } -#ifndef UNIV_HOTBACKUP /** Returns TRUE if the index contains a column or a prefix of that column. @param[in] index index @param[in] n column number @@ -1242,7 +1236,6 @@ dict_table_open_on_name( DBUG_RETURN(table); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Adds system columns to a table object. */ @@ -1292,7 +1285,6 @@ dict_table_add_system_columns( #endif } -#ifndef UNIV_HOTBACKUP /** Mark if table has big rows. @param[in,out] table table handler */ void @@ -2874,7 +2866,6 @@ dict_index_find_cols( return(TRUE); } -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Adds a column to index. */ @@ -2957,7 +2948,6 @@ dict_index_add_col( } } -#ifndef UNIV_HOTBACKUP /*******************************************************************//** Copies fields contained in index2 to index1. */ static @@ -3171,7 +3161,6 @@ dict_index_build_internal_clust( new_index, table, dict_table_get_sys_col(table, DATA_TRX_ID), 0); - for (i = 0; i < trx_id_pos; i++) { ulint fixed_size = dict_col_get_fixed_size( @@ -3489,7 +3478,6 @@ dict_foreign_find( return(NULL); } - /*********************************************************************//** Tries to find an index whose first fields are the columns in the array, in the same order and is not marked for deletion and is not the same @@ -3675,7 +3663,6 @@ dict_foreign_add_to_cache( } - if (ref_table && !for_in_cache->referenced_table) { ulint index_error; ulint err_col; @@ -4052,7 +4039,6 @@ dict_scan_col( return(ptr); } - /*********************************************************************//** Open a table from its database and table name, this is currently used by foreign constraint parser to get the referenced table. @@ -6348,7 +6334,6 @@ dict_set_merge_threshold_all_debug( } #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Inits dict_ind_redundant. */ @@ -6372,7 +6357,6 @@ dict_ind_init(void) dict_ind_redundant->cached = TRUE; } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Frees dict_ind_redundant. */ static @@ -7228,7 +7212,6 @@ dict_index_zip_failure( dict_index_zip_pad_unlock(index); } - /*********************************************************************//** Return the optimal page size, for which page will likely compress. @return page size beyond which page might not compress */ @@ -7444,7 +7427,6 @@ dict_space_get_id( return(id); } -#endif /* !UNIV_HOTBACKUP */ /** Determine the extent size (in pages) for the given table @param[in] table the table whose extent size is being diff --git a/storage/innobase/dict/dict0mem.cc b/storage/innobase/dict/dict0mem.cc index b150c1eb7a035..f4726ecd329cc 100644 --- a/storage/innobase/dict/dict0mem.cc +++ b/storage/innobase/dict/dict0mem.cc @@ -24,10 +24,8 @@ Data dictionary memory object creation Created 1/8/1996 Heikki Tuuri ***********************************************************************/ -#ifndef UNIV_HOTBACKUP #include "ha_prototypes.h" #include -#endif /* !UNIV_HOTBACKUP */ #include "dict0mem.h" @@ -41,11 +39,7 @@ Created 1/8/1996 Heikki Tuuri #include "dict0dict.h" #include "fts0priv.h" #include "ut0crc32.h" - -#ifndef UNIV_HOTBACKUP -# include "lock0lock.h" -#endif /* !UNIV_HOTBACKUP */ - +#include "lock0lock.h" #include "sync0sync.h" #include @@ -152,7 +146,6 @@ dict_mem_table_create( dict_table_stats_lock() will not be noop. */ dict_table_stats_latch_create(table, true); -#ifndef UNIV_HOTBACKUP table->autoinc_lock = static_cast( mem_heap_alloc(heap, lock_get_size())); @@ -169,7 +162,6 @@ dict_mem_table_create( } else { table->fts = NULL; } -#endif /* !UNIV_HOTBACKUP */ if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { dict_get_and_save_space_name(table, true); @@ -201,10 +193,8 @@ dict_mem_table_free( fts_free(table); } } -#ifndef UNIV_HOTBACKUP - dict_table_autoinc_destroy(table); -#endif /* UNIV_HOTBACKUP */ + dict_table_autoinc_destroy(table); dict_mem_table_free_foreign_vcol_set(table); dict_table_stats_latch_destroy(table); @@ -432,7 +422,6 @@ dict_mem_table_add_s_col( table->s_cols->push_back(s_col); } - /**********************************************************************//** Renames a column of a table in the data dictionary cache. */ static MY_ATTRIBUTE((nonnull)) @@ -632,10 +621,8 @@ dict_mem_fill_column_struct( ulint prtype, /*!< in: precise type */ ulint col_len) /*!< in: column length */ { -#ifndef UNIV_HOTBACKUP ulint mbminlen; ulint mbmaxlen; -#endif /* !UNIV_HOTBACKUP */ column->ind = (unsigned int) col_pos; column->ord_part = 0; @@ -643,10 +630,9 @@ dict_mem_fill_column_struct( column->mtype = (unsigned int) mtype; column->prtype = (unsigned int) prtype; column->len = (unsigned int) col_len; -#ifndef UNIV_HOTBACKUP - dtype_get_mblen(mtype, prtype, &mbminlen, &mbmaxlen); + + dtype_get_mblen(mtype, prtype, &mbminlen, &mbmaxlen); dict_col_set_mbminmaxlen(column, mbminlen, mbmaxlen); -#endif /* !UNIV_HOTBACKUP */ } /**********************************************************************//** @@ -693,7 +679,6 @@ dict_mem_index_create( return(index); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Creates and initializes a foreign constraint memory object. @return own: foreign constraint struct */ @@ -953,8 +938,6 @@ dict_mem_table_free_foreign_vcol_set( } } -#endif /* !UNIV_HOTBACKUP */ - /**********************************************************************//** Adds a field definition to an index. NOTE: does not take a copy of the column name if the field is a column. The memory occupied @@ -1150,4 +1133,3 @@ dict_mem_table_is_system( return true; } } - diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index 5cb6d9714c831..87502ef130ce8 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -23,8 +23,6 @@ Code used for calculating and manipulating table statistics. Created Jan 06, 2010 Vasil Dimov *******************************************************/ -#ifndef UNIV_HOTBACKUP - #include "univ.i" #include "ut0ut.h" @@ -4290,5 +4288,3 @@ test_dict_stats_all() #endif /* UNIV_ENABLE_UNIT_TEST_DICT_STATS */ /* @} */ - -#endif /* UNIV_HOTBACKUP */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index fae9ee3dbc1af..60ed4e014224e 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -29,7 +29,6 @@ Created 10/25/1995 Heikki Tuuri #include "fsp0pagecompress.h" #include "fil0crypt.h" -#ifndef UNIV_HOTBACKUP #include "btr0btr.h" #include "buf0buf.h" #include "dict0boot.h" @@ -51,11 +50,10 @@ Created 10/25/1995 Heikki Tuuri #include "srv0start.h" #include "trx0purge.h" #include "ut0new.h" -# include "buf0lru.h" -# include "ibuf0ibuf.h" -# include "os0event.h" -# include "sync0sync.h" -#endif /* !UNIV_HOTBACKUP */ +#include "buf0lru.h" +#include "ibuf0ibuf.h" +#include "os0event.h" +#include "sync0sync.h" #include "buf0flu.h" #include "srv0start.h" #include "trx0purge.h" @@ -161,11 +159,6 @@ fil_addr_t fil_addr_null = {FIL_NULL, 0}; initialized. */ fil_system_t* fil_system = NULL; -#ifdef UNIV_HOTBACKUP -static ulint srv_data_read; -static ulint srv_data_written; -#endif /* UNIV_HOTBACKUP */ - /** Determine if user has explicitly disabled fsync(). */ #ifndef _WIN32 # define fil_buffering_disabled(s) \ @@ -384,7 +377,6 @@ fil_space_get( return(space); } -#ifndef UNIV_HOTBACKUP /** Returns the latch of a file space. @param[in] id space id @param[out] flags tablespace flags @@ -456,7 +448,6 @@ fil_space_set_imported( mutex_exit(&fil_system->mutex); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Checks if all the file nodes in a space are flushed. The caller must hold @@ -698,13 +689,6 @@ fil_node_open_file( size_bytes = os_file_get_size(node->handle); ut_a(size_bytes != (os_offset_t) -1); -#ifdef UNIV_HOTBACKUP - if (space->id == 0) { - node->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); - os_file_close(node->handle); - goto add_size; - } -#endif /* UNIV_HOTBACKUP */ ut_a(space->purpose != FIL_TYPE_LOG); /* Read the first page of the tablespace */ @@ -813,19 +797,16 @@ fil_node_open_file( /* After apply-incremental, tablespaces are not extended to a whole megabyte. Do not cut off valid data. */ -#ifndef UNIV_HOTBACKUP + /* Truncate the size to a multiple of extent size. */ if (size_bytes >= extent_size) { size_bytes = ut_2pow_round(size_bytes, extent_size); } -#endif /* !UNIV_HOTBACKUP */ + node->size = (ulint) (size_bytes / page_size.physical()); -#ifdef UNIV_HOTBACKUP -add_size: -#endif /* UNIV_HOTBACKUP */ space->size += node->size; } } @@ -880,11 +861,9 @@ fil_node_close_file( ut_a(node->n_pending == 0); ut_a(node->n_pending_flushes == 0); ut_a(!node->being_extended); -#ifndef UNIV_HOTBACKUP ut_a(node->modification_counter == node->flush_counter || node->space->purpose == FIL_TYPE_TEMPORARY || srv_fast_shutdown == 2); -#endif /* !UNIV_HOTBACKUP */ ret = os_file_close(node->handle); ut_a(ret); @@ -1133,17 +1112,10 @@ fil_write_zeros( IORequest request(IORequest::WRITE); while (offset < end) { - -#ifdef UNIV_HOTBACKUP - err = os_file_write( - request, node->name, node->handle, buf, offset, - n_bytes); -#else err = os_aio( request, OS_AIO_SYNC, node->name, node->handle, buf, offset, n_bytes, read_only_mode, NULL, NULL, NULL); -#endif /* UNIV_HOTBACKUP */ if (err != DB_SUCCESS) { break; @@ -1690,8 +1662,6 @@ fil_space_create( UT_LIST_INIT(space->chain, &fil_node_t::chain); - /* This warning is not applicable while MEB scanning the redo logs */ -#ifndef UNIV_HOTBACKUP if ((purpose == FIL_TYPE_TABLESPACE || purpose == FIL_TYPE_IMPORT) && !recv_recovery_on && id > fil_system->max_assigned_id) { @@ -1706,7 +1676,7 @@ fil_space_create( fil_system->max_assigned_id = id; } -#endif /* !UNIV_HOTBACKUP */ + space->purpose = purpose; space->flags = flags; @@ -1733,9 +1703,7 @@ fil_space_create( rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); if (space->purpose == FIL_TYPE_TEMPORARY) { -#ifndef UNIV_HOTBACKUP ut_d(space->latch.set_temp_fsp()); -#endif /* !UNIV_HOTBACKUP */ } HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space); @@ -2320,7 +2288,6 @@ fil_write_flushed_lsn( return(err); } -#ifndef UNIV_HOTBACKUP /** Acquire a tablespace when it could be dropped concurrently. Used by background threads that do not necessarily hold proper locks for concurrency control. @@ -2391,7 +2358,6 @@ fil_space_release( space->n_pending_ops--; mutex_exit(&fil_system->mutex); } -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Creates the database directory for a table if it does not exist yet. */ @@ -2502,7 +2468,7 @@ fil_op_write_log( ut_ad(0); } } -#ifndef UNIV_HOTBACKUP + /** Write redo log for renaming a file. @param[in] space_id tablespace id @param[in] first_page_no first page number in the file @@ -2524,7 +2490,7 @@ fil_name_write_rename( MLOG_FILE_RENAME2, space_id, first_page_no, old_name, new_name, 0, mtr); } -#endif /* !UNIV_HOTBACKUP */ + /** Write MLOG_FILE_NAME for a file. @param[in] space_id tablespace id @param[in] first_page_no first page number in the file @@ -2557,7 +2523,6 @@ fil_name_write( fil_name_write(space->id, first_page_no, file->name, mtr); } -#ifndef UNIV_HOTBACKUP /********************************************************//** Recreates table indexes by applying TRUNCATE log record during recovery. @@ -2798,7 +2763,7 @@ fil_recreate_tablespace( return(err); } -#endif /* UNIV_HOTBACKUP */ + /** Replay a file rename operation if possible. @param[in] space_id tablespace identifier @param[in] first_page_no first page number in the file @@ -2814,9 +2779,6 @@ fil_op_replay_rename( const char* name, const char* new_name) { -#ifdef UNIV_HOTBACKUP - ut_ad(recv_replay_file_ops); -#endif /* UNIV_HOTBACKUP */ ut_ad(first_page_no == 0); /* In order to replay the rename, the following must hold: @@ -3147,7 +3109,6 @@ fil_delete_tablespace( ut_a(space); ut_a(path != 0); -#ifndef UNIV_HOTBACKUP /* IMPORTANT: Because we have set space::stop_new_ops there can't be any new ibuf merges, reads or flushes. We are here because node::n_pending was zero above. However, it is still @@ -3171,15 +3132,9 @@ fil_delete_tablespace( buf_LRU_flush_or_remove_pages(id, buf_remove, 0); -#endif /* !UNIV_HOTBACKUP */ - /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ { -#ifdef UNIV_HOTBACKUP - /* When replaying the operation in MySQL Enterprise - Backup, we do not try to write any log record. */ -#else /* UNIV_HOTBACKUP */ /* Before deleting the file, write a log record about it, so that InnoDB crash recovery will expect the file to be gone. */ @@ -3192,7 +3147,6 @@ fil_delete_tablespace( tablespace file, the record must have already been written to the redo log. */ log_write_up_to(mtr.commit_lsn(), true); -#endif /* UNIV_HOTBACKUP */ char* cfg_name = fil_make_filepath(path, NULL, CFG, false); if (cfg_name != NULL) { @@ -3255,7 +3209,6 @@ fil_delete_tablespace( return(err); } -#ifndef UNIV_HOTBACKUP /** Truncate the tablespace to needed size. @param[in] space_id id of tablespace to truncate @param[in] size_in_pages truncate size. @@ -3489,7 +3442,6 @@ fil_discard_tablespace( return(err); } -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Allocates and builds a file name from a path, a table or tablespace name @@ -3756,7 +3708,7 @@ fil_rename_tablespace( ut_ad(strchr(old_file_name, OS_PATH_SEPARATOR) != NULL); ut_ad(strchr(new_file_name, OS_PATH_SEPARATOR) != NULL); -#ifndef UNIV_HOTBACKUP + if (!recv_recovery_on) { mtr_t mtr; @@ -3766,7 +3718,6 @@ fil_rename_tablespace( mtr.commit(); log_mutex_enter(); } -#endif /* !UNIV_HOTBACKUP */ /* log_sys->mutex is above fil_system->mutex in the latching order */ ut_ad(log_mutex_own()); @@ -3794,11 +3745,9 @@ fil_rename_tablespace( node->name = new_file_name; } -#ifndef UNIV_HOTBACKUP if (!recv_recovery_on) { log_mutex_exit(); } -#endif /* !UNIV_HOTBACKUP */ ut_ad(space->name == old_space_name); if (success) { @@ -3983,11 +3932,11 @@ fil_ibd_create( page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); memset(page, '\0', UNIV_PAGE_SIZE); -#ifndef UNIV_HOTBACKUP + /* Add the UNIV_PAGE_SIZE to the table flags and write them to the tablespace header. */ flags = fsp_flags_set_page_size(flags, univ_page_size); -#endif /* !UNIV_HOTBACKUP */ + fsp_header_init_fields(page, space_id, flags); mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); @@ -4052,9 +4001,6 @@ fil_ibd_create( return(DB_ERROR); } - /* MEB creates isl files during copy-back, hence they - should not be created during apply log operation. */ -#ifndef UNIV_HOTBACKUP if (has_data_dir || has_shared_space) { /* Make the ISL file if the IBD file is not in the default location. */ @@ -4066,7 +4012,6 @@ fil_ibd_create( return(err); } } -#endif /* !UNIV_HOTBACKUP */ /* Create crypt data if the tablespace is either encrypted or user has requested it to remain unencrypted. */ @@ -4101,7 +4046,6 @@ fil_ibd_create( } #endif /* MYSQL_ENCRYPTION */ -#ifndef UNIV_HOTBACKUP if (!is_temp) { mtr_t mtr; const fil_node_t* file = UT_LIST_GET_FIRST(space->chain); @@ -4113,7 +4057,7 @@ fil_ibd_create( fil_name_write(space, 0, file, &mtr); mtr_commit(&mtr); } -#endif /* !UNIV_HOTBACKUP */ + err = DB_SUCCESS; /* Error code is set. Cleanup the various variables used. @@ -4132,7 +4076,6 @@ fil_ibd_create( return(err); } -#ifndef UNIV_HOTBACKUP /** Try to open a single-table tablespace and optionally check that the space id in it is correct. If this does not succeed, print an error message to the .err log. This function is used to open a tablespace when we start @@ -4529,32 +4472,6 @@ fil_ibd_open( return(err); } -#endif /* !UNIV_HOTBACKUP */ - -#ifdef UNIV_HOTBACKUP -/*******************************************************************//** -Allocates a file name for an old version of a single-table tablespace. -The string must be freed by caller with ut_free()! -@return own: file name */ -static -char* -fil_make_ibbackup_old_name( -/*=======================*/ - const char* name) /*!< in: original file name */ -{ - static const char suffix[] = "_ibbackup_old_vers_"; - char* path; - ulint len = strlen(name); - - path = static_cast(ut_malloc_nokey(len + 15 + sizeof(suffix))); - - memcpy(path, name, len); - memcpy(path + len, suffix, sizeof(suffix) - 1); - ut_sprintf_timestamp_without_extra_chars( - path + len + sizeof(suffix) - 1); - return(path); -} -#endif /* UNIV_HOTBACKUP */ /** Looks for a pre-existing fil_space_t with the given tablespace ID and, if found, returns the name and filepath in newly allocated buffers @@ -4640,7 +4557,7 @@ fil_path_to_space_name( } /** Discover the correct IBD file to open given a remote or missing -filepath from the REDO log. MEB and administrators can move a crashed +filepath from the REDO log. Administrators can move a crashed database to another location on the same machine and try to recover it. Remote IBD files might be moved as well to the new location. The problem with this is that the REDO log contains the old location @@ -4652,6 +4569,7 @@ both locations, we can chose on based on these priorities; @param[in] space_id tablespace ID @param[in] df Datafile object with path from redo @return true if a valid datafile was found, false if not */ +static bool fil_ibd_discover( ulint space_id, @@ -4813,11 +4731,7 @@ fil_ibd_load( fil_node_t* node = UT_LIST_GET_FIRST(space->chain); if (0 != strcmp(innobase_basename(filename), innobase_basename(node->name))) { -#ifdef UNIV_HOTBACKUP - ib::trace() -#else ib::info() -#endif /* UNIV_HOTBACKUP */ << "Ignoring data file '" << filename << "' with space ID " << space->id << ". Another data file called " << node->name @@ -4853,11 +4767,7 @@ fil_ibd_load( os_offset_t minimum_size; case DB_SUCCESS: if (file.space_id() != space_id) { -#ifdef UNIV_HOTBACKUP - ib::trace() -#else /* !UNIV_HOTBACKUP */ ib::info() -#endif /* UNIV_HOTBACKUP */ << "Ignoring data file '" << file.filepath() << "' with space ID " << file.space_id() @@ -4881,17 +4791,10 @@ fil_ibd_load( " single-table tablespace file '" << file.filepath() << "'"; } else if (size < minimum_size) { -#ifndef UNIV_HOTBACKUP ib::error() << "The size of tablespace file '" << file.filepath() << "' is only " << size << ", should be at least " << minimum_size << "!"; -#else - /* In MEB, we work around this error. */ - file.set_space_id(ULINT_UNDEFINED); - file.set_flags(0); - break; -#endif /* !UNIV_HOTBACKUP */ } else { /* Everything is fine so far. */ break; @@ -4900,12 +4803,6 @@ fil_ibd_load( /* Fall through to error handling */ case DB_TABLESPACE_EXISTS: -#ifdef UNIV_HOTBACKUP - if (file.flags() == ~(ulint)0) { - return FIL_LOAD_OK; - } -#endif /* UNIV_HOTBACKUP */ - return(FIL_LOAD_INVALID); default: @@ -4914,64 +4811,6 @@ fil_ibd_load( ut_ad(space == NULL); -#ifdef UNIV_HOTBACKUP - if (file.space_id() == ULINT_UNDEFINED || file.space_id() == 0) { - char* new_path; - - ib::info() << "Renaming tablespace file '" << file.filepath() - << "' with space ID " << file.space_id() << " to " - << file.name() << "_ibbackup_old_vers_" - " because its size " << size() << " is too small" - " (< 4 pages 16 kB each), or the space id in the" - " file header is not sensible. This can happen in" - " an mysqlbackup run, and is not dangerous."; - file.close(); - - new_path = fil_make_ibbackup_old_name(file.filepath()); - - bool success = os_file_rename( - innodb_data_file_key, file.filepath(), new_path); - - ut_a(success); - - ut_free(new_path); - - return(FIL_LOAD_ID_CHANGED); - } - - /* A backup may contain the same space several times, if the space got - renamed at a sensitive time. Since it is enough to have one version of - the space, we rename the file if a space with the same space id - already exists in the tablespace memory cache. We rather rename the - file than delete it, because if there is a bug, we do not want to - destroy valuable data. */ - - mutex_enter(&fil_system->mutex); - space = fil_space_get_by_id(space_id); - mutex_exit(&fil_system->mutex); - - if (space != NULL) { - ib::info() << "Renaming data file '" << file.filepath() - << "' with space ID " << space_id << " to " - << file.name() - << "_ibbackup_old_vers_ because space " - << space->name << " with the same id was scanned" - " earlier. This can happen if you have renamed tables" - " during an mysqlbackup run."; - file.close(); - - char* new_path = fil_make_ibbackup_old_name(file.filepath()); - - bool success = os_file_rename( - innodb_data_file_key, file.filepath(), new_path); - - ut_a(success); - - ut_free(new_path); - return(FIL_LOAD_OK); - } -#endif /* UNIV_HOTBACKUP */ - bool is_temp = FSP_FLAGS_GET_TEMPORARY(file.flags()); space = fil_space_create( file.name(), space_id, file.flags(), @@ -5066,7 +4905,6 @@ fil_report_missing_tablespace( " exists in the InnoDB internal data dictionary."; } -#ifndef UNIV_HOTBACKUP /** Returns true if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, there may be many tablespaces which are not yet in the memory cache. @@ -5243,7 +5081,7 @@ fil_space_for_table_exists_in_mem( return(false); } -#endif /* !UNIV_HOTBACKUP */ + /** Return the space ID based on the tablespace name. The tablespace must be found in the tablespace memory cache. This call is made from external to this module, so the mutex is not owned. @@ -5555,8 +5393,6 @@ fil_io( #endif ut_ad(fil_validate_skip()); -#ifndef UNIV_HOTBACKUP - /* ibuf bitmap pages must be read in the sync AIO mode: */ ut_ad(recv_no_ibuf_operations || req_type.is_write() @@ -5587,12 +5423,7 @@ fil_io( } else { mode = OS_AIO_NORMAL; } -#else /* !UNIV_HOTBACKUP */ - ut_a(sync); - ulint mode = OS_AIO_SYNC; -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP if (req_type.is_read()) { srv_stats.data_read.add(len); @@ -5604,7 +5435,6 @@ fil_io( srv_stats.data_written.add(len); } -#endif /* !UNIV_HOTBACKUP */ /* Reserve the fil_system mutex and make sure that we can open at least one file while holding it, if the file is not already open */ @@ -5807,33 +5637,14 @@ fil_io( req_type.block_size(node->block_size); - dberr_t err; - -#ifdef UNIV_HOTBACKUP - /* In mysqlbackup do normal i/o, not aio */ - if (req_type.is_read()) { - - err = os_file_read(req_type, node->handle, buf, offset, len); - - } else { - - ut_ad(!srv_read_only_mode - || fsp_is_system_temporary(page_id.space())); - - err = os_file_write( - req_type, node->name, node->handle, buf, offset, len); - } -#else /* UNIV_HOTBACKUP */ /* Queue the aio request */ - err = os_aio( + dberr_t err = os_aio( req_type, mode, name, node->handle, buf, offset, len, - fsp_is_system_temporary(page_id.space()) - ? false : srv_read_only_mode, + space->purpose != FIL_TYPE_TEMPORARY + && srv_read_only_mode, node, message, write_size); -#endif /* UNIV_HOTBACKUP */ - if (err == DB_IO_NO_PUNCH_HOLE) { err = DB_SUCCESS; @@ -5869,7 +5680,6 @@ fil_io( return(err); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Waits for an aio operation to complete. This function is used to write the handler for completed requests. The aio array of pending requests is divided @@ -5932,7 +5742,6 @@ fil_aio_wait( ut_ad(0); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Flushes to disk possible writes cached by the OS. If the space does not exist @@ -6135,7 +5944,6 @@ fil_page_set_type( mach_write_to_2(page + FIL_PAGE_TYPE, type); } -#ifndef UNIV_HOTBACKUP /** Reset the page type. Data files created before MySQL 5.1 may contain garbage in FIL_PAGE_TYPE. In MySQL 3.23.53, only undo log pages and index pages were tagged. @@ -6156,7 +5964,6 @@ fil_page_reset_type( << fil_page_get_type(page) << " to " << type << "."; mlog_write_ulint(page + FIL_PAGE_TYPE, type, MLOG_2BYTES, mtr); } -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Closes the tablespace memory cache. */ @@ -6182,7 +5989,6 @@ fil_close(void) } } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Initializes a buffer control block when the buf_pool is created. */ static @@ -6721,7 +6527,6 @@ fil_tablespace_iterate( return(err); } -#endif /* !UNIV_HOTBACKUP */ /** Set the tablespace table size. @param[in] page a page belonging to the tablespace */ @@ -6800,7 +6605,6 @@ fil_get_space_names( return(err); } -#ifndef UNIV_HOTBACKUP /** Generate redo log for swapping two .ibd files @param[in] old_table old table @param[in] new_table new table @@ -6899,7 +6703,7 @@ fil_mtr_rename_log( return(DB_SUCCESS); } -#endif /* !UNIV_HOTBACKUP */ + #ifdef UNIV_DEBUG /** Check that a tablespace is valid for mtr_commit(). @param[in] space persistent tablespace that has been changed */ @@ -6987,7 +6791,7 @@ fil_names_dirty_and_write( bogus_name, mtr); }); } -#ifndef UNIV_HOTBACKUP + /** On a log checkpoint, reset fil_names_dirty_and_write() flags and write out MLOG_FILE_NAME and MLOG_CHECKPOINT if needed. @param[in] lsn checkpoint LSN @@ -7159,7 +6963,6 @@ truncate_t::truncate( return(err); } -#endif /* !UNIV_HOTBACKUP */ /** Note that the file system where the file resides doesn't support PUNCH HOLE. diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 4479c06f1b203..98d732a9f9db3 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -48,14 +48,9 @@ Updated 14/02/2015 #include "trx0sys.h" #include "row0mysql.h" #include "ha_prototypes.h" // IB_LOG_ -#ifndef UNIV_HOTBACKUP -# include "buf0lru.h" -# include "ibuf0ibuf.h" -# include "sync0sync.h" -#else /* !UNIV_HOTBACKUP */ -# include "srv0srv.h" -static ulint srv_data_read, srv_data_written; -#endif /* !UNIV_HOTBACKUP */ +#include "buf0lru.h" +#include "ibuf0ibuf.h" +#include "sync0sync.h" #include "zlib.h" #ifdef __linux__ #include diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 4a7df5b33d3fb..0e1d2f3233b57 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -33,9 +33,6 @@ Created 2013-7-26 by Kevin Lewis #include "srv0start.h" #include "ut0new.h" #include "fil0crypt.h" -#ifdef UNIV_HOTBACKUP -#include "my_sys.h" -#endif /* UNIV_HOTBACKUP */ /** Initialize the name, size and order of this datafile @param[in] name tablespace name, will be copied @@ -459,39 +456,6 @@ Datafile::validate_for_recovery() switch (err) { case DB_SUCCESS: case DB_TABLESPACE_EXISTS: -#ifdef UNIV_HOTBACKUP - err = restore_from_doublewrite(0); - if (err != DB_SUCCESS) { - return(err); - } - /* Free the previously read first page and then re-validate. */ - free_first_page(); - err = validate_first_page(0, false); - if (err == DB_SUCCESS) { - std::string filepath = fil_space_get_first_path( - m_space_id); - if (is_intermediate_file(filepath.c_str())) { - /* Existing intermediate file with same space - id is obsolete.*/ - if (fil_space_free(m_space_id, FALSE)) { - err = DB_SUCCESS; - } - } else { - filepath.assign(m_filepath); - if (is_intermediate_file(filepath.c_str())) { - /* New intermediate file with same space id - shall be ignored.*/ - err = DB_TABLESPACE_EXISTS; - /* Set all bits of 'flags' as a special - indicator for "ignore tablespace". Hopefully - InnoDB will never use all bits or at least all - bits set will not be a meaningful setting - otherwise.*/ - m_flags = ~0; - } - } - } -#endif /* UNIV_HOTBACKUP */ break; default: diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 2ccad2801d16b..fe272accd5a7f 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -31,9 +31,6 @@ Created 11/29/1995 Heikki Tuuri #include "fsp0fsp.ic" #endif -#ifdef UNIV_HOTBACKUP -# include "fut0lst.h" -#else /* UNIV_HOTBACKUP */ #include "buf0buf.h" #include "fil0fil.h" #include "fil0crypt.h" @@ -215,7 +212,6 @@ fsp_flags_to_dict_tf( return(flags); } -#endif /* !UNIV_HOTBACKUP */ /** Validate the tablespace flags. These flags are stored in the tablespace header at offset FSP_SPACE_FLAGS. @@ -361,9 +357,7 @@ fsp_is_file_per_table( && !fsp_is_shared_tablespace(fsp_flags)); } -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG - /** Skip some of the sanity checks that are time consuming even in debug mode and can affect frequent verification runs that are done to ensure stability of the product. @@ -375,7 +369,6 @@ fsp_skip_sanity_check( return(srv_skip_temp_table_checks_debug && fsp_is_system_temporary(space_id)); } - #endif /* UNIV_DEBUG */ /**********************************************************************//** @@ -741,7 +734,6 @@ xdes_get_offset( + ((page_offset(descr) - XDES_ARR_OFFSET) / XDES_SIZE) * FSP_EXTENT_SIZE); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Inits a file page whose prior contents should be ignored. */ @@ -771,8 +763,7 @@ fsp_init_file_page_low( } } -#ifndef UNIV_HOTBACKUP -# ifdef UNIV_DEBUG +#ifdef UNIV_DEBUG /** Assert that the mini-transaction is compatible with updating an allocation bitmap page. @param[in] id tablespace identifier @@ -790,7 +781,6 @@ fsp_space_modify_check( when there is a higher-level redo log record written. */ break; case MTR_LOG_NO_REDO: -#ifdef UNIV_DEBUG { const fil_type_t type = fil_space_get_type(id); ut_a(srv_is_tablespace_truncated(id) @@ -800,7 +790,6 @@ fsp_space_modify_check( || type == FIL_TYPE_IMPORT || fil_space_is_redo_skipped(id)); } -#endif /* UNIV_DEBUG */ return; case MTR_LOG_ALL: /* We may only write redo log for a persistent tablespace. */ @@ -811,7 +800,7 @@ fsp_space_modify_check( ut_ad(0); } -# endif /* UNIV_DEBUG */ +#endif /* UNIV_DEBUG */ /** Initialize a file page. @param[in,out] block file page @@ -828,7 +817,6 @@ fsp_init_file_page( mlog_write_initial_log_record(buf_block_get_frame(block), MLOG_INIT_FILE_PAGE2, mtr); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of a file page init. @@ -889,8 +877,6 @@ fsp_header_init_fields( flags); } -#ifndef UNIV_HOTBACKUP - #if 0 /* MySQL 5.7 Encryption */ /** Fill the encryption info. @param[in] space tablespace @@ -1165,7 +1151,6 @@ fsp_header_init( return(true); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Reads the space id from the first page of a tablespace. @@ -1362,7 +1347,6 @@ fsp_header_get_encryption_key( } #endif /* ! */ -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Increases the space size field of a space. */ void @@ -4273,7 +4257,6 @@ fseg_print( fseg_print_low(inode, mtr); } #endif /* UNIV_BTR_PRINT */ -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /** Print the file segment header to the given output stream. diff --git a/storage/innobase/fsp/fsp0space.cc b/storage/innobase/fsp/fsp0space.cc index d50e35d0cbf91..45942f58dd340 100644 --- a/storage/innobase/fsp/fsp0space.cc +++ b/storage/innobase/fsp/fsp0space.cc @@ -27,14 +27,10 @@ Created 2012-11-16 by Sunny Bains as srv/srv0space.cc #include "fsp0space.h" #include "fsp0sysspace.h" -#ifndef UNIV_HOTBACKUP #include "fsp0fsp.h" #include "os0file.h" -#endif /* !UNIV_HOTBACKUP */ - #include "my_sys.h" - /** Check if two tablespaces have common data file names. @param other_space Tablespace to check against this. @return true if they have the same data filenames and paths */ @@ -180,7 +176,6 @@ Tablespace::find(const char* filename) return(false); } - /** Delete all the data files. */ void Tablespace::delete_files() diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index ac34be6f6a848..fa6a46890dbbe 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -29,7 +29,6 @@ Refactored 2013-7-26 by Kevin Lewis #include "fsp0sysspace.h" #include "srv0start.h" #include "trx0sys.h" -#ifndef UNIV_HOTBACKUP #include "dict0load.h" #include "mem0mem.h" #include "os0file.h" @@ -40,9 +39,6 @@ Refactored 2013-7-26 by Kevin Lewis If server passes the option for create/open DB to SE, we should remove such direct reference to server header and global variable */ #include "mysqld.h" -#else -my_bool opt_initialize = 0; -#endif /* !UNIV_HOTBACKUP */ /** The control info of the system tablespace. */ SysTablespace srv_sys_space; @@ -538,7 +534,6 @@ SysTablespace::open_file( return(err); } -#ifndef UNIV_HOTBACKUP /** Check the tablespace header for this tablespace. @param[out] flushed_lsn the value of FIL_PAGE_FILE_FLUSH_LSN @return DB_SUCCESS or error code */ @@ -613,7 +608,7 @@ SysTablespace::read_lsn_and_check_flags(lsn_t* flushed_lsn) return(DB_SUCCESS); } -#endif /* !UNIV_HOTBACKUP */ + /** Check if a file can be opened in the correct mode. @param[in] file data file object @param[out] reason exact reason if file_status check failed. @@ -760,7 +755,7 @@ SysTablespace::file_found( /* Need to create the system tablespace for new raw device. */ return(file.m_type == SRV_NEW_RAW); } -#ifndef UNIV_HOTBACKUP + /** Check the data file specification. @param[out] create_new_db true if a new database is to be created @param[in] min_expected_size Minimum expected tablespace size in bytes @@ -989,7 +984,7 @@ SysTablespace::open_or_create( return(err); } -#endif /* UNIV_HOTBACKUP */ + /** Normalize the file size, convert from megabytes to number of pages. */ void SysTablespace::normalize() diff --git a/storage/innobase/gis/gis0rtree.cc b/storage/innobase/gis/gis0rtree.cc index 9f8a3492db967..559ac2a6aa55b 100644 --- a/storage/innobase/gis/gis0rtree.cc +++ b/storage/innobase/gis/gis0rtree.cc @@ -28,8 +28,6 @@ Created 2013/03/27 Allen Lai and Jimmy Yang #include "page0cur.h" #include "page0zip.h" #include "gis0rtree.h" - -#ifndef UNIV_HOTBACKUP #include "btr0cur.h" #include "btr0sea.h" #include "btr0pcur.h" @@ -40,8 +38,6 @@ Created 2013/03/27 Allen Lai and Jimmy Yang #include "srv0mon.h" #include "gis0geo.h" -#endif /* UNIV_HOTBACKUP */ - /*************************************************************//** Initial split nodes info for R-tree split. @return initialized split nodes array */ diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 39ad4fe7d7d02..99543ef8c9c17 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -28,8 +28,6 @@ Created 2014/01/16 Jimmy Yang #include "page0cur.h" #include "page0zip.h" #include "gis0rtree.h" - -#ifndef UNIV_HOTBACKUP #include "btr0cur.h" #include "btr0sea.h" #include "btr0pcur.h" @@ -40,8 +38,6 @@ Created 2014/01/16 Jimmy Yang #include "srv0mon.h" #include "gis0geo.h" -#endif /* UNIV_HOTBACKUP */ - /*************************************************************//** Pop out used parent path entry, until we find the parent with matching page number */ @@ -1550,7 +1546,6 @@ rtr_copy_buf( (another) thread while it was copied. */ memcpy(&matches->block.page, &block->page, sizeof(buf_page_t)); matches->block.frame = block->frame; -#ifndef UNIV_HOTBACKUP matches->block.unzip_LRU = block->unzip_LRU; ut_d(matches->block.in_unzip_LRU_list = block->in_unzip_LRU_list); @@ -1572,7 +1567,6 @@ rtr_copy_buf( ut_d(matches->block.debug_latch = block->debug_latch); -#endif /* !UNIV_HOTBACKUP */ } /****************************************************************//** diff --git a/storage/innobase/ha/ha0ha.cc b/storage/innobase/ha/ha0ha.cc index f57a6d383d9a6..cf92f22ac1526 100644 --- a/storage/innobase/ha/ha0ha.cc +++ b/storage/innobase/ha/ha0ha.cc @@ -28,11 +28,10 @@ Created 8/22/1994 Heikki Tuuri #include "ha0ha.ic" #endif -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG # include "buf0buf.h" #endif /* UNIV_DEBUG */ -# include "btr0sea.h" +#include "btr0sea.h" #include "page0page.h" /*************************************************************//** @@ -537,4 +536,3 @@ builds, see http://bugs.mysql.com/36941 */ (ulong) n_bufs); } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/ha/hash0hash.cc b/storage/innobase/ha/hash0hash.cc index 234fd7ac03295..7c5798ae25461 100644 --- a/storage/innobase/ha/hash0hash.cc +++ b/storage/innobase/ha/hash0hash.cc @@ -32,8 +32,6 @@ Created 5/20/1997 Heikki Tuuri #include "mem0mem.h" #include "sync0sync.h" -#ifndef UNIV_HOTBACKUP - /************************************************************//** Reserves the mutex for a fold value in a hash table. */ void @@ -249,8 +247,6 @@ hash_unlock_x_all_but( } } -#endif /* !UNIV_HOTBACKUP */ - /*************************************************************//** Creates a hash table with >= n array cells. The actual number of cells is chosen to be a prime number slightly bigger than n. @@ -277,14 +273,12 @@ hash_create( table->type = HASH_TABLE_SYNC_NONE; table->array = array; table->n_cells = prime; -#ifndef UNIV_HOTBACKUP -# if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG +#if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG table->adaptive = FALSE; -# endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ +#endif /* UNIV_AHI_DEBUG || UNIV_DEBUG */ table->n_sync_obj = 0; table->sync_obj.mutexes = NULL; table->heaps = NULL; -#endif /* !UNIV_HOTBACKUP */ table->heap = NULL; ut_d(table->magic_n = HASH_TABLE_MAGIC_N); @@ -307,7 +301,6 @@ hash_table_free( ut_free(table); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Creates a sync object array to protect a hash table. ::sync_obj can be mutexes or rw_locks depening on the type of @@ -362,4 +355,3 @@ hash_create_sync_obj( table->n_sync_obj = n_sync_obj; } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 6940bbee5c2d4..612a2f9a69008 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -46,8 +46,6 @@ my_bool srv_ibuf_disable_background_merge; #include "ibuf0ibuf.ic" #endif -#ifndef UNIV_HOTBACKUP - #include "buf0buf.h" #include "buf0rea.h" #include "fsp0fsp.h" @@ -595,7 +593,6 @@ ibuf_max_size_update( } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Initializes an ibuf bitmap page. */ void @@ -618,10 +615,7 @@ ibuf_bitmap_page_init( memset(page + IBUF_BITMAP, 0, byte_offset); /* The remaining area (up to the page trailer) is uninitialized. */ - -#ifndef UNIV_HOTBACKUP mlog_write_initial_log_record(page, MLOG_IBUF_BITMAP_INIT, mtr); -#endif /* !UNIV_HOTBACKUP */ } /*********************************************************************//** @@ -644,7 +638,7 @@ ibuf_parse_bitmap_init( return(ptr); } -#ifndef UNIV_HOTBACKUP + # ifdef UNIV_DEBUG /** Gets the desired bits for a given page from a bitmap page. @param[in] page bitmap page @@ -2754,8 +2748,6 @@ ibuf_merge_in_background( } #endif /* UNIV_DEBUG || UNIV_IBUF_DEBUG */ - - while (sum_pages < n_pages) { ulint n_bytes; @@ -5132,5 +5124,3 @@ ibuf_set_bitmap_for_bulk_load( mtr_commit(&mtr); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/btr0btr.h b/storage/innobase/include/btr0btr.h index 9843eaf3cfeda..419c5db657d0c 100644 --- a/storage/innobase/include/btr0btr.h +++ b/storage/innobase/include/btr0btr.h @@ -37,7 +37,6 @@ Created 6/2/1994 Heikki Tuuri #include "btr0types.h" #include "gis0type.h" -#ifndef UNIV_HOTBACKUP /** Maximum record size which can be stored on a page, without using the special big record storage structure */ #define BTR_PAGE_MAX_REC_SIZE (UNIV_PAGE_SIZE / 2 - 200) @@ -140,7 +139,6 @@ record is in spatial index */ ((latch_mode) & ~(BTR_LATCH_FOR_INSERT \ | BTR_LATCH_FOR_DELETE \ | BTR_MODIFY_EXTERNAL)) -#endif /* UNIV_HOTBACKUP */ /**************************************************************//** Report that an index page is corrupted. */ @@ -161,7 +159,6 @@ btr_corruption_report( ut_error; \ } -#ifndef UNIV_HOTBACKUP /**************************************************************//** Gets the root node of a tree and sx-latches it for segment access. @return root page, sx-latched */ @@ -252,7 +249,6 @@ btr_page_get( dict_index_t* index, mtr_t* mtr) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /**************************************************************//** Gets the index id field of a page. @return index id */ @@ -262,7 +258,6 @@ btr_page_get_index_id( /*==================*/ const page_t* page) /*!< in: index page */ MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP /********************************************************//** Gets the node level field in an index page. @return level, leaf level == 0 */ @@ -514,9 +509,8 @@ btr_insert_on_non_leaf_level_func( const char* file, /*!< in: file name */ ulint line, /*!< in: line where called */ mtr_t* mtr); /*!< in: mtr */ -# define btr_insert_on_non_leaf_level(f,i,l,t,m) \ +#define btr_insert_on_non_leaf_level(f,i,l,t,m) \ btr_insert_on_non_leaf_level_func(f,i,l,t,__FILE__,__LINE__,m) -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Sets a record as the predefined minimum record. */ void @@ -525,7 +519,6 @@ btr_set_min_rec_mark( rec_t* rec, /*!< in/out: record */ mtr_t* mtr) /*!< in: mtr */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP /*************************************************************//** Deletes on the upper level the node pointer to a page. */ void @@ -578,7 +571,6 @@ btr_discard_page( btr_cur_t* cursor, /*!< in: cursor on the page to discard: not on the root page */ mtr_t* mtr); /*!< in: mtr */ -#endif /* !UNIV_HOTBACKUP */ /****************************************************************//** Parses the redo log record for setting an index record as the predefined minimum record. @@ -605,7 +597,6 @@ btr_parse_page_reorganize( buf_block_t* block, /*!< in: page to be reorganized, or NULL */ mtr_t* mtr) /*!< in: mtr or NULL */ MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP /**************************************************************//** Gets the number of pages in a B-tree. @return number of pages, or ULINT_UNDEFINED if the index is unavailable */ @@ -806,7 +797,6 @@ btr_lift_page_up( #define BTR_N_LEAF_PAGES 1 #define BTR_TOTAL_SIZE 2 -#endif /* !UNIV_HOTBACKUP */ #ifndef UNIV_NONINL #include "btr0btr.ic" diff --git a/storage/innobase/include/btr0btr.ic b/storage/innobase/include/btr0btr.ic index d01e19b5202f5..810ac8f8b67bd 100644 --- a/storage/innobase/include/btr0btr.ic +++ b/storage/innobase/include/btr0btr.ic @@ -25,7 +25,6 @@ Created 6/2/1994 Heikki Tuuri *******************************************************/ #include "mach0data.h" -#ifndef UNIV_HOTBACKUP #include "mtr0mtr.h" #include "mtr0log.h" #include "page0zip.h" @@ -132,8 +131,6 @@ btr_page_get( return ((page_t*)frame); } -#endif /* !UNIV_HOTBACKUP */ - /**************************************************************//** Gets the index id field of a page. @return index id */ @@ -146,7 +143,6 @@ btr_page_get_index_id( return(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID)); } -#ifndef UNIV_HOTBACKUP /********************************************************//** Gets the node level field in an index page. @return level, leaf level == 0 */ @@ -339,4 +335,3 @@ btr_leaf_page_release( mtr->memo_release(block, mode); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 579f725f6bd2d..a1e37ec46d048 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -60,7 +60,6 @@ struct btr_latch_leaves_t { ulint savepoints[3]; }; -#ifndef UNIV_HOTBACKUP #include "que0types.h" #include "row0types.h" #include "ha0ha.h" @@ -530,7 +529,6 @@ btr_cur_pessimistic_delete( bool rollback,/*!< in: performing rollback? */ mtr_t* mtr) /*!< in: mtr */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of updating a record in-place. @return end of log record or NULL */ @@ -565,7 +563,6 @@ btr_cur_parse_del_mark_set_sec_rec( byte* end_ptr,/*!< in: buffer end */ page_t* page, /*!< in/out: page or NULL */ page_zip_des_t* page_zip);/*!< in/out: compressed page, or NULL */ -#ifndef UNIV_HOTBACKUP /** Estimates the number of rows in a given index range. @param[in] index index @@ -941,7 +938,6 @@ btr_rec_set_deleted_flag( page_zip_des_t* page_zip,/*!< in/out: compressed page (or NULL) */ ulint flag); /*!< in: nonzero if delete marked */ - /** If pessimistic delete fails because of lack of file space, there is still a good change of success a little later. Try this many times. */ @@ -995,7 +991,6 @@ extern ulint btr_cur_n_non_sea_old; srv_refresh_innodb_monitor_stats(). Referenced by srv_printf_innodb_monitor(). */ extern ulint btr_cur_n_sea_old; -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /* Flag to limit optimistic insert records */ diff --git a/storage/innobase/include/btr0cur.ic b/storage/innobase/include/btr0cur.ic index 45c0d59a8aad6..b1e59651a1dbb 100644 --- a/storage/innobase/include/btr0cur.ic +++ b/storage/innobase/include/btr0cur.ic @@ -23,7 +23,6 @@ The index tree cursor Created 10/16/1994 Heikki Tuuri *******************************************************/ -#ifndef UNIV_HOTBACKUP #include "btr0btr.h" #ifdef UNIV_DEBUG @@ -231,5 +230,3 @@ btr_rec_set_deleted_flag( rec_set_deleted_flag_old(rec, flag); } } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/btr0defragment.h b/storage/innobase/include/btr0defragment.h index 6257c4bc996cc..21ba6d9f426ff 100644 --- a/storage/innobase/include/btr0defragment.h +++ b/storage/innobase/include/btr0defragment.h @@ -20,10 +20,6 @@ this program; if not, write to the Free Software Foundation, Inc., #ifndef btr0defragment_h #define btr0defragment_h -#include "univ.i" - -#ifndef UNIV_HOTBACKUP - #include "btr0pcur.h" /* Max number of pages to consider at once during defragmentation. */ @@ -98,7 +94,4 @@ DECLARE_THREAD(btr_defragment_thread)( /*==========================================*/ void* arg); /*!< in: a dummy parameter required by os_thread_create */ - - -#endif /* !UNIV_HOTBACKUP */ #endif diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 617bb2b9b5da8..e1c92c130c4d0 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -27,6 +27,9 @@ Created 11/5/1995 Heikki Tuuri #ifndef buf0buf_h #define buf0buf_h +/** Magic value to use instead of checksums when they are disabled */ +#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL + #include "univ.i" #include "fil0fil.h" #include "mtr0types.h" @@ -35,7 +38,6 @@ Created 11/5/1995 Heikki Tuuri #include "hash0hash.h" #include "ut0byte.h" #include "page0types.h" -#ifndef UNIV_HOTBACKUP #include "ut0rbt.h" #include "os0proc.h" #include "log0log.h" @@ -99,20 +101,11 @@ extern volatile ulint buf_withdraw_clock; /*!< the clock is incremented every time a pointer to a page may become obsolete */ -#ifdef UNIV_DEBUG +# ifdef UNIV_DEBUG extern my_bool buf_disable_resize_buffer_pool_debug; /*!< if TRUE, resizing buffer pool is not allowed. */ -#endif /* UNIV_DEBUG */ -#else /* !UNIV_HOTBACKUP */ -extern buf_block_t* back_block1; /*!< first block, for --apply-log */ -extern buf_block_t* back_block2; /*!< second block, for page reorganize */ -#endif /* !UNIV_HOTBACKUP */ -#endif /* !UNIV_INNOCHECKSUM */ - -/** Magic value to use instead of checksums when they are disabled */ -#define BUF_NO_CHECKSUM_MAGIC 0xDEADBEEFUL +# endif /* UNIV_DEBUG */ -#ifndef UNIV_INNOCHECKSUM /** @brief States of a control block @see buf_page_t @@ -341,7 +334,6 @@ operator<<( std::ostream& out, const page_id_t& page_id); -#ifndef UNIV_HOTBACKUP /********************************************************************//** Acquire mutex on all buffer pool instances */ UNIV_INLINE @@ -468,7 +460,7 @@ void buf_block_free( /*===========*/ buf_block_t* block); /*!< in, own: block to be freed */ -#endif /* !UNIV_HOTBACKUP */ + /*********************************************************************//** Copies contents of a buffer frame to a given buffer. @return buf */ @@ -478,7 +470,7 @@ buf_frame_copy( /*===========*/ byte* buf, /*!< in: buffer to copy to */ const buf_frame_t* frame); /*!< in: buffer frame */ -#ifndef UNIV_HOTBACKUP + /**************************************************************//** NOTE! The following macros should be used instead of buf_page_get_gen, to improve debugging. Only values RW_S_LATCH and RW_X_LATCH are allowed @@ -597,21 +589,6 @@ buf_page_create( const page_size_t& page_size, mtr_t* mtr); -#else /* !UNIV_HOTBACKUP */ - -/** Inits a page to the buffer buf_pool, for use in mysqlbackup --restore. -@param[in] page_id page id -@param[in] page_size page size -@param[in,out] block block to init */ -void -buf_page_init_for_backup_restore( - const page_id_t& page_id, - const page_size_t& page_size, - buf_block_t* block); - -#endif /* !UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP /********************************************************************//** Releases a compressed-only page acquired with buf_page_get_zip(). */ UNIV_INLINE @@ -788,17 +765,14 @@ buf_block_unfix( @param[in,out] b block to bufferfix @param[in] f file name where requested @param[in] l line number where requested */ -# define buf_block_buf_fix_inc(b,f,l) buf_block_buf_fix_inc_func(f,l,b) +# define buf_block_buf_fix_inc(b,f,l) buf_block_buf_fix_inc_func(f,l,b) # else /* UNIV_DEBUG */ /** Increments the bufferfix count. @param[in,out] b block to bufferfix @param[in] f file name where requested @param[in] l line number where requested */ -# define buf_block_buf_fix_inc(b,f,l) buf_block_buf_fix_inc_func(b) +# define buf_block_buf_fix_inc(b,f,l) buf_block_buf_fix_inc_func(b) # endif /* UNIV_DEBUG */ -#else /* !UNIV_HOTBACKUP */ -# define buf_block_modify_clock_inc(block) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_INNOCHECKSUM */ /** Checks if a page contains only zeroes. @@ -835,7 +809,6 @@ buf_page_is_corrupted( #endif /* UNIV_INNOCHECKSUM */ ) MY_ATTRIBUTE((warn_unused_result)); #ifndef UNIV_INNOCHECKSUM -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the space id, page offset, and byte offset within page of a pointer pointing to a buffer frame containing a file page. */ @@ -893,7 +866,7 @@ void buf_print(void); /*============*/ #endif /* UNIV_DEBUG_PRINT || UNIV_DEBUG || UNIV_BUF_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ + enum buf_page_print_flags { /** Do not crash at the end of buf_page_print(). */ BUF_PAGE_PRINT_NO_CRASH = 1, @@ -920,7 +893,7 @@ buf_zip_decompress( /*===============*/ buf_block_t* block, /*!< in/out: block */ ibool check); /*!< in: TRUE=verify the page checksum */ -#ifndef UNIV_HOTBACKUP + #ifdef UNIV_DEBUG /*********************************************************************//** Returns the number of latched pages in the buffer pool. @@ -990,7 +963,6 @@ this function is called: not latched and not modified. */ void buf_pool_invalidate(void); /*=====================*/ -#endif /* !UNIV_HOTBACKUP */ /*======================================================================== --------------------------- LOWER LEVEL ROUTINES ------------------------- @@ -1063,7 +1035,7 @@ buf_page_in_file( /*=============*/ const buf_page_t* bpage) /*!< in: pointer to control block */ MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Determines if a block should be on unzip_LRU list. @return TRUE if block belongs to unzip_LRU */ @@ -1221,7 +1193,7 @@ buf_page_get_block( /*===============*/ buf_page_t* bpage) /*!< in: control block, or NULL */ MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ + #ifdef UNIV_DEBUG /*********************************************************************//** Gets a pointer to the memory frame of a block. @@ -1241,7 +1213,6 @@ Gets the compressed page descriptor corresponding to an uncompressed page if applicable. */ #define buf_block_get_page_zip(block) \ ((block)->page.zip.data ? &(block)->page.zip : NULL) -#ifndef UNIV_HOTBACKUP /** Get a buffer block from an adaptive hash index pointer. This function does not return if the block is not identified. @@ -1425,13 +1396,6 @@ buf_page_hash_get_low() function. #define buf_block_hash_get(b, page_id) \ buf_block_hash_get_locked(b, page_id, NULL, 0) -/*********************************************************************//** -Gets the current length of the free list of buffer blocks. -@return length of the free list */ -ulint -buf_get_free_list_len(void); -/*=======================*/ - /********************************************************************//** Determine if a block is a sentinel for a buffer pool watch. @return TRUE if a sentinel for a buffer pool watch, FALSE if not */ @@ -1531,8 +1495,6 @@ buf_flush_update_zip_checksum( ulint size, lsn_t lsn); -#endif /* !UNIV_HOTBACKUP */ - /********************************************************************//** The hook that is called just before a page is written to disk. The function encrypts the content of the page and returns a pointer @@ -1635,7 +1597,6 @@ class buf_page_t { /** Block state. @see buf_page_in_file */ buf_page_state state; -#ifndef UNIV_HOTBACKUP unsigned flush_type:2; /*!< if this block is currently being flushed to disk, this tells the flush_type. @@ -1646,7 +1607,6 @@ class buf_page_t { # error "MAX_BUFFER_POOLS > 64; redefine buf_pool_index:6" # endif /* @} */ -#endif /* !UNIV_HOTBACKUP */ page_zip_des_t zip; /*!< compressed page; zip.data (but not the data it points to) is also protected by buf_pool->mutex; @@ -1678,7 +1638,6 @@ class buf_page_t { buf_tmp_buffer_t* slot; /*!< Slot for temporary memory used for encryption/compression or NULL */ - #ifndef UNIV_HOTBACKUP buf_page_t* hash; /*!< node used in chaining to buf_pool->page_hash or buf_pool->zip_hash */ @@ -1792,7 +1751,6 @@ class buf_page_t { protected by buf_pool->zip_mutex or buf_block_t::mutex. */ # endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ }; /** The buffer control block structure */ @@ -1810,7 +1768,6 @@ struct buf_block_t{ is of size UNIV_PAGE_SIZE, and aligned to an address divisible by UNIV_PAGE_SIZE */ -#ifndef UNIV_HOTBACKUP BPageLock lock; /*!< read-write lock of the buffer frame */ UT_LIST_NODE_T(buf_block_t) unzip_LRU; @@ -1916,7 +1873,6 @@ struct buf_block_t{ and accessed; we introduce this new mutex in InnoDB-5.1 to relieve contention on the buffer pool mutex */ -#endif /* !UNIV_HOTBACKUP */ }; /** Check if a buf_block_t object is in a valid state @@ -1927,7 +1883,6 @@ struct buf_block_t{ && (buf_block_get_state(block) <= BUF_BLOCK_REMOVE_HASH)) -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Compute the hash fold value for blocks in buf_pool->zip_hash. */ /* @{ */ @@ -2462,7 +2417,6 @@ Use these instead of accessing buf_pool->mutex directly. */ /** Release the buffer pool mutex. */ # define buf_pool_mutex_exit(b) mutex_exit(&b->mutex) #endif -#endif /* !UNIV_HOTBACKUP */ /* @} */ /********************************************************************** diff --git a/storage/innobase/include/buf0buf.ic b/storage/innobase/include/buf0buf.ic index 9ff2a1bfab5ea..f2b1b1515985f 100644 --- a/storage/innobase/include/buf0buf.ic +++ b/storage/innobase/include/buf0buf.ic @@ -32,7 +32,6 @@ Created 11/5/1995 Heikki Tuuri *******************************************************/ #include "mtr0mtr.h" -#ifndef UNIV_HOTBACKUP #include "buf0flu.h" #include "buf0lru.h" #include "buf0rea.h" @@ -208,7 +207,6 @@ buf_page_peek_if_too_old( return(!buf_page_peek_if_young(bpage)); } } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets the state of a block. @@ -385,7 +383,6 @@ buf_page_in_file( return(FALSE); } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Determines if a block should be on unzip_LRU list. @return TRUE if block belongs to unzip_LRU */ @@ -717,7 +714,6 @@ buf_page_get_block( return(NULL); } -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /*********************************************************************//** @@ -744,9 +740,7 @@ buf_block_get_frame( ut_error; break; case BUF_BLOCK_FILE_PAGE: -# ifndef UNIV_HOTBACKUP ut_a(block->page.buf_fix_count > 0); -# endif /* !UNIV_HOTBACKUP */ /* fall through */ case BUF_BLOCK_READY_FOR_USE: case BUF_BLOCK_MEMORY: @@ -796,7 +790,6 @@ buf_ptr_get_fsp_addr( addr->boffset = ut_align_offset(ptr, UNIV_PAGE_SIZE); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets the hash value of the page the pointer is pointing to. This can be used in searches in the lock hash table. @@ -866,7 +859,6 @@ buf_block_free( buf_pool_mutex_exit(buf_pool); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Copies contents of a buffer frame to a given buffer. @@ -885,7 +877,6 @@ buf_frame_copy( return(buf); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Gets the youngest modification log sequence number for a frame. Returns zero if not file page or no modification occurred yet. @@ -1460,5 +1451,3 @@ buf_pool_size_align( return (ulint)((size / m + 1) * m); } } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/buf0dblwr.h b/storage/innobase/include/buf0dblwr.h index eb13c3b35e500..fdb9ab15a2403 100644 --- a/storage/innobase/include/buf0dblwr.h +++ b/storage/innobase/include/buf0dblwr.h @@ -32,8 +32,6 @@ Created 2011/12/19 Inaam Rana #include "buf0types.h" #include "log0recv.h" -#ifndef UNIV_HOTBACKUP - /** Doublewrite system */ extern buf_dblwr_t* buf_dblwr; /** Set to TRUE when the doublewrite buffer is being created */ @@ -156,7 +154,4 @@ struct buf_dblwr_t{ cached to write_buf */ }; - -#endif /* UNIV_HOTBACKUP */ - #endif diff --git a/storage/innobase/include/buf0flu.h b/storage/innobase/include/buf0flu.h index 40083798d48a3..273f9666419e4 100644 --- a/storage/innobase/include/buf0flu.h +++ b/storage/innobase/include/buf0flu.h @@ -30,7 +30,6 @@ Created 11/5/1995 Heikki Tuuri #include "univ.i" #include "ut0byte.h" #include "log0log.h" -#ifndef UNIV_HOTBACKUP #include "buf0types.h" /** Flag indicating if the page_cleaner is in active state. */ @@ -77,7 +76,6 @@ void buf_flush_write_complete( /*=====================*/ buf_page_t* bpage); /*!< in: pointer to the block in question */ -#endif /* !UNIV_HOTBACKUP */ /** Initialize a page for writing to the tablespace. @param[in] block buffer block; NULL if bypassing the buffer pool @param[in,out] page page frame @@ -92,7 +90,6 @@ buf_flush_init_for_writing( lsn_t newest_lsn, bool skip_checksum); -#ifndef UNIV_HOTBACKUP # if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG /********************************************************************//** Writes a flushable page asynchronously from the buffer pool to a file. @@ -128,7 +125,6 @@ buf_flush_do_batch( lsn_t lsn_limit, flush_counters_t* n); - /** This utility flushes dirty blocks from the end of the flush list of all buffer pool instances. NOTE: The calling thread is not allowed to own any latches on pages! @@ -438,8 +434,6 @@ class FlushObserver { bool m_interrupted; }; -#endif /* !UNIV_HOTBACKUP */ - /******************************************************************//** Start a buffer flush batch for LRU or flush list */ ibool diff --git a/storage/innobase/include/buf0flu.ic b/storage/innobase/include/buf0flu.ic index ecb98e3261950..5a682ed121a8c 100644 --- a/storage/innobase/include/buf0flu.ic +++ b/storage/innobase/include/buf0flu.ic @@ -23,7 +23,6 @@ The database buffer pool flush algorithm Created 11/5/1995 Heikki Tuuri *******************************************************/ -#ifndef UNIV_HOTBACKUP #include "buf0buf.h" #include "mtr0mtr.h" #include "srv0srv.h" @@ -148,4 +147,3 @@ buf_flush_recv_note_modification( buf_page_mutex_exit(block); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h index 0cbd77878ec32..2ad1211a5487a 100644 --- a/storage/innobase/include/buf0lru.h +++ b/storage/innobase/include/buf0lru.h @@ -27,7 +27,6 @@ Created 11/5/1995 Heikki Tuuri #define buf0lru_h #include "univ.i" -#ifndef UNIV_HOTBACKUP #include "ut0byte.h" #include "buf0types.h" @@ -292,6 +291,4 @@ Increments the page_zip_decompress() counter in buf_LRU_stat_cur. */ #include "buf0lru.ic" #endif -#endif /* !UNIV_HOTBACKUP */ - #endif diff --git a/storage/innobase/include/data0data.h b/storage/innobase/include/data0data.h index 5537d70548a71..8385fe3d0d3a6 100644 --- a/storage/innobase/include/data0data.h +++ b/storage/innobase/include/data0data.h @@ -196,7 +196,7 @@ dfield_dup( dfield_t* field, /*!< in/out: data field */ mem_heap_t* heap) /*!< in: memory heap where allocated */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Tests if two data fields are equal. If len==0, tests the data length and content for equality. @@ -222,7 +222,6 @@ dfield_data_is_binary_equal( ulint len, /*!< in: data length or UNIV_SQL_NULL */ const byte* data) /*!< in: data */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets number of fields in a data tuple. @return number of fields */ @@ -338,7 +337,6 @@ dtuple_create( ulint n_fields)/*!< in: number of fields */ MY_ATTRIBUTE((nonnull, malloc)); - /** Initialize the virtual field data in a dtuple_t @param[in,out] vrow dtuple contains the virtual fields */ UNIV_INLINE @@ -631,7 +629,6 @@ struct dtuple_t { #endif /* UNIV_DEBUG */ }; - /** A slot for a field in a big rec vector */ struct big_rec_field_t { diff --git a/storage/innobase/include/data0data.ic b/storage/innobase/include/data0data.ic index dc51735d340df..71698cc12b36e 100644 --- a/storage/innobase/include/data0data.ic +++ b/storage/innobase/include/data0data.ic @@ -285,7 +285,6 @@ dfield_dup( } } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Tests if two data fields are equal. If len==0, tests the data length and content for equality. @@ -330,7 +329,6 @@ dfield_data_is_binary_equal( && (len == UNIV_SQL_NULL || !memcmp(dfield_get_data(field), data, len))); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets info bits in a data tuple. diff --git a/storage/innobase/include/data0type.h b/storage/innobase/include/data0type.h index 27310963ec577..d38aa7c15339d 100644 --- a/storage/innobase/include/data0type.h +++ b/storage/innobase/include/data0type.h @@ -260,7 +260,6 @@ the underling datatype of GEOMETRY(not DATA_POINT) data. */ /* Mask to get the Charset Collation number (0x7fff) */ #define CHAR_COLL_MASK MAX_CHAR_COLL_NUM -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Gets the MySQL type code from a dtype. @return MySQL type code; this is NOT an InnoDB type code! */ @@ -286,7 +285,6 @@ dtype_get_at_most_n_mbchars( ulint data_len, /*!< in: length of str (in bytes) */ const char* str); /*!< in: the string whose prefix length is being determined */ -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Checks if a data main type is a string type. Also a BLOB is considered a string type. @@ -350,7 +348,7 @@ ulint dtype_get_prtype( /*=============*/ const dtype_t* type); /*!< in: data type */ -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Compute the mbminlen and mbmaxlen members of a data type structure. */ UNIV_INLINE @@ -391,7 +389,6 @@ ibool dtype_is_utf8( /*==========*/ ulint prtype);/*!< in: precise data type */ -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets the type length. @return fixed length of the type, in bytes, or 0 if variable-length */ @@ -400,7 +397,7 @@ ulint dtype_get_len( /*==========*/ const dtype_t* type); /*!< in: data type */ -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Gets the minimum length of a character, in bytes. @return minimum length of a char, in bytes, or 0 if this is not a @@ -432,7 +429,6 @@ dtype_set_mbminmaxlen( ulint mbmaxlen); /*!< in: maximum length of a char, in bytes, or 0 if this is not a character type */ -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Returns the size of a fixed size data type, 0 if not a fixed size type. @return fixed size, or 0 */ @@ -446,7 +442,7 @@ dtype_get_fixed_size_low( ulint mbminmaxlen, /*!< in: minimum and maximum length of a multibyte character, in bytes */ ulint comp); /*!< in: nonzero=ROW_FORMAT=COMPACT */ -#ifndef UNIV_HOTBACKUP + /***********************************************************************//** Returns the minimum size of a data type. @return minimum size */ @@ -469,7 +465,6 @@ dtype_get_max_size_low( /*===================*/ ulint mtype, /*!< in: main type */ ulint len); /*!< in: length */ -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Returns the ROW_FORMAT=REDUNDANT stored SQL NULL size of a type. For fixed length types it is the fixed length of the type, otherwise 0. @@ -480,7 +475,7 @@ dtype_get_sql_null_size( /*====================*/ const dtype_t* type, /*!< in: type */ ulint comp); /*!< in: nonzero=ROW_FORMAT=COMPACT */ -#ifndef UNIV_HOTBACKUP + /**********************************************************************//** Reads to a type the stored information which determines its alphabetical ordering and the storage size of an SQL NULL value. */ @@ -528,8 +523,6 @@ dtype_sql_name( char* name, /*!< out: SQL name */ unsigned name_sz);/*!< in: size of the name buffer */ -#endif /* !UNIV_HOTBACKUP */ - /*********************************************************************//** Validates a data type structure. @return TRUE if ok */ @@ -572,13 +565,11 @@ struct dtype_t{ string data (in addition to the string, MySQL uses 1 or 2 bytes to store the string length) */ -#ifndef UNIV_HOTBACKUP unsigned mbminmaxlen:5; /*!< minimum and maximum length of a character, in bytes; DATA_MBMINMAXLEN(mbminlen,mbmaxlen); mbminlen=DATA_MBMINLEN(mbminmaxlen); mbmaxlen=DATA_MBMINLEN(mbminmaxlen) */ -#endif /* !UNIV_HOTBACKUP */ }; #ifndef UNIV_NONINL diff --git a/storage/innobase/include/data0type.ic b/storage/innobase/include/data0type.ic index 57770ec0e17fa..869d0ba8ed653 100644 --- a/storage/innobase/include/data0type.ic +++ b/storage/innobase/include/data0type.ic @@ -24,8 +24,7 @@ Created 1/16/1996 Heikki Tuuri *******************************************************/ #include "mach0data.h" -#ifndef UNIV_HOTBACKUP -# include "ha_prototypes.h" +#include "ha_prototypes.h" /*********************************************************************//** Gets the MySQL charset-collation code for MySQL string types. @@ -137,9 +136,6 @@ dtype_set_mblen( ut_ad(dtype_validate(type)); } -#else /* !UNIV_HOTBACKUP */ -# define dtype_set_mblen(type) (void) 0 -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Sets a data type structure. */ @@ -218,7 +214,6 @@ dtype_get_len( return(type->len); } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Gets the minimum length of a character, in bytes. @return minimum length of a char, in bytes, or 0 if this is not a @@ -471,8 +466,6 @@ dtype_sql_name( return(name); } -#endif /* !UNIV_HOTBACKUP */ - /***********************************************************************//** Returns the size of a fixed size data type, 0 if not a fixed size type. @return fixed size, or 0 */ @@ -513,7 +506,6 @@ dtype_get_fixed_size_low( case DATA_POINT: return(len); case DATA_MYSQL: -#ifndef UNIV_HOTBACKUP if (prtype & DATA_BINARY_TYPE) { return(len); } else if (!comp) { @@ -534,9 +526,6 @@ dtype_get_fixed_size_low( return(len); } } -#else /* !UNIV_HOTBACKUP */ - return(len); -#endif /* !UNIV_HOTBACKUP */ /* fall through for variable-length charsets */ case DATA_VARCHAR: case DATA_BINARY: @@ -553,7 +542,6 @@ dtype_get_fixed_size_low( return(0); } -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Returns the minimum size of a data type. @return minimum size */ @@ -659,7 +647,6 @@ dtype_get_max_size_low( return(ULINT_MAX); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Returns the ROW_FORMAT=REDUNDANT stored SQL NULL size of a type. @@ -672,11 +659,6 @@ dtype_get_sql_null_size( const dtype_t* type, /*!< in: type */ ulint comp) /*!< in: nonzero=ROW_FORMAT=COMPACT */ { -#ifndef UNIV_HOTBACKUP return(dtype_get_fixed_size_low(type->mtype, type->prtype, type->len, type->mbminmaxlen, comp)); -#else /* !UNIV_HOTBACKUP */ - return(dtype_get_fixed_size_low(type->mtype, type->prtype, type->len, - 0, 0)); -#endif /* !UNIV_HOTBACKUP */ } diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index d42a5f6ee4bc8..be6f6b1d9a599 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -51,8 +51,7 @@ Created 1/8/1996 Heikki Tuuri extern bool innodb_table_stats_not_found; extern bool innodb_index_stats_not_found; -#ifndef UNIV_HOTBACKUP -# include "sync0rw.h" +#include "sync0rw.h" /********************************************************************//** Get the database name length in a table name. @return database name length */ @@ -228,7 +227,6 @@ dict_max_v_field_len_store_undo( dict_table_t* table, ulint col_no); -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /*********************************************************************//** Assert that a column and a data type match. @@ -241,7 +239,7 @@ dict_col_type_assert_equal( const dtype_t* type) /*!< in: data type */ MY_ATTRIBUTE((nonnull, warn_unused_result)); #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP + /***********************************************************************//** Returns the minimum size of the column. @return minimum size */ @@ -380,7 +378,6 @@ dict_table_autoinc_unlock( /*======================*/ dict_table_t* table) /*!< in/out: table */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Adds system columns to a table object. */ void @@ -389,7 +386,7 @@ dict_table_add_system_columns( dict_table_t* table, /*!< in/out: table */ mem_heap_t* heap) /*!< in: temporary heap */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP + /** Mark if table has big rows. @param[in,out] table table handler */ void @@ -731,7 +728,6 @@ dict_table_get_next_index( # define dict_table_get_last_index(table) UT_LIST_GET_LAST((table)->indexes) # define dict_table_get_next_index(index) UT_LIST_GET_NEXT(indexes, index) #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /* Skip corrupted index */ #define dict_table_skip_corrupt_index(index) \ @@ -894,7 +890,6 @@ dict_table_n_rows_dec( dict_table_t* table) /*!< in/out: table */ MY_ATTRIBUTE((nonnull)); - /** Get nth virtual column @param[in] table target table @param[in] col_nr column number in MySQL Table definition @@ -953,7 +948,7 @@ dict_table_get_sys_col_no( const dict_table_t* table, /*!< in: table */ ulint sys) /*!< in: DATA_ROW_ID, ... */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#ifndef UNIV_HOTBACKUP + /********************************************************************//** Returns the minimum data size of an index record. @return minimum data size in bytes */ @@ -963,7 +958,6 @@ dict_index_get_min_size( /*====================*/ const dict_index_t* index) /*!< in: index */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Check whether the table uses the compact page format. @return TRUE if table uses the compact page format */ @@ -1076,7 +1070,6 @@ dict_table_page_size( const dict_table_t* table) MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Obtain exclusive locks on all index trees of the table. This is to prevent accessing index trees while InnoDB is updating internal metadata for @@ -1206,7 +1199,6 @@ dict_index_add_to_cache_w_vcol( ulint page_no, ibool strict) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Gets the number of fields in the internal representation of an index, including fields added by the dictionary system. @@ -1397,7 +1389,7 @@ dict_index_add_col( dict_col_t* col, /*!< in: column */ ulint prefix_len) /*!< in: column prefix length */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP + /*******************************************************************//** Copies types of fields contained in index to tuple. */ void @@ -1408,7 +1400,6 @@ dict_index_copy_types( ulint n_fields) /*!< in: number of field types to copy */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Gets the field column. @return field->col, pointer to the table column */ @@ -1418,7 +1409,7 @@ dict_field_get_col( /*===============*/ const dict_field_t* field) /*!< in: index field */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#ifndef UNIV_HOTBACKUP + /**********************************************************************//** Returns an index object if it is found in the dictionary cache. Assumes that dict_sys->mutex is already being held. @@ -1789,7 +1780,6 @@ struct dict_sys_t{ table_non_LRU; /*!< List of tables that can't be evicted from the cache */ }; -#endif /* !UNIV_HOTBACKUP */ /** dummy index for ROW_FORMAT=REDUNDANT supremum and infimum records */ extern dict_index_t* dict_ind_redundant; @@ -1881,7 +1871,7 @@ Closes the data dictionary module. */ void dict_close(void); /*============*/ -#ifndef UNIV_HOTBACKUP + /**********************************************************************//** Check whether the table is corrupted. @return nonzero for corrupted table, zero for valid tables */ @@ -1902,7 +1892,6 @@ dict_index_is_corrupted( const dict_index_t* index) /*!< in: index */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Flags an index and table corrupted both in the data dictionary cache and in the system table SYS_INDEXES. */ @@ -2014,7 +2003,6 @@ dict_disable_redo_if_temporary( const dict_table_t* table, /*!< in: table to check */ mtr_t* mtr); /*!< out: mini-transaction */ -#ifndef UNIV_HOTBACKUP /*********************************************************************//** This function should be called whenever a page is successfully compressed. Updates the compression padding information. */ @@ -2115,9 +2103,6 @@ bool dict_table_have_virtual_index( dict_table_t* table); -#endif /* !UNIV_HOTBACKUP */ - - #ifndef UNIV_NONINL #include "dict0dict.ic" #endif diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 9bf6b8df4bc9b..b99cb421ab260 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -25,7 +25,6 @@ Created 1/8/1996 Heikki Tuuri ***********************************************************************/ #include "data0type.h" -#ifndef UNIV_HOTBACKUP #include "dict0load.h" #include "rem0types.h" #include "fsp0fsp.h" @@ -101,8 +100,6 @@ dict_col_is_virtual( return(col->prtype & DATA_VIRTUAL); } -#endif /* !UNIV_HOTBACKUP */ - #ifdef UNIV_DEBUG /*********************************************************************//** Assert that a column and a data type match. @@ -120,15 +117,12 @@ dict_col_type_assert_equal( ut_ad(col->mtype == type->mtype); ut_ad(col->prtype == type->prtype); //ut_ad(col->len == type->len); -# ifndef UNIV_HOTBACKUP ut_ad(col->mbminmaxlen == type->mbminmaxlen); -# endif /* !UNIV_HOTBACKUP */ return(TRUE); } #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Returns the minimum size of the column. @return minimum size */ @@ -152,7 +146,6 @@ dict_col_get_max_size( { return(dtype_get_max_size_low(col->mtype, col->len)); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Returns the size of a fixed size column, 0 if not a fixed size column. @return fixed size, or 0 */ @@ -245,7 +238,6 @@ dict_col_get_index_pos( return(ULINT_UNDEFINED); } -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG /********************************************************************//** Gets the first index on the table (the clustered index). @@ -293,7 +285,6 @@ dict_table_get_next_index( return(UT_LIST_GET_NEXT(indexes, (dict_index_t*) index)); } #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Check whether the index is the clustered index. @@ -1191,7 +1182,6 @@ dict_table_page_size( return(dict_tf_get_page_size(table->flags)); } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Obtain exclusive locks on all index trees of the table. This is to prevent accessing index trees while InnoDB is updating internal metadata for @@ -1234,7 +1224,6 @@ dict_table_x_unlock_indexes( rw_lock_x_unlock(dict_index_get_lock(index)); } } -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Gets the number of fields in the internal representation of an index, @@ -1440,7 +1429,6 @@ dict_index_get_nth_col_pos( prefix_col_pos)); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Returns the minimum data size of an index record. @return minimum data size in bytes */ @@ -1974,4 +1962,3 @@ dict_table_have_virtual_index( return(false); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/dict0mem.h b/storage/innobase/include/dict0mem.h index 74bb00b730b02..538306f7af83e 100644 --- a/storage/innobase/include/dict0mem.h +++ b/storage/innobase/include/dict0mem.h @@ -35,11 +35,9 @@ Created 1/8/1996 Heikki Tuuri #include "row0types.h" #include "rem0types.h" #include "btr0types.h" -#ifndef UNIV_HOTBACKUP -# include "lock0types.h" -# include "que0types.h" -# include "sync0rw.h" -#endif /* !UNIV_HOTBACKUP */ +#include "lock0types.h" +#include "que0types.h" +#include "sync0rw.h" #include "ut0mem.h" #include "ut0rnd.h" #include "ut0byte.h" @@ -50,7 +48,6 @@ Created 1/8/1996 Heikki Tuuri #include "gis0type.h" #include "os0once.h" #include "ut0new.h" - #include "fil0fil.h" #include #include "fil0crypt.h" @@ -869,7 +866,6 @@ struct dict_index_t{ id_name_t name; /*!< index name */ const char* table_name;/*!< table name */ dict_table_t* table; /*!< back pointer to table */ -#ifndef UNIV_HOTBACKUP unsigned space:32; /*!< space where the index tree is placed */ unsigned page:32;/*!< index tree root page number */ @@ -878,7 +874,6 @@ struct dict_index_t{ data size drops below this limit in percent, merging it to a neighbor is tried */ # define DICT_INDEX_MERGE_THRESHOLD_DEFAULT 50 -#endif /* !UNIV_HOTBACKUP */ unsigned type:DICT_IT_BITS; /*!< index type (DICT_CLUSTERED, DICT_UNIQUE, DICT_IBUF, DICT_CORRUPT) */ @@ -940,7 +935,6 @@ struct dict_index_t{ bool has_new_v_col; /*!< whether it has a newly added virtual column in ALTER */ -#ifndef UNIV_HOTBACKUP UT_LIST_NODE_T(dict_index_t) indexes;/*!< list of indexes of the table */ btr_search_t* search_info; @@ -1028,7 +1022,6 @@ struct dict_index_t{ ut_ad(committed || !(type & DICT_CLUSTERED)); uncommitted = !committed; } -#endif /* !UNIV_HOTBACKUP */ }; /** The status of online index creation */ @@ -1514,7 +1507,6 @@ struct dict_table_t { /*!< !DICT_FRM_CONSISTENT==0 if data dictionary information and MySQL FRM information mismatch. */ -#ifndef UNIV_HOTBACKUP /** Hash chain node. */ hash_node_t name_hash; @@ -1775,8 +1767,6 @@ struct dict_table_t { /** Timestamp of the last modification of this table. */ time_t update_time; -#endif /* !UNIV_HOTBACKUP */ - bool is_encrypted; #ifdef UNIV_DEBUG diff --git a/storage/innobase/include/dict0mem.ic b/storage/innobase/include/dict0mem.ic index a50fb615a093d..933f233aae6be 100644 --- a/storage/innobase/include/dict0mem.ic +++ b/storage/innobase/include/dict0mem.ic @@ -60,11 +60,9 @@ dict_mem_fill_index_struct( /* Assign a ulint to a 4-bit-mapped field. Only the low-order 4 bits are assigned. */ index->type = type; -#ifndef UNIV_HOTBACKUP index->space = (unsigned int) space; index->page = FIL_NULL; index->merge_threshold = DICT_INDEX_MERGE_THRESHOLD_DEFAULT; -#endif /* !UNIV_HOTBACKUP */ index->table_name = table_name; index->n_fields = (unsigned int) n_fields; /* The '1 +' above prevents allocation diff --git a/storage/innobase/include/dict0priv.ic b/storage/innobase/include/dict0priv.ic index fd10c566be6fe..fb7af2772fcb8 100644 --- a/storage/innobase/include/dict0priv.ic +++ b/storage/innobase/include/dict0priv.ic @@ -26,7 +26,6 @@ Created Wed 13 Oct 2010 16:10:14 EST Sunny Bains #include "dict0dict.h" #include "dict0load.h" #include "dict0priv.h" -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Gets a table; loads it to the dictionary cache if necessary. A low-level @@ -125,4 +124,3 @@ dict_table_check_if_in_cache_low( !strcmp(table->name.m_name, table_name)); DBUG_RETURN(table); } -#endif /*! UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 918a849be3d2c..1e07c4f8fb534 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -33,64 +33,11 @@ Created 10/25/1995 Heikki Tuuri #include "log0recv.h" #include "dict0types.h" #include "page0size.h" -#ifndef UNIV_HOTBACKUP #include "ibuf0types.h" -#else -#include "log0log.h" -#include "os0file.h" -#include "m_string.h" -#endif /* !UNIV_HOTBACKUP */ #include #include -#ifdef UNIV_HOTBACKUP -#include -/** determine if file is intermediate / temporary.These files are created during -reorganize partition, rename tables, add / drop columns etc. -@param[in] filepath asbosolute / relative or simply file name -@retvalue true if it is intermediate file -@retvalue false if it is normal file */ -inline -bool -is_intermediate_file(const std::string& filepath) -{ - std::string file_name = filepath; - - // extract file name from relative or absolute file name - std::size_t pos = file_name.rfind(OS_PATH_SEPARATOR); - if (pos != std::string::npos) - file_name = file_name.substr(++pos); - - transform(file_name.begin(), file_name.end(), - file_name.begin(), ::tolower); - - if (file_name[0] != '#') { - pos = file_name.rfind("#tmp#.ibd"); - if (pos != std::string::npos) - return true; - else - return false; /* normal file name */ - } - - std::vector file_name_patterns = {"#sql-", "#sql2-", - "#tmp#", "#ren#"}; - - /* search for the unsupported patterns */ - for (auto itr = file_name_patterns.begin(); - itr != file_name_patterns.end(); - itr++) { - - if (0 == std::strncmp(file_name.c_str(), - itr->c_str(), itr->length())){ - return true; - } - } - - return false; -} -#endif /* UNIV_HOTBACKUP */ - extern const char general_space_name[]; // Forward declaration @@ -205,10 +152,8 @@ struct fil_space_t { Protected by fil_system->mutex. */ hash_node_t hash; /*!< hash chain node */ hash_node_t name_hash;/*!< hash chain the name_hash table */ -#ifndef UNIV_HOTBACKUP rw_lock_t latch; /*!< latch protecting the file space storage allocation */ -#endif /* !UNIV_HOTBACKUP */ UT_LIST_NODE_T(fil_space_t) unflushed_spaces; /*!< list of spaces with at least one unflushed file we have written to */ @@ -619,9 +564,7 @@ fil_space_get( data space) is stored here; below we talk about tablespaces, but also the ib_logfiles form a 'space' and it is handled here */ struct fil_system_t { -#ifndef UNIV_HOTBACKUP ib_mutex_t mutex; /*!< The mutex protecting the cache */ -#endif /* !UNIV_HOTBACKUP */ hash_table_t* spaces; /*!< The hash table of spaces in the system; they are hashed on the space id */ @@ -683,7 +626,6 @@ extern fil_system_t* fil_system; #include "fil0crypt.h" -#ifndef UNIV_HOTBACKUP /** Returns the latch of a file space. @param[in] id space id @param[out] flags tablespace flags @@ -710,15 +652,14 @@ void fil_space_set_imported( ulint id); -# ifdef UNIV_DEBUG +#ifdef UNIV_DEBUG /** Determine if a tablespace is temporary. @param[in] id tablespace identifier @return whether it is a temporary tablespace */ bool fsp_is_temporary(ulint id) MY_ATTRIBUTE((warn_unused_result, pure)); -# endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ +#endif /* UNIV_DEBUG */ /** Append a file to the chain of files of a space. @param[in] name file name of a file that is not open @@ -884,7 +825,6 @@ void fil_set_max_space_id_if_bigger( /*===========================*/ ulint max_id);/*!< in: maximum known id */ -#ifndef UNIV_HOTBACKUP /** Write the flushed LSN to the page header of the first page in the system tablespace. @@ -976,8 +916,6 @@ class FilSpace fil_space_t* m_space; }; -#endif /* !UNIV_HOTBACKUP */ - /********************************************************//** Creates the database directory for a table if it does not exist yet. */ void @@ -1077,7 +1015,7 @@ fil_close_tablespace( /*=================*/ trx_t* trx, /*!< in/out: Transaction covering the close */ ulint id); /*!< in: space id */ -#ifndef UNIV_HOTBACKUP + /*******************************************************************//** Discards a single-table tablespace. The tablespace must be cached in the memory cache. Discarding is like deleting a tablespace, but @@ -1097,7 +1035,6 @@ fil_discard_tablespace( /*===================*/ ulint id) /*!< in: space id */ MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /** Test if a tablespace file can be renamed to a new filepath by checking if that the old filepath exists and the new filepath does not exist. @@ -1243,7 +1180,6 @@ fil_file_readdir_next_file( os_file_dir_t dir, /*!< in: directory stream */ os_file_stat_t* info); /*!< in/out: buffer where the info is returned */ -#ifndef UNIV_HOTBACKUP /*******************************************************************//** Returns true if a matching tablespace exists in the InnoDB tablespace memory cache. Note that if we have not done a crash recovery at the database startup, @@ -1265,7 +1201,7 @@ fil_space_for_table_exists_in_mem( mem_heap_t* heap, /*!< in: heap memory */ table_id_t table_id, /*!< in: table id */ dict_table_t* table); /*!< in: table or NULL */ -#endif /* !UNIV_HOTBACKUP */ + /** Try to extend a tablespace if it is smaller than the specified size. @param[in,out] space tablespace @param[in] size desired size in pages diff --git a/storage/innobase/include/fut0lst.h b/storage/innobase/include/fut0lst.h index 9c980d1358d75..e29973c1696f1 100644 --- a/storage/innobase/include/fut0lst.h +++ b/storage/innobase/include/fut0lst.h @@ -48,7 +48,7 @@ typedef byte flst_node_t; /* The physical size of a list node in bytes */ #define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE) -#if !defined UNIV_HOTBACKUP && !defined UNIV_INNOCHECKSUM +#ifndef UNIV_INNOCHECKSUM /********************************************************************//** Initializes a list base node. */ UNIV_INLINE @@ -204,6 +204,6 @@ flst_print( #include "fut0lst.ic" #endif -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM*/ +#endif /* !UNIV_INNOCHECKSUM */ #endif diff --git a/storage/innobase/include/ha0ha.h b/storage/innobase/include/ha0ha.h index 15a99ddf683d8..6857670a01bc8 100644 --- a/storage/innobase/include/ha0ha.h +++ b/storage/innobase/include/ha0ha.h @@ -174,7 +174,7 @@ ha_search_and_delete_if_found( hash_table_t* table, /*!< in: hash table */ ulint fold, /*!< in: folded value of the searched data */ const rec_t* data); /*!< in: pointer to the data */ -#ifndef UNIV_HOTBACKUP + /*****************************************************************//** Removes from the chain determined by fold all nodes whose data pointer points to the page given. */ @@ -202,7 +202,6 @@ ha_print_info( /*==========*/ FILE* file, /*!< in: file where to print */ hash_table_t* table); /*!< in: hash table */ -#endif /* !UNIV_HOTBACKUP */ /** The hash table external chain node */ struct ha_node_t { diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h index 116ca781726f7..c10644be8329e 100644 --- a/storage/innobase/include/ha_prototypes.h +++ b/storage/innobase/include/ha_prototypes.h @@ -31,7 +31,7 @@ simple headers. #include "univ.i" -#if !defined UNIV_HOTBACKUP && !defined UNIV_INNOCHECKSUM +#ifndef UNIV_INNOCHECKSUM /* Forward declarations */ class THD; @@ -668,5 +668,5 @@ innobase_destroy_background_thd(MYSQL_THD); void innobase_reset_background_thd(MYSQL_THD); -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ #endif /* HA_INNODB_PROTOTYPES_H */ diff --git a/storage/innobase/include/hash0hash.h b/storage/innobase/include/hash0hash.h index a7bcee1185bb8..2922d424d3720 100644 --- a/storage/innobase/include/hash0hash.h +++ b/storage/innobase/include/hash0hash.h @@ -28,9 +28,7 @@ Created 5/20/1997 Heikki Tuuri #include "univ.i" #include "mem0mem.h" -#ifndef UNIV_HOTBACKUP -# include "sync0rw.h" -#endif /* !UNIV_HOTBACKUP */ +#include "sync0rw.h" struct hash_table_t; struct hash_cell_t; @@ -60,7 +58,7 @@ hash_table_t* hash_create( /*========*/ ulint n); /*!< in: number of array cells */ -#ifndef UNIV_HOTBACKUP + /*************************************************************//** Creates a sync object array array to protect a hash table. ::sync_obj can be mutexes or rw_locks depening on the type of @@ -74,7 +72,6 @@ hash_create_sync_obj( latch_id_t id, /*!< in: mutex/rw_lock ID */ ulint n_sync_obj);/*!< in: number of sync objects, must be a power of 2 */ -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Frees a hash table. */ @@ -91,15 +88,11 @@ hash_calc_hash( /*===========*/ ulint fold, /*!< in: folded value */ hash_table_t* table); /*!< in: hash table */ -#ifndef UNIV_HOTBACKUP /********************************************************************//** Assert that the mutex for the table is held */ -# define HASH_ASSERT_OWN(TABLE, FOLD) \ +#define HASH_ASSERT_OWN(TABLE, FOLD) \ ut_ad((TABLE)->type != HASH_TABLE_SYNC_MUTEX \ || (mutex_own(hash_get_mutex((TABLE), FOLD)))); -#else /* !UNIV_HOTBACKUP */ -# define HASH_ASSERT_OWN(TABLE, FOLD) -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Inserts a struct to a hash table. */ @@ -337,7 +330,6 @@ do {\ mem_heap_free_top(hash_get_heap(TABLE, fold111), sizeof(TYPE));\ } while (0) -#ifndef UNIV_HOTBACKUP /****************************************************************//** Move all hash table entries from OLD_TABLE to NEW_TABLE. */ @@ -537,22 +529,6 @@ hash_unlock_x_all_but( hash_table_t* table, /*!< in: hash table */ rw_lock_t* keep_lock); /*!< in: lock to keep */ -#else /* !UNIV_HOTBACKUP */ -# define hash_get_heap(table, fold) ((table)->heap) -# define hash_mutex_enter(table, fold) ((void) 0) -# define hash_mutex_exit(table, fold) ((void) 0) -# define hash_mutex_enter_all(table) ((void) 0) -# define hash_mutex_exit_all(table) ((void) 0) -# define hash_mutex_exit_all_but(t, m) ((void) 0) -# define hash_lock_s(t, f) ((void) 0) -# define hash_lock_x(t, f) ((void) 0) -# define hash_unlock_s(t, f) ((void) 0) -# define hash_unlock_x(t, f) ((void) 0) -# define hash_lock_x_all(t) ((void) 0) -# define hash_unlock_x_all(t) ((void) 0) -# define hash_unlock_x_all_but(t, l) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ - struct hash_cell_t{ void* node; /*!< hash chain node, NULL if none */ }; @@ -561,15 +537,13 @@ struct hash_cell_t{ struct hash_table_t { enum hash_table_sync_t type; /*n_cells)); } -#ifndef UNIV_HOTBACKUP /************************************************************//** Gets the sync object index for a fold value in a hash table. @return index */ @@ -276,4 +275,3 @@ hash_lock_x_confirm( return(hash_lock); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/ibuf0ibuf.h b/storage/innobase/include/ibuf0ibuf.h index f3fd5e9a3646a..e646f6789649f 100644 --- a/storage/innobase/include/ibuf0ibuf.h +++ b/storage/innobase/include/ibuf0ibuf.h @@ -32,9 +32,7 @@ Created 7/19/1997 Heikki Tuuri #include "mtr0mtr.h" #include "dict0mem.h" #include "fsp0fsp.h" - -#ifndef UNIV_HOTBACKUP -# include "ibuf0types.h" +#include "ibuf0types.h" /** Default value for maximum on-disk size of change buffer in terms of percentage of the buffer pool. */ @@ -377,7 +375,6 @@ ibuf_merge_space( /*=============*/ ulint space); /*!< in: space id */ -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Parses a redo log record of an ibuf bitmap page init. @return end of log record or NULL */ @@ -388,9 +385,8 @@ ibuf_parse_bitmap_init( byte* end_ptr,/*!< in: buffer end */ buf_block_t* block, /*!< in: block or NULL */ mtr_t* mtr); /*!< in: mtr or NULL */ -#ifndef UNIV_HOTBACKUP -#ifdef UNIV_IBUF_COUNT_DEBUG +#ifdef UNIV_IBUF_COUNT_DEBUG /** Gets the ibuf count for a given page. @param[in] page_id page id @return number of entries in the insert buffer currently buffered for @@ -398,7 +394,6 @@ this page */ ulint ibuf_count_get( const page_id_t& page_id); - #endif /******************************************************************//** Looks if the insert buffer is empty. @@ -447,8 +442,6 @@ ibuf_set_bitmap_for_bulk_load( #define IBUF_HEADER_PAGE_NO FSP_IBUF_HEADER_PAGE_NO #define IBUF_TREE_ROOT_PAGE_NO FSP_IBUF_TREE_ROOT_PAGE_NO -#endif /* !UNIV_HOTBACKUP */ - /* The ibuf header page currently contains only the file segment header for the file segment from which the pages for the ibuf tree are allocated */ #define IBUF_HEADER PAGE_DATA diff --git a/storage/innobase/include/ibuf0ibuf.ic b/storage/innobase/include/ibuf0ibuf.ic index de39592ae6b0a..09070c140591d 100644 --- a/storage/innobase/include/ibuf0ibuf.ic +++ b/storage/innobase/include/ibuf0ibuf.ic @@ -26,7 +26,6 @@ Created 7/19/1997 Heikki Tuuri #include "page0page.h" #include "page0zip.h" #include "fsp0types.h" -#ifndef UNIV_HOTBACKUP #include "buf0lru.h" /** An index page must contain at least UNIV_PAGE_SIZE / @@ -340,4 +339,3 @@ ibuf_update_free_bits_if_full( ibuf_set_free_bits(block, after, before); } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index caa067cd4ba0d..68dc8faf51961 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -35,9 +35,7 @@ Created 12/9/1995 Heikki Tuuri #include "univ.i" #include "dyn0buf.h" -#ifndef UNIV_HOTBACKUP #include "sync0rw.h" -#endif /* !UNIV_HOTBACKUP */ #include "log0crypt.h" #include "log0types.h" @@ -72,7 +70,6 @@ log_calc_where_lsn_is( files */ int64_t log_file_size); /*!< in: log file size (including the header) */ -#ifndef UNIV_HOTBACKUP /** Append a string to the log. @param[in] str string @param[in] len string length @@ -268,19 +265,6 @@ log_write_checkpoint_info( mtr_buf_t* log_append_on_checkpoint( mtr_buf_t* buf); -#else /* !UNIV_HOTBACKUP */ -/******************************************************//** -Writes info to a buffer of a log group when log files are created in -backup restoration. */ -void -log_reset_first_header_and_checkpoint( -/*==================================*/ - byte* hdr_buf,/*!< in: buffer which will be written to the - start of the first log file */ - ib_uint64_t start); /*!< in: lsn of the start of the first log file; - we pretend that there is a checkpoint at - start + LOG_BLOCK_HDR_SIZE */ -#endif /* !UNIV_HOTBACKUP */ /** Checks that there is enough free space in the log to start a new query step. Flushes the log buffer or makes a new checkpoint if necessary. NOTE: this @@ -288,7 +272,7 @@ function may only be called if the calling thread owns no synchronization objects! */ void log_check_margins(void); -#ifndef UNIV_HOTBACKUP + /******************************************************//** Reads a specified log segment to a buffer. */ void @@ -316,7 +300,6 @@ lsn_t log_group_get_capacity( /*===================*/ const log_group_t* group); /*!< in: log group */ -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Gets a log block flush bit. @return TRUE if this block was the first to be written in a log flush */ @@ -422,17 +405,6 @@ log_block_init( /*===========*/ byte* log_block, /*!< in: pointer to the log buffer */ lsn_t lsn); /*!< in: lsn within the log block */ -#ifdef UNIV_HOTBACKUP -/************************************************************//** -Initializes a log block in the log buffer in the old, < 3.23.52 format, where -there was no checksum yet. */ -UNIV_INLINE -void -log_block_init_in_old_format( -/*=========================*/ - byte* log_block, /*!< in: pointer to the log buffer */ - lsn_t lsn); /*!< in: lsn within the log block */ -#endif /* UNIV_HOTBACKUP */ /************************************************************//** Converts a lsn to a log block number. @return log block number, it is > 0 and <= 1G */ @@ -648,7 +620,7 @@ struct log_t{ lsn_t lsn; /*!< log sequence number */ ulint buf_free; /*!< first free offset within the log buffer in use */ -#ifndef UNIV_HOTBACKUP + char pad2[CACHE_LINE_SIZE];/*!< Padding */ LogSysMutex mutex; /*!< mutex protecting the log */ char pad3[CACHE_LINE_SIZE]; /*!< Padding */ @@ -663,7 +635,6 @@ struct log_t{ mtr_commit and still ensure that insertions in the flush_list happen in the LSN order. */ -#endif /* !UNIV_HOTBACKUP */ byte* buf_ptr; /*!< unaligned log buffer, which should be of double of buf_size */ byte* buf; /*!< log buffer currently in use; @@ -692,7 +663,6 @@ struct log_t{ UT_LIST_BASE_NODE_T(log_group_t) log_groups; /*!< log groups */ -#ifndef UNIV_HOTBACKUP /** The fields involved in the log buffer flush @{ */ ulint buf_next_to_write;/*!< first offset in the log buffer @@ -776,7 +746,6 @@ struct log_t{ checkpoint write is running; a thread should wait for this without owning the log mutex */ -#endif /* !UNIV_HOTBACKUP */ byte* checkpoint_buf_ptr;/* unaligned checkpoint header */ byte* checkpoint_buf; /*!< checkpoint header is read to this buffer */ diff --git a/storage/innobase/include/log0log.ic b/storage/innobase/include/log0log.ic index a53f8770cea84..82a9426577644 100644 --- a/storage/innobase/include/log0log.ic +++ b/storage/innobase/include/log0log.ic @@ -297,32 +297,6 @@ log_block_init( log_block_set_first_rec_group(log_block, 0); } -#ifdef UNIV_HOTBACKUP -/************************************************************//** -Initializes a log block in the log buffer in the old format, where there -was no checksum yet. */ -UNIV_INLINE -void -log_block_init_in_old_format( -/*=========================*/ - byte* log_block, /*!< in: pointer to the log buffer */ - lsn_t lsn) /*!< in: lsn within the log block */ -{ - ulint no; - - ut_ad(log_mutex_own()); - - no = log_block_convert_lsn_to_no(lsn); - - log_block_set_hdr_no(log_block, no); - mach_write_to_4(log_block + OS_FILE_LOG_BLOCK_SIZE - - LOG_BLOCK_CHECKSUM, no); - log_block_set_data_len(log_block, LOG_BLOCK_HDR_SIZE); - log_block_set_first_rec_group(log_block, 0); -} -#endif /* UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP /** Append a string to the log. @param[in] str string @param[in] len string length @@ -529,4 +503,3 @@ log_free_check(void) log_check_margins(); } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/log0recv.h b/storage/innobase/include/log0recv.h index bd7118654f3a7..a9d98b08d85e1 100644 --- a/storage/innobase/include/log0recv.h +++ b/storage/innobase/include/log0recv.h @@ -37,83 +37,16 @@ Created 9/20/1997 Heikki Tuuri #include #include -#ifdef UNIV_HOTBACKUP -extern bool recv_replay_file_ops; +/** @return whether recovery is currently running. */ +#define recv_recovery_is_on() recv_recovery_on -/*******************************************************************//** -Reads the checkpoint info needed in hot backup. -@return TRUE if success */ -ibool -recv_read_checkpoint_info_for_backup( -/*=================================*/ - const byte* hdr, /*!< in: buffer containing the log group - header */ - lsn_t* lsn, /*!< out: checkpoint lsn */ - lsn_t* offset, /*!< out: checkpoint offset in the log group */ - lsn_t* cp_no, /*!< out: checkpoint number */ - lsn_t* first_header_lsn) - /*!< out: lsn of of the start of the - first log file */ - MY_ATTRIBUTE((nonnull)); -/*******************************************************************//** -Scans the log segment and n_bytes_scanned is set to the length of valid -log scanned. */ -void -recv_scan_log_seg_for_backup( -/*=========================*/ - byte* buf, /*!< in: buffer containing log data */ - ulint buf_len, /*!< in: data length in that buffer */ - lsn_t* scanned_lsn, /*!< in/out: lsn of buffer start, - we return scanned lsn */ - ulint* scanned_checkpoint_no, - /*!< in/out: 4 lowest bytes of the - highest scanned checkpoint number so - far */ - ulint* n_bytes_scanned);/*!< out: how much we were able to - scan, smaller than buf_len if log - data ended here */ -#endif /* UNIV_HOTBACKUP */ -/*******************************************************************//** -Returns TRUE if recovery is currently running. -@return recv_recovery_on */ -UNIV_INLINE -bool -recv_recovery_is_on(void); -/*=====================*/ -/************************************************************************//** -Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. */ +/** Apply the hashed log records to the page, if the page lsn is less than the +lsn of a log record. +@param just_read_in whether the page recently arrived to the I/O handler +@param block the page in the buffer pool */ void -recv_recover_page_func( -/*===================*/ -#ifndef UNIV_HOTBACKUP - ibool just_read_in, - /*!< in: TRUE if the i/o handler calls - this for a freshly read page */ -#endif /* !UNIV_HOTBACKUP */ - buf_block_t* block); /*!< in/out: buffer block */ -#ifndef UNIV_HOTBACKUP -/** Wrapper for recv_recover_page_func(). -Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. -@param jri in: TRUE if just read in (the i/o handler calls this for -a freshly read page) -@param block in/out: the buffer block -*/ -# define recv_recover_page(jri, block) recv_recover_page_func(jri, block) -#else /* !UNIV_HOTBACKUP */ -/** Wrapper for recv_recover_page_func(). -Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. -@param jri in: TRUE if just read in (the i/o handler calls this for -a freshly read page) -@param block in/out: the buffer block -*/ -# define recv_recover_page(jri, block) recv_recover_page_func(block) -#endif /* !UNIV_HOTBACKUP */ +recv_recover_page(bool just_read_in, buf_block_t* block); + /** Start recovering from a redo log checkpoint. @see recv_recovery_from_checkpoint_finish @param[in] flush_lsn FIL_PAGE_FILE_FLUSH_LSN @@ -140,18 +73,6 @@ recv_reset_logs( OS_FILE_LOG_BLOCK_SIZE, after which we add LOG_BLOCK_HDR_SIZE */ -#ifdef UNIV_HOTBACKUP -/******************************************************//** -Creates new log files after a backup has been restored. */ -void -recv_reset_log_files_for_backup( -/*============================*/ - const char* log_dir, /*!< in: log file directory path */ - ulint n_log_files, /*!< in: number of log files */ - lsn_t log_file_size, /*!< in: log file size */ - lsn_t lsn); /*!< in: new start lsn, must be - divisible by OS_FILE_LOG_BLOCK_SIZE */ -#endif /* UNIV_HOTBACKUP */ /********************************************************//** Creates the recovery system. */ void @@ -173,7 +94,6 @@ void recv_sys_init( /*==========*/ ulint available_memory); /*!< in: available memory in bytes */ -#ifndef UNIV_HOTBACKUP /********************************************************//** Frees the recovery system. */ void @@ -184,7 +104,6 @@ Reset the state of the recovery system variables. */ void recv_sys_var_init(void); /*===================*/ -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Empties the hash table of stored log records, applying them to appropriate pages. */ @@ -200,14 +119,6 @@ recv_apply_hashed_log_recs( can be generated during the application */ __attribute__((warn_unused_result)); -#ifdef UNIV_HOTBACKUP -/*******************************************************************//** -Applies log records in the hash table to a backup. */ -void -recv_apply_log_recs_for_backup(void); -/*================================*/ -#endif /* UNIV_HOTBACKUP */ - /** Block of log record data */ struct recv_data_t{ recv_data_t* next; /*!< pointer to the next block or NULL */ @@ -291,7 +202,6 @@ typedef std::vector > /** Recovery system data structure */ struct recv_sys_t{ -#ifndef UNIV_HOTBACKUP ib_mutex_t mutex; /*!< mutex protecting the fields apply_log_recs, n_addrs, and the state field in each recv_addr struct */ @@ -305,7 +215,6 @@ struct recv_sys_t{ buf_flush_t flush_type;/*!< type of the flush request. BUF_FLUSH_LRU: flush end of LRU, keeping free blocks. BUF_FLUSH_LIST: flush all of blocks. */ -#endif /* !UNIV_HOTBACKUP */ ibool apply_log_recs; /*!< this is TRUE when log rec application to pages is allowed; this flag tells the @@ -391,15 +300,9 @@ extern bool recv_no_log_write; number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by recv_recovery_from_checkpoint_start(). */ extern bool recv_lsn_checks_on; -#ifdef UNIV_HOTBACKUP -/** TRUE when the redo log is being backed up */ -extern bool recv_is_making_a_backup; -#endif /* UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP /** Flag indicating if recv_writer thread is active. */ extern volatile bool recv_writer_thread_active; -#endif /* !UNIV_HOTBACKUP */ /** Size of the parsing buffer; it must accommodate RECV_SCAN_SIZE many times! */ @@ -424,8 +327,4 @@ log_block_checksum_is_ok( const byte* block, /*!< in: pointer to a log block */ bool print_err); /*!< in print error ? */ -#ifndef UNIV_NONINL -#include "log0recv.ic" -#endif - #endif diff --git a/storage/innobase/include/log0recv.ic b/storage/innobase/include/log0recv.ic deleted file mode 100644 index d197e5e333727..0000000000000 --- a/storage/innobase/include/log0recv.ic +++ /dev/null @@ -1,38 +0,0 @@ -/***************************************************************************** - -Copyright (c) 1997, 2013, Oracle and/or its affiliates. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file include/log0recv.ic -Recovery - -Created 9/20/1997 Heikki Tuuri -*******************************************************/ - -#include "univ.i" - -/*******************************************************************//** -Returns TRUE if recovery is currently running. -@return recv_recovery_on */ -UNIV_INLINE -bool -recv_recovery_is_on() -/*=================*/ -{ - return(recv_recovery_on); -} - diff --git a/storage/innobase/include/mach0data.h b/storage/innobase/include/mach0data.h index 4d32e2e717028..de05de55d7742 100644 --- a/storage/innobase/include/mach0data.h +++ b/storage/innobase/include/mach0data.h @@ -271,7 +271,7 @@ ib_uint64_t mach_u64_parse_compressed( const byte** ptr, const byte* end_ptr); -#ifndef UNIV_HOTBACKUP + /*********************************************************//** Reads a double. It is stored in a little-endian format. @return double read */ @@ -388,7 +388,6 @@ mach_read_ulint( mlog_id_t type) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_INNOCHECKSUM */ #ifndef UNIV_NONINL diff --git a/storage/innobase/include/mach0data.ic b/storage/innobase/include/mach0data.ic index 20018844e4b69..31520c47825ba 100644 --- a/storage/innobase/include/mach0data.ic +++ b/storage/innobase/include/mach0data.ic @@ -609,7 +609,7 @@ mach_u64_parse_compressed( return(val); } -#ifndef UNIV_HOTBACKUP + /*********************************************************//** Reads a double. It is stored in a little-endian format. @return double read */ @@ -947,5 +947,4 @@ mach_read_ulint( return(0); } -#endif /* !UNIV_HOTBACKUP */ #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/mem0mem.h b/storage/innobase/include/mem0mem.h index f8fdb53e13260..991d117909803 100644 --- a/storage/innobase/include/mem0mem.h +++ b/storage/innobase/include/mem0mem.h @@ -390,7 +390,7 @@ struct mem_block_info_t { user data in the block */ ulint start; /*!< the value of the struct field 'free' at the creation of the block */ -#ifndef UNIV_HOTBACKUP + void* free_block; /* if the MEM_HEAP_BTR_SEARCH bit is set in type, and this is the heap root, this can contain an @@ -401,7 +401,6 @@ struct mem_block_info_t { /* if this block has been allocated from the buffer pool, this contains the buf_block_t handle; otherwise, this is NULL */ -#endif /* !UNIV_HOTBACKUP */ }; #define MEM_BLOCK_MAGIC_N 764741555 diff --git a/storage/innobase/include/mem0mem.ic b/storage/innobase/include/mem0mem.ic index 3b4109ee52d00..82c90584e6f51 100644 --- a/storage/innobase/include/mem0mem.ic +++ b/storage/innobase/include/mem0mem.ic @@ -61,14 +61,12 @@ mem_heap_block_free( mem_heap_t* heap, /*!< in: heap */ mem_block_t* block); /*!< in: block to free */ -#ifndef UNIV_HOTBACKUP /******************************************************************//** Frees the free_block field from a memory heap. */ void mem_heap_free_block_free( /*=====================*/ mem_heap_t* heap); /*!< in: heap */ -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** Adds a new block to a memory heap. @@ -302,11 +300,10 @@ mem_heap_empty( mem_heap_t* heap) { mem_heap_free_heap_top(heap, (byte*) heap + mem_block_get_start(heap)); -#ifndef UNIV_HOTBACKUP + if (heap->free_block) { mem_heap_free_block_free(heap); } -#endif /* !UNIV_HOTBACKUP */ } /** Returns a pointer to the topmost element in a memory heap. @@ -524,11 +521,9 @@ mem_heap_free( block = UT_LIST_GET_LAST(heap->base); -#ifndef UNIV_HOTBACKUP if (heap->free_block) { mem_heap_free_block_free(heap); } -#endif /* !UNIV_HOTBACKUP */ while (block != NULL) { /* Store the contents of info before freeing current block @@ -556,11 +551,9 @@ mem_heap_get_size( size = heap->total_size; -#ifndef UNIV_HOTBACKUP if (heap->free_block) { size += UNIV_PAGE_SIZE; } -#endif /* !UNIV_HOTBACKUP */ return(size); } diff --git a/storage/innobase/include/mtr0log.h b/storage/innobase/include/mtr0log.h index 3819d3f694ae4..b5c94ff7d4615 100644 --- a/storage/innobase/include/mtr0log.h +++ b/storage/innobase/include/mtr0log.h @@ -33,7 +33,6 @@ Created 12/7/1995 Heikki Tuuri // Forward declaration struct dict_index_t; -#ifndef UNIV_HOTBACKUP /********************************************************//** Writes 1, 2 or 4 bytes to a file page. Writes the corresponding log record to the mini-transaction log if mtr is not NULL. */ @@ -178,10 +177,6 @@ mlog_write_initial_log_record_fast( byte* log_ptr,/*!< in: pointer to mtr log which has been opened */ mtr_t* mtr); /*!< in: mtr */ -#else /* !UNIV_HOTBACKUP */ -# define mlog_write_initial_log_record(ptr,type,mtr) ((void) 0) -# define mlog_write_initial_log_record_fast(ptr,type,log_ptr,mtr) ((byte*) 0) -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses an initial log record written by mlog_write_initial_log_record. @return parsed record end, NULL if not a complete record */ @@ -216,7 +211,6 @@ mlog_parse_string( byte* page, /*!< in: page where to apply the log record, or NULL */ void* page_zip);/*!< in/out: compressed page, or NULL */ -#ifndef UNIV_HOTBACKUP /********************************************************//** Opens a buffer for mlog, writes the initial log record and, if needed, the field lengths of an index. Reserves space @@ -233,7 +227,6 @@ mlog_open_and_write_index( ulint size); /*!< in: requested buffer size in bytes (if 0, calls mlog_close() and returns NULL) */ -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses a log record written by mlog_open_and_write_index. @@ -246,11 +239,9 @@ mlog_parse_index( ibool comp, /*!< in: TRUE=compact record format */ dict_index_t** index); /*!< out, own: dummy index */ -#ifndef UNIV_HOTBACKUP /** Insert, update, and maybe other functions may use this value to define an extra mlog buffer size for variable size data */ #define MLOG_BUF_MARGIN 256 -#endif /* !UNIV_HOTBACKUP */ #ifndef UNIV_NONINL #include "mtr0log.ic" diff --git a/storage/innobase/include/mtr0log.ic b/storage/innobase/include/mtr0log.ic index 4015fe36d1929..701f1482e6407 100644 --- a/storage/innobase/include/mtr0log.ic +++ b/storage/innobase/include/mtr0log.ic @@ -65,7 +65,6 @@ mlog_close( mtr->get_log()->close(ptr); } -#ifndef UNIV_HOTBACKUP /********************************************************//** Catenates 1 - 4 bytes to the mtr log. The value is not compressed. */ UNIV_INLINE @@ -247,4 +246,3 @@ mlog_write_initial_log_record_fast( return(mlog_write_initial_log_record_low(type, space, offset, log_ptr, mtr)); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index 5e36cfc2ac0b6..f65d5f8dc1b6a 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -918,7 +918,6 @@ struct os_file_stat_t { if type == OS_FILE_TYPE_FILE */ }; -#ifndef UNIV_HOTBACKUP /** Create a temporary file. This function is like tmpfile(3), but the temporary file is created in the given parameter path. If the path is null then it will create the file in the mysql server configuration @@ -928,7 +927,6 @@ parameter (--tmpdir). FILE* os_file_create_tmpfile( const char* path); -#endif /* !UNIV_HOTBACKUP */ /** The os_file_opendir() function opens a directory stream corresponding to the directory named by the dirname argument. The directory stream is positioned @@ -1569,14 +1567,6 @@ to original un-instrumented file I/O APIs */ #endif /* UNIV_PFS_IO */ -#ifdef UNIV_HOTBACKUP -/** Closes a file handle. -@param[in] file handle to a file -@return true if success */ -bool -os_file_close_no_error_handling(os_file_t file); -#endif /* UNIV_HOTBACKUP */ - /** Gets a file size. @param[in] file handle to a file @return file size if OK, else set m_total_size to ~0 and m_alloc_size @@ -1914,7 +1904,6 @@ os_file_get_status( bool check_rw_perm, bool read_only); -#if !defined(UNIV_HOTBACKUP) /** Creates a temporary file in the location specified by the parameter path. If the path is NULL then it will be created on --tmpdir location. This function is defined in ha_innodb.cc. @@ -1923,8 +1912,6 @@ This function is defined in ha_innodb.cc. int innobase_mysql_tmpfile( const char* path); -#endif /* !UNIV_HOTBACKUP */ - /** If it is a compressed page return the compressed page data + footer size @param[in] buf Buffer to check, must include header + 10 bytes diff --git a/storage/innobase/include/page0cur.h b/storage/innobase/include/page0cur.h index 94b5896d3ad8d..2bb922cc829f0 100644 --- a/storage/innobase/include/page0cur.h +++ b/storage/innobase/include/page0cur.h @@ -135,7 +135,7 @@ void page_cur_move_to_prev( /*==================*/ page_cur_t* cur); /*!< in/out: cursor; not before first */ -#ifndef UNIV_HOTBACKUP + /***********************************************************//** Inserts a record next to page cursor. Returns pointer to inserted record if succeed, i.e., enough space available, NULL otherwise. The cursor stays at @@ -163,7 +163,6 @@ page_cur_tuple_insert( /*!< in: if true, then use record cache to hold the tuple converted record. */ MY_ATTRIBUTE((nonnull(1,2,3,4,5), warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Inserts a record next to page cursor. Returns pointer to inserted record if succeed, i.e., enough space available, NULL otherwise. The cursor stays at @@ -248,7 +247,7 @@ page_cur_delete_rec( const ulint* offsets,/*!< in: rec_get_offsets( cursor->rec, index) */ mtr_t* mtr); /*!< in: mini-transaction handle */ -#ifndef UNIV_HOTBACKUP + /** Search the right position for a page cursor. @param[in] block buffer block @param[in] index index tree @@ -331,7 +330,6 @@ page_cur_open_on_rnd_user_rec( /*==========================*/ buf_block_t* block, /*!< in: page */ page_cur_t* cursor);/*!< out: page cursor */ -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a log record of a record insert on a page. @return end of log record or NULL */ diff --git a/storage/innobase/include/page0cur.ic b/storage/innobase/include/page0cur.ic index bfd9da47803ed..5eb1bc0cbc558 100644 --- a/storage/innobase/include/page0cur.ic +++ b/storage/innobase/include/page0cur.ic @@ -197,7 +197,6 @@ page_cur_move_to_prev( cur->rec = page_rec_get_prev(cur->rec); } -#ifndef UNIV_HOTBACKUP /** Search the right position for a page cursor. @param[in] block buffer block @param[in] index index tree @@ -294,7 +293,6 @@ page_cur_tuple_insert( ut_ad(!rec || !cmp_dtuple_rec(tuple, rec, *offsets)); return(rec); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Inserts a record next to page cursor. Returns pointer to inserted record if diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index b64602fe077e6..e61e7d2ae16df 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -309,7 +309,7 @@ page_header_set_ptr( uncompressed part will be updated, or NULL */ ulint field, /*!< in/out: PAGE_FREE, ... */ const byte* ptr); /*!< in: pointer or NULL*/ -#ifndef UNIV_HOTBACKUP + /*************************************************************//** Resets the last insert info field in the page header. Writes to mlog about this operation. */ @@ -321,7 +321,6 @@ page_header_reset_last_insert( page_zip_des_t* page_zip,/*!< in/out: compressed page whose uncompressed part will be updated, or NULL */ mtr_t* mtr); /*!< in: mtr */ -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Gets the offset of the first record on the page. @return offset of the first record in record list, relative from page */ @@ -362,7 +361,7 @@ page_rec_get_nth( page_t* page, /*< in: page */ ulint nth) /*!< in: nth record */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#ifndef UNIV_HOTBACKUP + /************************************************************//** Returns the middle record of the records on the page. If there is an even number of records in the list, returns the first record of the @@ -374,7 +373,6 @@ page_get_middle_rec( /*================*/ page_t* page) /*!< in: page */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Gets the page number. @return page number */ @@ -756,7 +754,7 @@ rec_t* page_rec_find_owner_rec( /*====================*/ rec_t* rec); /*!< in: the physical record */ -#ifndef UNIV_HOTBACKUP + /***********************************************************************//** Write a 32-bit field in a data dictionary record. */ UNIV_INLINE @@ -768,7 +766,6 @@ page_rec_write_field( ulint val, /*!< in: value to write */ mtr_t* mtr) /*!< in/out: mini-transaction */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Returns the maximum combined size of records which can be inserted on top of record heap. @@ -1069,7 +1066,7 @@ page_parse_create( buf_block_t* block, ulint comp, bool is_rtree); -#ifndef UNIV_HOTBACKUP + /************************************************************//** Prints record contents including the data relevant only in the index page context. */ @@ -1115,7 +1112,6 @@ page_print( ulint rn); /*!< in: print rn first and last records in directory */ # endif /* UNIV_BTR_PRINT */ -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** The following is used to validate a record on a page. This function differs from rec_validate as it can also check the n_owned field and diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index a3bbc09e60f82..97f9d5a578fe7 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -32,9 +32,7 @@ Created 2/2/1994 Heikki Tuuri #ifdef UNIV_DEBUG # include "log0recv.h" #endif /* !UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP -# include "rem0cmp.h" -#endif /* !UNIV_HOTBACKUP */ +#include "rem0cmp.h" #include "mtr0log.h" #include "page0zip.h" @@ -148,20 +146,17 @@ page_set_ssn_id( mtr_t* mtr) /*!< in/out: mini-transaction */ { page_t* page = buf_block_get_frame(block); -#ifndef UNIV_HOTBACKUP + ut_ad(!mtr || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_SX_FIX) || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)); -#endif /* !UNIV_HOTBACKUP */ if (page_zip) { mach_write_to_8(page + FIL_RTREE_SPLIT_SEQ_NUM, ssn_id); page_zip_write_header(page_zip, page + FIL_RTREE_SPLIT_SEQ_NUM, 8, mtr); -#ifndef UNIV_HOTBACKUP } else if (mtr) { mlog_write_ull(page + FIL_RTREE_SPLIT_SEQ_NUM, ssn_id, mtr); -#endif /* !UNIV_HOTBACKUP */ } else { mach_write_to_8(page + FIL_RTREE_SPLIT_SEQ_NUM, ssn_id); } @@ -264,7 +259,6 @@ page_header_set_ptr( page_header_set_field(page, page_zip, field, offs); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Resets the last insert info field in the page header. Writes to mlog about this operation. */ @@ -290,7 +284,6 @@ page_header_reset_last_insert( MLOG_2BYTES, mtr); } } -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Determine whether the page is in new-style compact format. @@ -618,7 +611,6 @@ page_rec_get_nth( return((rec_t*) page_rec_get_nth_const(page, nth)); } -#ifndef UNIV_HOTBACKUP /************************************************************//** Returns the middle record of the records on the page. If there is an even number of records in the list, returns the first record of the @@ -634,7 +626,6 @@ page_get_middle_rec( return(page_rec_get_nth(page, middle)); } -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Gets the page number. @@ -1141,7 +1132,6 @@ page_get_free_space_of_empty( - 2 * PAGE_DIR_SLOT_SIZE)); } -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Write a 32-bit field in a data dictionary record. */ UNIV_INLINE @@ -1162,7 +1152,6 @@ page_rec_write_field( mlog_write_ulint(data, val, MLOG_4BYTES, mtr); } -#endif /* !UNIV_HOTBACKUP */ /************************************************************//** Each user record on a page, and also the deleted user records in the heap diff --git a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h index 7b5df3d306be3..4a32595af666a 100644 --- a/storage/innobase/include/page0zip.h +++ b/storage/innobase/include/page0zip.h @@ -97,7 +97,6 @@ page_zip_set_size( page_zip_des_t* page_zip, /*!< in/out: compressed page */ ulint size); /*!< in: size in bytes */ -#ifndef UNIV_HOTBACKUP /** Determine if a record is so big that it needs to be stored externally. @param[in] rec_size length of the record in bytes @param[in] comp nonzero=compact format @@ -132,7 +131,6 @@ bool page_zip_is_too_big( const dict_index_t* index, const dtuple_t* entry); -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Initialize a compressed page descriptor. */ @@ -459,7 +457,7 @@ page_zip_reorganize( dict_index_t* index, /*!< in: index of the B-tree node */ mtr_t* mtr) /*!< in: mini-transaction */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP + /**********************************************************************//** Copy the records of a page byte for byte. Do not copy the page header or trailer, except those B-tree header fields that are directly @@ -476,7 +474,6 @@ page_zip_copy_recs( const page_t* src, /*!< in: page */ dict_index_t* index, /*!< in: index of the B-tree */ mtr_t* mtr); /*!< in: mini-transaction */ -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Parses a log record of compressing an index page. diff --git a/storage/innobase/include/page0zip.ic b/storage/innobase/include/page0zip.ic index 9963fe01c8294..9cc54dc42fa15 100644 --- a/storage/innobase/include/page0zip.ic +++ b/storage/innobase/include/page0zip.ic @@ -148,7 +148,6 @@ page_zip_set_size( ut_ad(page_zip_get_size(page_zip) == size); } -#ifndef UNIV_HOTBACKUP /** Determine if a record is so big that it needs to be stored externally. @param[in] rec_size length of the record in bytes @param[in] comp nonzero=compact format @@ -188,7 +187,6 @@ page_zip_rec_needs_ext( return(rec_size >= page_get_free_space_of_empty(comp) / 2); } -#endif /* !UNIV_HOTBACKUP */ #ifdef UNIV_DEBUG /**********************************************************************//** @@ -364,9 +362,7 @@ page_zip_write_header( /* ut_ad(page_zip_validate(page_zip, str - pos)); */ if (mtr) { -#ifndef UNIV_HOTBACKUP page_zip_write_header_log(str, length, mtr); -#endif /* !UNIV_HOTBACKUP */ } } diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index 8490e7c9c8836..ed700139b5338 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -773,7 +773,6 @@ rec_copy( const rec_t* rec, const ulint* offsets); -#ifndef UNIV_HOTBACKUP /**********************************************************//** Determines the size of a data tuple prefix in a temporary file. @return total size */ @@ -846,7 +845,6 @@ rec_fold( ulint n_bytes, index_id_t tree_id) MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /*********************************************************//** Builds a physical record out of a data tuple and stores it into the given buffer. @@ -911,7 +909,6 @@ rec_get_converted_size( const dtuple_t* dtuple, /*!< in: data tuple */ ulint n_ext) /*!< in: number of externally stored columns */ MY_ATTRIBUTE((warn_unused_result, nonnull)); -#ifndef UNIV_HOTBACKUP /**************************************************************//** Copies the first n fields of a physical record to a data tuple. The fields are copied to the memory heap. */ @@ -925,7 +922,6 @@ rec_copy_prefix_to_dtuple( to copy */ mem_heap_t* heap) /*!< in: memory heap */ MY_ATTRIBUTE((nonnull)); -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** Validates the consistency of a physical record. @return TRUE if ok */ @@ -943,7 +939,6 @@ rec_print_old( FILE* file, /*!< in: file where to print */ const rec_t* rec) /*!< in: physical record */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP /***************************************************************//** Prints a physical record in ROW_FORMAT=COMPACT. Ignores the record header. */ @@ -1104,7 +1099,6 @@ rec_get_trx_id( const dict_index_t* index) /*!< in: clustered index */ MY_ATTRIBUTE((nonnull, warn_unused_result)); # endif /* UNIV_DEBUG */ -#endif /* UNIV_HOTBACKUP */ /* Maximum lengths for the data in a physical record if the offsets are given in one byte (resp. two byte) format. */ diff --git a/storage/innobase/include/rem0rec.ic b/storage/innobase/include/rem0rec.ic index b855a39da9ea2..e16eab6218164 100644 --- a/storage/innobase/include/rem0rec.ic +++ b/storage/innobase/include/rem0rec.ic @@ -1696,7 +1696,6 @@ rec_get_converted_size( return(data_size + extra_size); } -#ifndef UNIV_HOTBACKUP /** Fold a prefix of a physical record. @param[in] rec index record @param[in] offsets return value of rec_get_offsets() @@ -1761,4 +1760,3 @@ rec_fold( return(fold); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/row0upd.h b/storage/innobase/include/row0upd.h index 3c1033fe41903..86b282553d80b 100644 --- a/storage/innobase/include/row0upd.h +++ b/storage/innobase/include/row0upd.h @@ -33,12 +33,9 @@ Created 12/27/1996 Heikki Tuuri #include "dict0types.h" #include "trx0types.h" #include - -#ifndef UNIV_HOTBACKUP -# include "btr0pcur.h" -# include "que0types.h" -# include "pars0types.h" -#endif /* !UNIV_HOTBACKUP */ +#include "btr0pcur.h" +#include "que0types.h" +#include "pars0types.h" /** The std::deque to store cascade update nodes, that uses mem_heap_t as allocator. */ @@ -80,7 +77,7 @@ upd_get_nth_field( #else # define upd_get_nth_field(update, n) ((update)->fields + (n)) #endif -#ifndef UNIV_HOTBACKUP + /*********************************************************************//** Sets an index field number to be updated by an update vector field. */ UNIV_INLINE @@ -191,7 +188,6 @@ row_upd_changes_disowned_external( /*==============================*/ const upd_t* update) /*!< in: update vector */ MY_ATTRIBUTE((nonnull, warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Replaces the new column values stored in the update vector to the record given. No field size changes are allowed. This function is @@ -207,7 +203,7 @@ row_upd_rec_in_place( const upd_t* update, /*!< in: update vector */ page_zip_des_t* page_zip);/*!< in: compressed page with enough space available, or NULL */ -#ifndef UNIV_HOTBACKUP + /***************************************************************//** Builds an update vector from those fields which in a secondary index entry differ from a record that has the equal ordering fields. NOTE: we compare @@ -403,7 +399,6 @@ que_thr_t* row_upd_step( /*=========*/ que_thr_t* thr); /*!< in: query thread */ -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Parses the log data of system field values. @return log data end or NULL */ @@ -449,7 +444,6 @@ struct upd_field_t{ index. If this field is a virtual column, then field_no represents the nth virtual column in the table */ -#ifndef UNIV_HOTBACKUP unsigned orig_len:16; /*!< original length of the locally stored part of an externally stored column, or 0 */ @@ -457,7 +451,6 @@ struct upd_field_t{ value: it refers to column values and constants in the symbol table of the query graph */ -#endif /* !UNIV_HOTBACKUP */ dfield_t new_val; /*!< new value for the column */ dfield_t* old_v_val; /*!< old value for the virtual column */ }; @@ -516,7 +509,6 @@ struct upd_t{ }; -#ifndef UNIV_HOTBACKUP /* Update node structure which also implements the delete operation of a row */ @@ -659,7 +651,6 @@ struct upd_node_t{ #define UPD_NODE_NO_SIZE_CHANGE 2 /* no record field size will be changed in the update */ -#endif /* !UNIV_HOTBACKUP */ #ifndef UNIV_NONINL #include "row0upd.ic" diff --git a/storage/innobase/include/row0upd.ic b/storage/innobase/include/row0upd.ic index ab1dc5c70767b..8b794e47a07b1 100644 --- a/storage/innobase/include/row0upd.ic +++ b/storage/innobase/include/row0upd.ic @@ -24,12 +24,10 @@ Created 12/27/1996 Heikki Tuuri *******************************************************/ #include "mtr0log.h" -#ifndef UNIV_HOTBACKUP -# include "trx0trx.h" -# include "trx0undo.h" -# include "row0row.h" -# include "lock0lock.h" -#endif /* !UNIV_HOTBACKUP */ +#include "trx0trx.h" +#include "trx0undo.h" +#include "row0row.h" +#include "lock0lock.h" #include "page0zip.h" /*********************************************************************//** @@ -87,7 +85,6 @@ upd_get_nth_field( } #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Sets an index field number to be updated by an update vector field. */ UNIV_INLINE @@ -220,4 +217,3 @@ row_upd_rec_sys_fields( trx_write_roll_ptr(rec + offset + DATA_TRX_ID_LEN, roll_ptr); } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/srv0mon.h b/storage/innobase/include/srv0mon.h index d5a305bdf68fb..e4e1394c2d3de 100644 --- a/storage/innobase/include/srv0mon.h +++ b/storage/innobase/include/srv0mon.h @@ -38,9 +38,6 @@ Created 12/15/2009 Jimmy Yang #include -#ifndef UNIV_HOTBACKUP - - /** Possible status values for "mon_status" in "struct monitor_value" */ enum monitor_running_status { MONITOR_STARTED = 1, /*!< Monitor has been turned on */ @@ -899,9 +896,5 @@ srv_mon_default_on(void); #ifndef UNIV_NONINL #include "srv0mon.ic" #endif -#else /* !UNIV_HOTBACKUP */ -# define MONITOR_INC(x) ((void) 0) -# define MONITOR_DEC(x) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ #endif diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 748a21fdcd25e..92572a0f8b35e 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -48,7 +48,6 @@ Created 10/10/1995 Heikki Tuuri #include "mysql/psi/psi.h" #include "univ.i" -#ifndef UNIV_HOTBACKUP #include "log0log.h" #include "os0event.h" #include "que0types.h" @@ -309,8 +308,6 @@ extern long srv_mtflush_threads; /* If this flag is TRUE, then we will use multi threaded flush. */ extern my_bool srv_use_mtflush; -#endif /* !UNIV_HOTBACKUP */ - /** Server undo tablespaces directory, can be absolute path. */ extern char* srv_undo_dir; @@ -349,7 +346,6 @@ extern const ulint SRV_UNDO_TABLESPACE_SIZE_IN_PAGES; extern char* srv_log_group_home_dir; -#ifndef UNIV_HOTBACKUP /** Maximum number of srv_n_log_files, or innodb_log_files_in_group */ #define SRV_N_LOG_FILES_MAX 100 extern ulong srv_n_log_files; @@ -660,8 +656,6 @@ extern PSI_stage_info srv_stage_alter_table_read_pk_internal_sort; extern PSI_stage_info srv_stage_buffer_pool_load; #endif /* HAVE_PSI_STAGE_INTERFACE */ -#endif /* !UNIV_HOTBACKUP */ - #ifndef _WIN32 /** Alternatives for the file flush option in Unix; see the InnoDB manual about what these mean */ @@ -738,7 +732,6 @@ typedef enum srv_stats_method_name_enum srv_stats_method_name_t; extern ulong srv_debug_compress; #endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP /** Types of threads existing in the system. */ enum srv_thread_type { SRV_NONE, /*!< None */ @@ -1152,20 +1145,6 @@ struct srv_slot_t{ (only used for user threads) */ }; -#else /* !UNIV_HOTBACKUP */ -# define srv_use_adaptive_hash_indexes FALSE -# define srv_use_native_aio FALSE -# define srv_numa_interleave FALSE -# define srv_force_recovery 0UL -# define srv_set_io_thread_op_info(t,info) ((void) 0) -# define srv_reset_io_thread_op_info() ((void) 0) -# define srv_is_being_started 0 -# define srv_win_file_flush_method SRV_WIN_IO_UNBUFFERED -# define srv_unix_file_flush_method SRV_UNIX_O_DSYNC -# define srv_start_raw_disk_in_use 0 -# define srv_file_per_table 1 -#endif /* !UNIV_HOTBACKUP */ - #ifdef WITH_WSREP UNIV_INTERN void diff --git a/storage/innobase/include/srv0start.h b/storage/innobase/include/srv0start.h index 3b7b2375788c8..fb41ae50d2a23 100644 --- a/storage/innobase/include/srv0start.h +++ b/storage/innobase/include/srv0start.h @@ -71,7 +71,7 @@ char* srv_add_path_separator_if_needed( /*=============================*/ char* str); /*!< in: null-terminated character string */ -#ifndef UNIV_HOTBACKUP + /****************************************************************//** Starts Innobase and creates a new database if database files are not found and the user wants. @@ -156,7 +156,6 @@ extern bool srv_startup_is_before_trx_rollback_phase; /** TRUE if a raw partition is in use */ extern ibool srv_start_raw_disk_in_use; - /** Shutdown state */ enum srv_shutdown_t { SRV_SHUTDOWN_NONE = 0, /*!< Database running normally */ @@ -176,6 +175,4 @@ enum srv_shutdown_t { /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to SRV_SHUTDOWN_CLEANUP and then to SRV_SHUTDOWN_LAST_PHASE, and so on */ extern enum srv_shutdown_t srv_shutdown_state; -#endif /* !UNIV_HOTBACKUP */ - #endif diff --git a/storage/innobase/include/sync0rw.h b/storage/innobase/include/sync0rw.h index 396156e95188b..9a9da2d145ac5 100644 --- a/storage/innobase/include/sync0rw.h +++ b/storage/innobase/include/sync0rw.h @@ -34,7 +34,6 @@ Created 9/11/1995 Heikki Tuuri #define sync0rw_h #include "univ.i" -#ifndef UNIV_HOTBACKUP #include "ut0counter.h" #include "os0event.h" #include "ut0mutex.h" @@ -42,8 +41,6 @@ Created 9/11/1995 Heikki Tuuri /** Enable semaphore request instrumentation */ extern my_bool srv_instrument_semaphores; -#endif /* !UNIV_HOTBACKUP */ - /** Counters for RW locks. */ struct rw_lock_stats_t { typedef ib_counter_t int64_counter_t; @@ -96,7 +93,6 @@ enum rw_lock_type_t { RW_NO_LATCH = 8 }; -#ifndef UNIV_HOTBACKUP /* We decrement lock_word by X_LOCK_DECR for each x_lock. It is also the start value for the lock_word, meaning that it limits the maximum number of concurrent read locks before the rw_lock breaks. */ @@ -897,11 +893,8 @@ pfs_rw_lock_free_func( rw_lock_t* lock); /*!< in: rw-lock */ #endif /* UNIV_PFS_RWLOCK */ - #ifndef UNIV_NONINL #include "sync0rw.ic" #endif /* !UNIV_NONINL */ -#endif /* !UNIV_HOTBACKUP */ - #endif /* sync0rw.h */ diff --git a/storage/innobase/include/trx0rec.h b/storage/innobase/include/trx0rec.h index b7a2deac63e28..90f4604043b2b 100644 --- a/storage/innobase/include/trx0rec.h +++ b/storage/innobase/include/trx0rec.h @@ -35,9 +35,7 @@ Created 3/26/1996 Heikki Tuuri #include "rem0types.h" #include "page0types.h" #include "row0log.h" - -#ifndef UNIV_HOTBACKUP -# include "que0types.h" +#include "que0types.h" /***********************************************************************//** Copies the undo record to the heap. @@ -276,7 +274,6 @@ trx_undo_prev_version_build( into this function by purge thread or not. And if we read "after image" of undo log */ -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of adding an undo log record. @return end of log record or NULL */ @@ -344,8 +341,6 @@ trx_undo_read_v_idx( bool* is_undo_log, ulint* field_no); -#ifndef UNIV_HOTBACKUP - /* Types of an undo log record: these have to be smaller than 16, as the compilation info multiplied by 16 is ORed to this value in an undo log record */ @@ -373,6 +368,4 @@ record */ #include "trx0rec.ic" #endif -#endif /* !UNIV_HOTBACKUP */ - #endif /* trx0rec_h */ diff --git a/storage/innobase/include/trx0rec.ic b/storage/innobase/include/trx0rec.ic index 111c05c60aa71..c2c756484b201 100644 --- a/storage/innobase/include/trx0rec.ic +++ b/storage/innobase/include/trx0rec.ic @@ -23,7 +23,6 @@ Transaction undo log record Created 3/26/1996 Heikki Tuuri *******************************************************/ -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Reads from an undo log record the record type. @return record type */ @@ -98,4 +97,3 @@ trx_undo_rec_copy( ut_ad(len < UNIV_PAGE_SIZE); return((trx_undo_rec_t*) mem_heap_dup(heap, undo_rec, len)); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index ddf535158b693..ae46d2abae9b4 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -31,7 +31,6 @@ Created 3/26/1996 Heikki Tuuri #include "buf0buf.h" #include "fil0fil.h" #include "trx0types.h" -#ifndef UNIV_HOTBACKUP #include "mem0mem.h" #include "mtr0mtr.h" #include "ut0byte.h" @@ -367,40 +366,6 @@ Check if there are any active (non-prepared) transactions. ulint trx_sys_any_active_transactions(void); /*=================================*/ -#else /* !UNIV_HOTBACKUP */ -/*****************************************************************//** -Prints to stderr the MySQL binlog info in the system header if the -magic number shows it valid. */ -void -trx_sys_print_mysql_binlog_offset_from_page( -/*========================================*/ - const byte* page); /*!< in: buffer containing the trx - system header page, i.e., page number - TRX_SYS_PAGE_NO in the tablespace */ -/*****************************************************************//** -Reads the file format id from the first system table space file. -Even if the call succeeds and returns TRUE, the returned format id -may be ULINT_UNDEFINED signalling that the format id was not present -in the data file. -@return TRUE if call succeeds */ -ibool -trx_sys_read_file_format_id( -/*========================*/ - const char *pathname, /*!< in: pathname of the first system - table space file */ - ulint *format_id); /*!< out: file format of the system table - space */ -/*****************************************************************//** -Reads the file format id from the given per-table data file. -@return TRUE if call succeeds */ -ibool -trx_sys_read_pertable_file_format_id( -/*=================================*/ - const char *pathname, /*!< in: pathname of a per-table - datafile */ - ulint *format_id); /*!< out: file format of the per-table - data file */ -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** Get the name representation of the file format from its id. @return pointer to the max format name */ @@ -590,7 +555,6 @@ identifier is added to this 64-bit constant. */ | TRX_SYS_FILE_FORMAT_TAG_MAGIC_N_LOW) /* @} */ -#ifndef UNIV_HOTBACKUP /** The transaction system central memory data structure. */ struct trx_sys_t { @@ -683,7 +647,6 @@ struct trx_sys_t { two) is assigned, the field TRX_SYS_TRX_ID_STORE on the transaction system page is updated */ #define TRX_SYS_TRX_ID_WRITE_MARGIN ((trx_id_t) 256) -#endif /* !UNIV_HOTBACKUP */ /** Test if trx_sys->mutex is owned. */ #define trx_sys_mutex_own() (trx_sys->mutex.is_owned()) diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic index 6158aea0c48d5..b07f487984cbf 100644 --- a/storage/innobase/include/trx0sys.ic +++ b/storage/innobase/include/trx0sys.ic @@ -25,9 +25,8 @@ Created 3/26/1996 Heikki Tuuri #include "trx0trx.h" #include "data0type.h" -#ifndef UNIV_HOTBACKUP -# include "srv0srv.h" -# include "mtr0log.h" +#include "srv0srv.h" +#include "mtr0log.h" /* The typedef for rseg slot in the file copy */ typedef byte trx_sysf_rseg_t; @@ -198,7 +197,6 @@ trx_sysf_rseg_set_page_no( page_no, MLOG_4BYTES, mtr); } -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** Writes a trx id to an index page. In case that the id size changes in @@ -230,7 +228,6 @@ trx_sys_is_noredo_rseg_slot( return(slot_id > 0 && slot_id < (srv_tmp_undo_logs + 1)); } -#ifndef UNIV_HOTBACKUP /*****************************************************************//** Reads a trx id from an index page. In case that the id size changes in some future version, this function should be used instead of @@ -503,5 +500,3 @@ trx_sys_rw_trx_add(trx_t* trx) trx_sys->rw_trx_set.insert(TrxTrack(trx->id, trx)); ut_d(trx->in_rw_trx_list = true); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 19b0747c6a366..2159fc76548a6 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -36,7 +36,6 @@ Created 3/26/1996 Heikki Tuuri #include "trx0types.h" #include "ut0new.h" -#ifndef UNIV_HOTBACKUP #include "lock0types.h" #include "log0log.h" #include "usr0types.h" @@ -421,7 +420,6 @@ trx_set_dict_operation( enum trx_dict_op_t op); /*!< in: operation, not TRX_DICT_OP_NONE */ -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Determines if a transaction is in the given state. The caller must hold trx_sys->mutex, or it must be the thread @@ -468,9 +466,6 @@ ibool trx_is_strict( /*==========*/ trx_t* trx); /*!< in: transaction */ -#else /* !UNIV_HOTBACKUP */ -#define trx_is_interrupted(trx) FALSE -#endif /* !UNIV_HOTBACKUP */ /*******************************************************************//** Calculates the "weight" of a transaction. The weight of one transaction @@ -1613,6 +1608,5 @@ class TrxInInnoDB { #ifndef UNIV_NONINL #include "trx0trx.ic" #endif -#endif /* !UNIV_HOTBACKUP */ #endif diff --git a/storage/innobase/include/trx0undo.h b/storage/innobase/include/trx0undo.h index 60fbb9d2304a0..1acb5b7c1ada0 100644 --- a/storage/innobase/include/trx0undo.h +++ b/storage/innobase/include/trx0undo.h @@ -34,7 +34,6 @@ Created 3/26/1996 Heikki Tuuri #include "page0types.h" #include "trx0xa.h" -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Builds a roll pointer. @return roll pointer */ @@ -75,7 +74,6 @@ trx_undo_trx_id_is_insert( /*======================*/ const byte* trx_id) /*!< in: DB_TRX_ID, followed by DB_ROLL_PTR */ MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** Writes a roll ptr to an index page. In case that the size changes in some future version, this function should be used instead of @@ -97,7 +95,6 @@ roll_ptr_t trx_read_roll_ptr( /*==============*/ const byte* ptr); /*!< in: pointer to memory from where to read */ -#ifndef UNIV_HOTBACKUP /** Gets an undo log page and x-latches it. @param[in] page_id page id @@ -366,7 +363,6 @@ bool trx_undo_truncate_tablespace( undo::Truncate* undo_trunc); -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses the redo log entry of an undo log page initialization. @return end of log record or NULL */ @@ -427,7 +423,7 @@ trx_undo_mem_free( #define TRX_UNDO_PREPARED 5 /* contains an undo log of an prepared transaction */ -#if !defined UNIV_HOTBACKUP && !defined UNIV_INNOCHECKSUM +#ifndef UNIV_INNOCHECKSUM /** Transaction undo log memory object; this is protected by the undo_mutex in the corresponding transaction object */ @@ -488,7 +484,7 @@ struct trx_undo_t { /*!< undo log objects in the rollback segment are chained into lists */ }; -#endif /* !UNIV_HOTBACKUP && !UNIV_INNOCHECKSUM */ +#endif /* !UNIV_INNOCHECKSUM */ /** The offset of the undo log page header on pages of the undo log */ #define TRX_UNDO_PAGE_HDR FSEG_PAGE_DATA @@ -553,7 +549,6 @@ log segment */ #define TRX_UNDO_SEG_HDR_SIZE (4 + FSEG_HEADER_SIZE + FLST_BASE_NODE_SIZE) /* @} */ - /** The undo log header. There can be several undo log headers on the first page of an update undo log segment. */ /* @{ */ diff --git a/storage/innobase/include/trx0undo.ic b/storage/innobase/include/trx0undo.ic index f8e74d0fb0360..62d9e64b13fb6 100644 --- a/storage/innobase/include/trx0undo.ic +++ b/storage/innobase/include/trx0undo.ic @@ -26,7 +26,6 @@ Created 3/26/1996 Heikki Tuuri #include "data0type.h" #include "page0page.h" -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Builds a roll pointer. @return roll pointer */ @@ -116,7 +115,6 @@ trx_undo_trx_id_is_insert( #endif return(static_cast(trx_id[DATA_TRX_ID_LEN] >> 7)); } -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** Writes a roll ptr to an index page. In case that the size changes in @@ -153,8 +151,6 @@ trx_read_roll_ptr( return(mach_read_from_7(ptr)); } -#ifndef UNIV_HOTBACKUP - /** Gets an undo log page and x-latches it. @param[in] page_id page id @param[in] page_size page size @@ -361,4 +357,3 @@ trx_undo_page_get_first_rec( return(undo_page + start); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index f885bd2191f1d..f877eedf5516c 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -34,10 +34,6 @@ Created 1/20/1994 Heikki Tuuri #ifndef univ_i #define univ_i -#ifdef UNIV_HOTBACKUP -#include "hb_univ.i" -#endif /* UNIV_HOTBACKUP */ - /* aux macros to convert M into "123" (string) if M is defined like #define M 123 */ #define _IB_TO_STR(s) #s @@ -79,27 +75,22 @@ the virtual method table (vtable) in GCC 3. */ # include #endif /* _WIN32 */ -/* The defines used with MySQL */ - -#ifndef UNIV_HOTBACKUP - /* Include a minimum number of SQL header files so that few changes made in SQL code cause a complete InnoDB rebuild. These headers are used throughout InnoDB but do not include too much themselves. They support cross-platform development and expose comonly used SQL names. */ -# include +#include /* JAN: TODO: missing 5.7 header */ #ifdef HAVE_MY_THREAD_H //# include #endif -# ifndef UNIV_INNOCHECKSUM -# include -# include -# endif /* !UNIV_INNOCHECKSUM */ -#endif /* !UNIV_HOTBACKUP */ +#ifndef UNIV_INNOCHECKSUM +# include +# include +#endif /* !UNIV_INNOCHECKSUM */ /* Include to get S_I... macros defined for os0file.cc */ #include @@ -107,13 +98,7 @@ support cross-platform development and expose comonly used SQL names. */ #ifndef _WIN32 # include /* mmap() for os0proc.cc */ # include -#endif /* !_WIN32 */ - -/* Include the header file generated by CMake */ -#ifndef _WIN32 -# ifndef UNIV_HOTBACKUP -# include "my_config.h" -# endif /* UNIV_HOTBACKUP */ +# include "my_config.h" #endif #include @@ -126,7 +111,7 @@ support cross-platform development and expose comonly used SQL names. */ /* Following defines are to enable performance schema instrumentation in each of five InnoDB modules if HAVE_PSI_INTERFACE is defined. */ -#if defined(HAVE_PSI_INTERFACE) && !defined(UNIV_HOTBACKUP) +#ifdef HAVE_PSI_INTERFACE # define UNIV_PFS_MUTEX # define UNIV_PFS_RWLOCK /* For I/O instrumentation, performance schema rely diff --git a/storage/innobase/include/ut0counter.h b/storage/innobase/include/ut0counter.h index 175427df333e6..70381f26a84f8 100644 --- a/storage/innobase/include/ut0counter.h +++ b/storage/innobase/include/ut0counter.h @@ -32,19 +32,11 @@ Created 2012/04/12 by Sunny Bains #include "os0thread.h" /** CPU cache line size */ -#ifndef UNIV_HOTBACKUP -# ifdef CPU_LEVEL1_DCACHE_LINESIZE -# define CACHE_LINE_SIZE CPU_LEVEL1_DCACHE_LINESIZE -# else -# error CPU_LEVEL1_DCACHE_LINESIZE is undefined -# endif /* CPU_LEVEL1_DCACHE_LINESIZE */ +#ifdef CPU_LEVEL1_DCACHE_LINESIZE +# define CACHE_LINE_SIZE CPU_LEVEL1_DCACHE_LINESIZE #else -#ifdef powerpc -#define CACHE_LINE_SIZE 128 -#else -# define CACHE_LINE_SIZE 64 -#endif /* __powerpc__ */ -#endif /* UNIV_HOTBACKUP */ +# error CPU_LEVEL1_DCACHE_LINESIZE is undefined +#endif /* CPU_LEVEL1_DCACHE_LINESIZE */ /** Default number of slots to use in ib_counter_t */ #define IB_N_SLOTS 64 diff --git a/storage/innobase/include/ut0mem.h b/storage/innobase/include/ut0mem.h index 6d56be4d82018..f20273daca661 100644 --- a/storage/innobase/include/ut0mem.h +++ b/storage/innobase/include/ut0mem.h @@ -27,10 +27,8 @@ Created 5/30/1994 Heikki Tuuri #define ut0mem_h #include "univ.i" -#ifndef UNIV_HOTBACKUP -# include "os0event.h" -# include "ut0mutex.h" -#endif /* !UNIV_HOTBACKUP */ +#include "os0event.h" +#include "ut0mutex.h" /** Wrapper for memcpy(3). Copy memory area when the source and target are not overlapping. diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 087f175db50f7..fb8b66ddd3978 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -52,61 +52,60 @@ Created 1/20/1994 Heikki Tuuri /** Time stamp */ typedef time_t ib_time_t; -#ifndef UNIV_HOTBACKUP -# if defined(HAVE_PAUSE_INSTRUCTION) +#ifdef HAVE_PAUSE_INSTRUCTION /* According to the gcc info page, asm volatile means that the instruction has important side-effects and must not be removed. Also asm volatile may trigger a memory barrier (spilling all registers to memory). */ -# ifdef __SUNPRO_CC -# define UT_RELAX_CPU() asm ("pause" ) -# else -# define UT_RELAX_CPU() __asm__ __volatile__ ("pause") -# endif /* __SUNPRO_CC */ - -# elif defined(HAVE_FAKE_PAUSE_INSTRUCTION) -# define UT_RELAX_CPU() __asm__ __volatile__ ("rep; nop") -# elif defined _WIN32 +# ifdef __SUNPRO_CC +# define UT_RELAX_CPU() asm ("pause" ) +# else +# define UT_RELAX_CPU() __asm__ __volatile__ ("pause") +# endif /* __SUNPRO_CC */ + +#elif defined(HAVE_FAKE_PAUSE_INSTRUCTION) +# define UT_RELAX_CPU() __asm__ __volatile__ ("rep; nop") +#elif defined _WIN32 /* In the Win32 API, the x86 PAUSE instruction is executed by calling the YieldProcessor macro defined in WinNT.h. It is a CPU architecture- independent way by using YieldProcessor. */ -# define UT_RELAX_CPU() YieldProcessor() -# elif defined(__powerpc__) && defined __GLIBC__ -#include -# define UT_RELAX_CPU() do { \ +# define UT_RELAX_CPU() YieldProcessor() +#elif defined(__powerpc__) && defined __GLIBC__ +# include +# define UT_RELAX_CPU() do { \ volatile lint volatile_var = __ppc_get_timebase(); \ } while (0) -# else -# define UT_RELAX_CPU() do { \ +#else +# define UT_RELAX_CPU() do { \ volatile int32 volatile_var; \ int32 oldval= 0; \ my_atomic_cas32(&volatile_var, &oldval, 1); \ } while (0) -# endif +#endif #if defined (__GNUC__) -# define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") +# define UT_COMPILER_BARRIER() __asm__ __volatile__ ("":::"memory") #elif defined (_MSC_VER) -# define UT_COMPILER_BARRIER() _ReadWriteBarrier() +# define UT_COMPILER_BARRIER() _ReadWriteBarrier() #else -# define UT_COMPILER_BARRIER() +# define UT_COMPILER_BARRIER() #endif -# if defined(HAVE_HMT_PRIORITY_INSTRUCTION) -#include -# define UT_LOW_PRIORITY_CPU() __ppc_set_ppr_low() -# define UT_RESUME_PRIORITY_CPU() __ppc_set_ppr_med() -# else -# define UT_LOW_PRIORITY_CPU() ((void)0) -# define UT_RESUME_PRIORITY_CPU() ((void)0) -# endif +#if defined(HAVE_HMT_PRIORITY_INSTRUCTION) +# include +# define UT_LOW_PRIORITY_CPU() __ppc_set_ppr_low() +# define UT_RESUME_PRIORITY_CPU() __ppc_set_ppr_med() +#else +# define UT_LOW_PRIORITY_CPU() ((void)0) +# define UT_RESUME_PRIORITY_CPU() ((void)0) +#endif /*********************************************************************//** Delays execution for at most max_wait_us microseconds or returns earlier if cond becomes true. @param cond in: condition to wait for; evaluated every 2 ms @param max_wait_us in: maximum delay to wait, in microseconds */ -#define UT_WAIT_FOR(cond, max_wait_us) \ +# define UT_WAIT_FOR(cond, max_wait_us) \ do { \ uintmax_t start_us; \ start_us = ut_time_us(NULL); \ @@ -116,7 +115,6 @@ do { \ os_thread_sleep(2000 /* 2 ms */); \ } \ } while (0) -#endif /* !UNIV_HOTBACKUP */ #define ut_max std::max #define ut_min std::min @@ -229,7 +227,7 @@ the only way to manipulate it is to use the function ut_difftime. ib_time_t ut_time(void); /*=========*/ -#ifndef UNIV_HOTBACKUP + /**********************************************************//** Returns system time. Upon successful completion, the value 0 is returned; otherwise the @@ -259,7 +257,6 @@ purposes. ulint ut_time_ms(void); /*============*/ -#endif /* !UNIV_HOTBACKUP */ /**********************************************************//** Returns the number of milliseconds since some epoch. The @@ -314,23 +311,6 @@ void ut_sprintf_timestamp( /*=================*/ char* buf); /*!< in: buffer where to sprintf */ -#ifdef UNIV_HOTBACKUP -/**********************************************************//** -Sprintfs a timestamp to a buffer with no spaces and with ':' characters -replaced by '_'. */ -void -ut_sprintf_timestamp_without_extra_chars( -/*=====================================*/ - char* buf); /*!< in: buffer where to sprintf */ -/**********************************************************//** -Returns current year, month, day. */ -void -ut_get_year_month_day( -/*==================*/ - ulint* year, /*!< out: current year */ - ulint* month, /*!< out: month */ - ulint* day); /*!< out: day */ -#else /* UNIV_HOTBACKUP */ /*************************************************************//** Runs an idle loop on CPU. The argument gives the desired delay in microseconds on 100 MHz Pentium + Visual C++. @@ -339,7 +319,6 @@ ulint ut_delay( /*=====*/ ulint delay); /*!< in: delay in microseconds on 100 MHz Pentium */ -#endif /* UNIV_HOTBACKUP */ /*************************************************************//** Prints the contents of a memory buffer in hex and ascii. */ void @@ -368,7 +347,6 @@ ut_print_buf( ulint len) /*!< in: length of the buffer */ MY_ATTRIBUTE((nonnull)); -#ifndef UNIV_HOTBACKUP /* Forward declaration of transaction handle */ struct trx_t; @@ -418,7 +396,6 @@ ut_copy_file( /*=========*/ FILE* dest, /*!< in: output file */ FILE* src); /*!< in: input file to be appended to output */ -#endif /* !UNIV_HOTBACKUP */ #ifdef _WIN32 /**********************************************************************//** diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 0a5e3b27bc097..979a40e22fe99 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -42,7 +42,6 @@ Created 12/9/1995 Heikki Tuuri #include "mem0mem.h" #include "buf0buf.h" -#ifndef UNIV_HOTBACKUP #include "buf0flu.h" #include "srv0srv.h" #include "log0recv.h" @@ -56,7 +55,6 @@ Created 12/9/1995 Heikki Tuuri #include "trx0roll.h" #include "srv0mon.h" #include "sync0sync.h" -#endif /* !UNIV_HOTBACKUP */ /* Used for debugging */ // #define DEBUG_CRYPT 1 @@ -144,7 +142,6 @@ void log_io_complete_checkpoint(void); /*============================*/ -#ifndef UNIV_HOTBACKUP /****************************************************************//** Returns the oldest modified block lsn in the pool, or log_sys->lsn if none exists. @@ -167,7 +164,6 @@ log_buf_pool_get_oldest_modification(void) return(lsn); } -#endif /* !UNIV_HOTBACKUP */ /** Extends the log buffer. @param[in] len requested minimum size in bytes */ @@ -261,7 +257,6 @@ log_buffer_extend( << LOG_BUFFER_SIZE << "."; } -#ifndef UNIV_HOTBACKUP /** Calculate actual length in redo buffer and file including block header and trailer. @param[in] len length to write @@ -352,7 +347,7 @@ log_margin_checkpoint_age( return; } -#endif /* !UNIV_HOTBACKUP */ + /** Open the log for log_write_low. The log must be closed with log_close. @param[in] len length of the data to be written @return start lsn of the log record */ @@ -712,7 +707,7 @@ log_group_set_fields( group->lsn_offset = log_group_calc_lsn_offset(lsn, group); group->lsn = lsn; } -#ifndef UNIV_HOTBACKUP + /*****************************************************************//** Calculates the recommended highest values for lsn - last_checkpoint_lsn and lsn - buf_get_oldest_modification(). @@ -920,7 +915,7 @@ log_group_init( UT_LIST_ADD_LAST(log_sys->log_groups, group); return(log_calc_max_ages()); } -#endif /* !UNIV_HOTBACKUP */ + /******************************************************//** Completes an i/o to a log file. */ void @@ -1265,7 +1260,6 @@ log_write_up_to( } #ifdef _WIN32 -# ifndef UNIV_HOTBACKUP /* write requests during fil_flush() might not be good for Windows */ if (log_sys->n_pending_flushes > 0 || !os_event_is_set(log_sys->flush_event)) { @@ -1273,11 +1267,6 @@ log_write_up_to( os_event_wait(log_sys->flush_event); goto loop; } -# else - if (log_sys->n_pending_flushes > 0) { - goto loop; - } -# endif /* !UNIV_HOTBACKUP */ #endif /* _WIN32 */ /* If it is a write call we should just go ahead and do it @@ -1481,7 +1470,7 @@ log_flush_margin(void) log_write_up_to(lsn, false); } } -#ifndef UNIV_HOTBACKUP + /** Advances the smallest lsn for which there are unflushed dirty blocks in the buffer pool. NOTE: this function may only be called if the calling thread owns no @@ -1550,7 +1539,7 @@ log_preflush_pool_modified_pages( return(success); } -#endif /* !UNIV_HOTBACKUP */ + /******************************************************//** Completes a checkpoint. */ static @@ -1665,50 +1654,6 @@ log_group_checkpoint( ut_ad(((ulint) group & 0x1UL) == 0); } -#ifdef UNIV_HOTBACKUP -/******************************************************//** -Writes info to a buffer of a log group when log files are created in -backup restoration. */ -void -log_reset_first_header_and_checkpoint( -/*==================================*/ - byte* hdr_buf,/*!< in: buffer which will be written to the - start of the first log file */ - ib_uint64_t start) /*!< in: lsn of the start of the first log file; - we pretend that there is a checkpoint at - start + LOG_BLOCK_HDR_SIZE */ -{ - byte* buf; - ib_uint64_t lsn; - - mach_write_to_4(hdr_buf + LOG_HEADER_FORMAT, - LOG_HEADER_FORMAT_CURRENT); - mach_write_to_8(hdr_buf + LOG_HEADER_START_LSN, start); - - lsn = start + LOG_BLOCK_HDR_SIZE; - - /* Write the label of mysqlbackup --restore */ - strcpy((char*)hdr_buf + LOG_HEADER_CREATOR, LOG_HEADER_CREATOR_CURRENT); - ut_sprintf_timestamp((char*) hdr_buf - + (LOG_HEADER_CREATOR - + (sizeof LOG_HEADER_CREATOR_CURRENT) - 1)); - buf = hdr_buf + LOG_CHECKPOINT_1; - memset(buf, 0, OS_FILE_LOG_BLOCK_SIZE); - - /*mach_write_to_8(buf + LOG_CHECKPOINT_NO, 0);*/ - mach_write_to_8(buf + LOG_CHECKPOINT_LSN, lsn); - - log_crypt_write_checkpoint_buf(buf); - - mach_write_to_8(buf + LOG_CHECKPOINT_OFFSET, - LOG_FILE_HDR_SIZE + LOG_BLOCK_HDR_SIZE); - mach_write_to_8(buf + LOG_CHECKPOINT_LOG_BUF_SIZE, 2 * 1024 * 1024); - - log_block_set_checksum(buf, log_block_calc_checksum_crc32(buf)); -} -#endif /* UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP /** Read a log group header page to log_sys->checkpoint_buf. @param[in] group log group @param[in] header 0 or LOG_CHEKCPOINT_1 or LOG_CHECKPOINT2 */ @@ -2675,4 +2620,3 @@ DECLARE_THREAD(log_scrub_thread)( OS_THREAD_DUMMY_RETURN; } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 6901ba070aff3..1ecebe098bdaf 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -34,16 +34,11 @@ Created 9/20/1997 Heikki Tuuri #include "log0recv.h" -#ifdef UNIV_NONINL -#include "log0recv.ic" -#endif - #ifdef HAVE_MY_AES_H #include #endif #include "log0crypt.h" - #include "mem0mem.h" #include "buf0buf.h" #include "buf0flu.h" @@ -60,19 +55,11 @@ Created 9/20/1997 Heikki Tuuri #include "fsp0sysspace.h" #include "ut0new.h" #include "row0trunc.h" -#ifndef UNIV_HOTBACKUP -# include "buf0rea.h" -# include "srv0srv.h" -# include "srv0start.h" -# include "trx0roll.h" -# include "row0merge.h" -#else /* !UNIV_HOTBACKUP */ -/** This is set to false if the backup was originally taken with the -mysqlbackup --include regexp option: then we do not want to create tables in -directories which were not included */ -bool recv_replay_file_ops = true; -#include "fut0lst.h" -#endif /* !UNIV_HOTBACKUP */ +#include "buf0rea.h" +#include "srv0srv.h" +#include "srv0start.h" +#include "trx0roll.h" +#include "row0merge.h" /** Log records are stored in the hash table in chunks at most of this size; this must be less than UNIV_PAGE_SIZE as it is stored in the buffer pool */ @@ -88,18 +75,13 @@ otherwise. Note that this is FALSE while a background thread is rolling back incomplete transactions. */ volatile bool recv_recovery_on; -#ifndef UNIV_HOTBACKUP /** TRUE when recv_init_crash_recovery() has been called. */ bool recv_needed_recovery; -#else -# define recv_needed_recovery false -# define buf_pool_get_curr_size() (5 * 1024 * 1024) -#endif /* !UNIV_HOTBACKUP */ -# ifdef UNIV_DEBUG +#ifdef UNIV_DEBUG /** TRUE if writing to the redo log (mtr_commit) is forbidden. Protected by log_sys->mutex. */ bool recv_no_log_write = false; -# endif /* UNIV_DEBUG */ +#endif /* UNIV_DEBUG */ /** TRUE if buf_page_is_corrupted() should check if the log sequence number (FIL_PAGE_LSN) is in the future. Initially FALSE, and set by @@ -115,24 +97,7 @@ buffer pool before the pages have been recovered to the up-to-date state. TRUE means that recovery is running and no operations on the log files are allowed yet: the variable name is misleading. */ -#ifndef UNIV_HOTBACKUP bool recv_no_ibuf_operations; -/** TRUE when the redo log is being backed up */ -# define recv_is_making_a_backup false -/** TRUE when recovering from a backed up redo log file */ -# define recv_is_from_backup false -#else /* !UNIV_HOTBACKUP */ -/** true if the backup is an offline backup */ -volatile bool is_online_redo_copy = true; -/**true if the last flushed lsn read at the start of backup */ -volatile lsn_t backup_redo_log_flushed_lsn; - -/** TRUE when the redo log is being backed up */ -bool recv_is_making_a_backup = false; -/** TRUE when recovering from a backed up redo log file */ -bool recv_is_from_backup = false; -# define buf_pool_get_curr_size() (5 * 1024 * 1024) -#endif /* !UNIV_HOTBACKUP */ /** The following counter is used to decide when to print info on log scan */ static ulint recv_scan_print_counter; @@ -159,16 +124,11 @@ lsn_t recv_max_page_lsn; #ifdef UNIV_PFS_THREAD mysql_pfs_key_t trx_rollback_clean_thread_key; -#endif /* UNIV_PFS_THREAD */ - -#ifndef UNIV_HOTBACKUP -# ifdef UNIV_PFS_THREAD mysql_pfs_key_t recv_writer_thread_key; -# endif /* UNIV_PFS_THREAD */ +#endif /* UNIV_PFS_THREAD */ /** Flag indicating if recv_writer thread is active. */ volatile bool recv_writer_thread_active = false; -#endif /* !UNIV_HOTBACKUP */ #ifndef DBUG_OFF /** Return string name of the redo log record type. @@ -180,7 +140,6 @@ get_mlog_string(mlog_id_t type); /* prototypes */ -#ifndef UNIV_HOTBACKUP /*******************************************************//** Initialize crash recovery environment. Can be called iff recv_needed_recovery == false. */ @@ -188,7 +147,6 @@ static void recv_init_crash_recovery(void); /*===========================*/ -#endif /* !UNIV_HOTBACKUP */ /** Tablespace item during recovery */ struct file_name_t { @@ -344,7 +302,6 @@ fil_name_process( case FIL_LOAD_INVALID: ut_ad(space == NULL); if (srv_force_recovery == 0) { -#ifndef UNIV_HOTBACKUP ib::warn() << "We do not continue the crash" " recovery, because the table may" " become corrupt if we cannot apply" @@ -366,13 +323,6 @@ fil_name_process( " remove the .ibd file, you can set" " --innodb_force_recovery."; recv_sys->found_corrupt_fs = true; -#else - ib::warn() << "We do not continue the apply-log" - " operation because the tablespace may" - " become corrupt if we cannot apply" - " the log records in the redo log" - " records to it."; -#endif /* !UNIV_BACKUP */ processed = false; break; } @@ -387,7 +337,6 @@ fil_name_process( return(processed); } -#ifndef UNIV_HOTBACKUP /** Parse or process a MLOG_FILE_* record. @param[in] ptr redo log record @param[in] end end of the redo log buffer @@ -514,286 +463,6 @@ fil_name_parse( return(end_ptr); } -#else /* !UNIV_HOTBACKUP */ -/** Parse a file name retrieved from a MLOG_FILE_* record, -and return the absolute file path corresponds to backup dir -as well as in the form of database/tablespace -@param[in] file_name path emitted by the redo log -@param[out] absolute_path absolute path of tablespace -corresponds to backup dir -@param[out] tablespace_name name in the form of database/table */ -static -void -make_abs_file_path( - const std::string& name, - std::string& absolute_path, - std::string& tablespace_name) -{ - std::string file_name = name; - std::string path = fil_path_to_mysql_datadir; - size_t pos = std::string::npos; - - if (is_absolute_path(file_name.c_str())) { - - pos = file_name.rfind(OS_PATH_SEPARATOR); - std::string temp_name = file_name.substr(0, pos); - pos = temp_name.rfind(OS_PATH_SEPARATOR); - ++pos; - file_name = file_name.substr(pos, file_name.length()); - path += OS_PATH_SEPARATOR + file_name; - } else { - pos = file_name.find(OS_PATH_SEPARATOR); - ++pos; - file_name = file_name.substr(pos, file_name.length()); - path += OS_PATH_SEPARATOR + file_name; - } - - absolute_path = path; - - /* remove the .ibd extension */ - pos = file_name.rfind(".ibd"); - if (pos != std::string::npos) - tablespace_name = file_name.substr(0, pos); - - /* space->name uses '/', not OS_PATH_SEPARATOR, - update the seperator */ - if (OS_PATH_SEPARATOR != '/') { - pos = tablespace_name.find(OS_PATH_SEPARATOR); - while (pos != std::string::npos) { - tablespace_name[pos] = '/'; - pos = tablespace_name.find(OS_PATH_SEPARATOR); - } - } - -} - -/** Wrapper around fil_name_process() -@param[in] name absolute path of tablespace file -@param[in] space_id the tablespace ID -@retval true if able to process file successfully. -@retval false if unable to process the file */ -bool -fil_name_process( - const char* name, - ulint space_id) -{ - size_t length = strlen(name); - ++length; - - char* file_name = static_cast(ut_malloc_nokey(length)); - strncpy(file_name, name,length); - - bool processed = fil_name_process(file_name, length, space_id, false); - - ut_free(file_name); - return(processed); -} - -/** Parse or process a MLOG_FILE_* record. -@param[in] ptr redo log record -@param[in] end end of the redo log buffer -@param[in] space_id the tablespace ID -@param[in] first_page_no first page number in the file -@param[in] type MLOG_FILE_NAME or MLOG_FILE_DELETE -or MLOG_FILE_CREATE2 or MLOG_FILE_RENAME2 -@param[in] apply whether to apply the record -@retval pointer to next redo log record -@retval NULL if this log record was truncated */ -static -byte* -fil_name_parse( - byte* ptr, - const byte* end, - ulint space_id, - ulint first_page_no, - mlog_id_t type, - bool apply) -{ - - ulint flags = mach_read_from_4(ptr); - - if (type == MLOG_FILE_CREATE2) { - if (end < ptr + 4) { - return(NULL); - } - ptr += 4; - } - - if (end < ptr + 2) { - return(NULL); - } - - ulint len = mach_read_from_2(ptr); - ptr += 2; - if (end < ptr + len) { - return(NULL); - } - - os_normalize_path(reinterpret_cast(ptr)); - - /* MLOG_FILE_* records should only be written for - user-created tablespaces. The name must be long enough - and end in .ibd. */ - bool corrupt = is_predefined_tablespace(space_id) - || first_page_no != 0 // TODO: multi-file user tablespaces - || len < sizeof "/a.ibd\0" - || memcmp(ptr + len - 5, DOT_IBD, 5) != 0 - || memchr(ptr, OS_PATH_SEPARATOR, len) == NULL; - - byte* end_ptr = ptr + len; - - if (corrupt) { - recv_sys->found_corrupt_log = true; - return(end_ptr); - } - - std::string abs_file_path, tablespace_name; - char* name = reinterpret_cast(ptr); - char* new_name = NULL; - recv_spaces_t::iterator itr; - - make_abs_file_path(name, abs_file_path, tablespace_name); - - if (!recv_is_making_a_backup) { - - name = static_cast(ut_malloc_nokey( - (abs_file_path.length() + 1))); - strcpy(name, abs_file_path.c_str()); - len = strlen(name) + 1; - } - switch (type) { - default: - ut_ad(0); // the caller checked this - case MLOG_FILE_NAME: - /* Don't validate tablespaces while copying redo logs - because backup process might keep some tablespace handles - open in server datadir. - Maintain "map of dirty tablespaces" so that assumptions - for other redo log records are not broken even for dirty - tablespaces during apply log */ - if (!recv_is_making_a_backup) { - recv_spaces.insert(std::make_pair(space_id, - file_name_t(abs_file_path, - false))); - } - break; - case MLOG_FILE_DELETE: - /* Don't validate tablespaces while copying redo logs - because backup process might keep some tablespace handles - open in server datadir. */ - if (recv_is_making_a_backup) - break; - - fil_name_process( - name, len, space_id, true); - - if (apply && recv_replay_file_ops - && fil_space_get(space_id)) { - dberr_t err = fil_delete_tablespace( - space_id, BUF_REMOVE_FLUSH_NO_WRITE); - ut_a(err == DB_SUCCESS); - } - - break; - case MLOG_FILE_CREATE2: - if (recv_is_making_a_backup - || (!recv_replay_file_ops) - || (is_intermediate_file(abs_file_path.c_str())) - || (fil_space_get(space_id)) - || (fil_space_get_id_by_name( - tablespace_name.c_str()) != ULINT_UNDEFINED)) { - /* Don't create table while :- - 1. scanning the redo logs during backup - 2. apply-log on a partial backup - 3. if it is intermediate file - 4. tablespace is already loaded in memory */ - } else { - itr = recv_spaces.find(space_id); - if (itr == recv_spaces.end() - || (itr->second.name != abs_file_path)) { - - dberr_t ret = fil_ibd_create( - space_id, tablespace_name.c_str(), - abs_file_path.c_str(), - flags, FIL_IBD_FILE_INITIAL_SIZE); - - if (ret != DB_SUCCESS) { - ib::fatal() << "Could not create the" - << " tablespace : " - << abs_file_path - << " with space Id : " - << space_id; - } - } - } - break; - case MLOG_FILE_RENAME2: - /* The new name follows the old name. */ - byte* new_table_name = end_ptr + 2; - if (end < new_table_name) { - return(NULL); - } - - ulint new_len = mach_read_from_2(end_ptr); - - if (end < end_ptr + 2 + new_len) { - return(NULL); - } - - end_ptr += 2 + new_len; - - char* new_table = reinterpret_cast(new_table_name); - os_normalize_path(new_table); - - corrupt = corrupt - || new_len < sizeof "/a.ibd\0" - || memcmp(new_table_name + new_len - 5, DOT_IBD, 5) != 0 - || !memchr(new_table_name, OS_PATH_SEPARATOR, new_len); - - if (corrupt) { - recv_sys->found_corrupt_log = true; - break; - } - - if (recv_is_making_a_backup - || (!recv_replay_file_ops) - || (is_intermediate_file(name)) - || (is_intermediate_file(new_table))) { - /* Don't rename table while :- - 1. scanning the redo logs during backup - 2. apply-log on a partial backup - 3. The new name is already used. - 4. A tablespace is not open in memory with the old name. - This will prevent unintended renames during recovery. */ - break; - } else { - make_abs_file_path(new_table, abs_file_path, - tablespace_name); - - new_name = static_cast(ut_malloc_nokey( - (abs_file_path.length() + 1))); - strcpy(new_name, abs_file_path.c_str()); - new_len = strlen(new_name) + 1; - } - - fil_name_process(name, len, space_id, false); - fil_name_process( new_name, new_len, space_id, false); - - if (!fil_op_replay_rename( - space_id, first_page_no, - name, - new_name)) { - recv_sys->found_corrupt_fs = true; - } - } - - if (!recv_is_making_a_backup) { - ut_free(name); - ut_free(new_name); - } - return(end_ptr); -} -#endif /* UNIV_HOTBACKUP */ /********************************************************//** Creates the recovery system. */ @@ -829,7 +498,7 @@ recv_sys_close(void) if (recv_sys->heap != NULL) { mem_heap_free(recv_sys->heap); } -#ifndef UNIV_HOTBACKUP + if (recv_sys->flush_start != NULL) { os_event_destroy(recv_sys->flush_start); } @@ -837,14 +506,12 @@ recv_sys_close(void) if (recv_sys->flush_end != NULL) { os_event_destroy(recv_sys->flush_end); } -#endif /* !UNIV_HOTBACKUP */ + ut_free(recv_sys->buf); ut_free(recv_sys->last_block_buf_start); -#ifndef UNIV_HOTBACKUP ut_ad(!recv_writer_thread_active); mutex_free(&recv_sys->writer_mutex); -#endif /* !UNIV_HOTBACKUP */ mutex_free(&recv_sys->mutex); @@ -869,7 +536,7 @@ recv_sys_mem_free(void) if (recv_sys->heap != NULL) { mem_heap_free(recv_sys->heap); } -#ifndef UNIV_HOTBACKUP + if (recv_sys->flush_start != NULL) { os_event_destroy(recv_sys->flush_start); } @@ -877,7 +544,7 @@ recv_sys_mem_free(void) if (recv_sys->flush_end != NULL) { os_event_destroy(recv_sys->flush_end); } -#endif /* !UNIV_HOTBACKUP */ + ut_free(recv_sys->buf); ut_free(recv_sys->last_block_buf_start); ut_free(recv_sys); @@ -885,7 +552,6 @@ recv_sys_mem_free(void) } } -#ifndef UNIV_HOTBACKUP /************************************************************ Reset the state of the recovery system variables. */ void @@ -961,7 +627,6 @@ DECLARE_THREAD(recv_writer_thread)( OS_THREAD_DUMMY_RETURN; } -#endif /* !UNIV_HOTBACKUP */ /************************************************************ Inits the recovery system for a recovery operation. */ @@ -975,7 +640,6 @@ recv_sys_init( return; } -#ifndef UNIV_HOTBACKUP mutex_enter(&(recv_sys->mutex)); recv_sys->heap = mem_heap_create_typed(256, @@ -985,10 +649,6 @@ recv_sys_init( recv_sys->flush_start = os_event_create(0); recv_sys->flush_end = os_event_create(0); } -#else /* !UNIV_HOTBACKUP */ - recv_sys->heap = mem_heap_create(256); - recv_is_from_backup = true; -#endif /* !UNIV_HOTBACKUP */ /* Set appropriate value of recv_n_pool_free_frames. */ if (buf_pool_get_curr_size() >= (10 * 1024 * 1024)) { @@ -1046,8 +706,6 @@ recv_sys_empty_hash(void) recv_sys->addr_hash = hash_create(buf_pool_get_curr_size() / 512); } -#ifndef UNIV_HOTBACKUP - /********************************************************//** Frees the recovery system. */ void @@ -1145,7 +803,6 @@ recv_synchronize_groups(void) log_write_checkpoint_info(true); log_mutex_enter(); } -#endif /* !UNIV_HOTBACKUP */ /** Check the consistency of a log header block. @param[in] log header block @@ -1159,7 +816,6 @@ recv_check_log_header_checksum( == log_block_calc_checksum_crc32(buf)); } -#ifndef UNIV_HOTBACKUP /** Find the latest checkpoint in the format-0 log header. @param[out] max_group log group, or NULL @param[out] max_field LOG_CHECKPOINT_1 or LOG_CHECKPOINT_2 @@ -1421,58 +1077,6 @@ recv_find_max_checkpoint( return(DB_SUCCESS); } -#else /* !UNIV_HOTBACKUP */ -/*******************************************************************//** -Reads the checkpoint info needed in hot backup. -@return TRUE if success */ -ibool -recv_read_checkpoint_info_for_backup( -/*=================================*/ - const byte* hdr, /*!< in: buffer containing the log group - header */ - lsn_t* lsn, /*!< out: checkpoint lsn */ - lsn_t* offset, /*!< out: checkpoint offset in the log group */ - lsn_t* cp_no, /*!< out: checkpoint number */ - lsn_t* first_header_lsn) - /*!< out: lsn of of the start of the - first log file */ -{ - ulint max_cp = 0; - ib_uint64_t max_cp_no = 0; - const byte* cp_buf; - - cp_buf = hdr + LOG_CHECKPOINT_1; - - if (recv_check_log_header_checksum(cp_buf)) { - max_cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO); - max_cp = LOG_CHECKPOINT_1; - } - - cp_buf = hdr + LOG_CHECKPOINT_2; - - if (recv_check_log_header_checksum(cp_buf)) { - if (mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO) > max_cp_no) { - max_cp = LOG_CHECKPOINT_2; - } - } - - if (max_cp == 0) { - return(FALSE); - } - - cp_buf = hdr + max_cp; - - *lsn = mach_read_from_8(cp_buf + LOG_CHECKPOINT_LSN); - *offset = mach_read_from_8( - cp_buf + LOG_CHECKPOINT_OFFSET); - - *cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO); - - *first_header_lsn = mach_read_from_8(hdr + LOG_HEADER_START_LSN); - - return(TRUE); -} -#endif /* !UNIV_HOTBACKUP */ /** Check the 4-byte checksum to the trailer checksum field of a log block. @@ -1497,97 +1101,6 @@ log_block_checksum_is_ok( == log_block_calc_checksum(block)); } -#ifdef UNIV_HOTBACKUP -/*******************************************************************//** -Scans the log segment and n_bytes_scanned is set to the length of valid -log scanned. */ -void -recv_scan_log_seg_for_backup( -/*=========================*/ - byte* buf, /*!< in: buffer containing log data */ - ulint buf_len, /*!< in: data length in that buffer */ - lsn_t* scanned_lsn, /*!< in/out: lsn of buffer start, - we return scanned lsn */ - ulint* scanned_checkpoint_no, - /*!< in/out: 4 lowest bytes of the - highest scanned checkpoint number so - far */ - ulint* n_bytes_scanned)/*!< out: how much we were able to - scan, smaller than buf_len if log - data ended here */ -{ - ulint data_len; - byte* log_block; - ulint no; - - *n_bytes_scanned = 0; - - for (log_block = buf; log_block < buf + buf_len; - log_block += OS_FILE_LOG_BLOCK_SIZE) { - - no = log_block_get_hdr_no(log_block); - -#if 0 - fprintf(stderr, "Log block header no %lu\n", no); -#endif - - if (no != log_block_convert_lsn_to_no(*scanned_lsn) - || !log_block_checksum_is_ok(log_block)) { -#if 0 - fprintf(stderr, - "Log block n:o %lu, scanned lsn n:o %lu\n", - no, log_block_convert_lsn_to_no(*scanned_lsn)); -#endif - /* Garbage or an incompletely written log block */ - - log_block += OS_FILE_LOG_BLOCK_SIZE; -#if 0 - fprintf(stderr, - "Next log block n:o %lu\n", - log_block_get_hdr_no(log_block)); -#endif - break; - } - - if (*scanned_checkpoint_no > 0 - && log_block_get_checkpoint_no(log_block) - < *scanned_checkpoint_no - && *scanned_checkpoint_no - - log_block_get_checkpoint_no(log_block) - > 0x80000000UL) { - - /* Garbage from a log buffer flush which was made - before the most recent database recovery */ -#if 0 - fprintf(stderr, - "Scanned cp n:o %lu, block cp n:o %lu\n", - *scanned_checkpoint_no, - log_block_get_checkpoint_no(log_block)); -#endif - break; - } - - data_len = log_block_get_data_len(log_block); - - *scanned_checkpoint_no - = log_block_get_checkpoint_no(log_block); - *scanned_lsn += data_len; - - *n_bytes_scanned += data_len; - - if (data_len < OS_FILE_LOG_BLOCK_SIZE) { - /* Log data ends here */ - -#if 0 - fprintf(stderr, "Log block data len %lu\n", - data_len); -#endif - break; - } - } -} -#endif /* UNIV_HOTBACKUP */ - #ifdef MYSQL_ENCRYPTION /** Parse or process a write encryption info record. @@ -1725,9 +1238,7 @@ recv_parse_or_apply_log_rec_body( mtr_t* mtr) { ut_ad(!block == !mtr); -#ifndef UNIV_HOTBACKUP ut_ad(!apply || recv_sys->mlog_checkpoint_lsn != 0); -#endif /* !UNIV_HOTBACKUP */ switch (type) { case MLOG_FILE_NAME: @@ -1740,66 +1251,6 @@ recv_parse_or_apply_log_rec_body( return(fil_name_parse(ptr, end_ptr, space_id, page_no, type, apply)); case MLOG_INDEX_LOAD: -#ifdef UNIV_HOTBACKUP - /* While scaning redo logs during backup phase a - MLOG_INDEX_LOAD type redo log record indicates a DDL - (create index, alter table...)is performed with - 'algorithm=inplace'. This redo log indicates that - - 1. The DDL was started after MEB started backing up, in which - case MEB will not be able to take a consistent backup and should - fail. or - 2. There is a possibility of this record existing in the REDO - even after the completion of the index create operation. This is - because of InnoDB does not checkpointing after the flushing the - index pages. - - If MEB gets the last_redo_flush_lsn and that is less than the - lsn of the current record MEB fails the backup process. - Error out in case of online backup and emit a warning in case - of offline backup and continue. - */ - if (!recv_recovery_on) { - if (is_online_redo_copy) { - if (backup_redo_log_flushed_lsn - < recv_sys->recovered_lsn) { - ib::trace() << "Last flushed lsn: " - << backup_redo_log_flushed_lsn - << " load_index lsn " - << recv_sys->recovered_lsn; - - if (backup_redo_log_flushed_lsn == 0) - ib::error() << "MEB was not " - "able to determine the" - "InnoDB Engine Status"; - - ib::fatal() << "An optimized(without" - " redo logging) DDLoperation" - " has been performed. All" - " modified pages may not have" - " been flushed to the disk yet." - " \n MEB will not be able" - " take a consistent backup." - " Retry the backup operation"; - } - /** else the index is flushed to disk before - backup started hence no error */ - } else { - /* offline backup */ - ib::trace() << "Last flushed lsn: " - << backup_redo_log_flushed_lsn - << " load_index lsn " - << recv_sys->recovered_lsn; - - ib::warn() << "An optimized(without redo" - " logging) DDL operation has been" - " performed. All modified pages may not" - " have been flushed to the disk yet." - " \n This offline backup may not" - " be consistent"; - } - } -#endif /* UNIV_HOTBACKUP */ if (end_ptr < ptr + 8) { return(NULL); } @@ -2377,19 +1828,12 @@ recv_data_copy_to_buf( } } -/************************************************************************//** -Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. */ +/** Apply the hashed log records to the page, if the page lsn is less than the +lsn of a log record. +@param just_read_in whether the page recently arrived to the I/O handler +@param block the page in the buffer pool */ void -recv_recover_page_func( -/*===================*/ -#ifndef UNIV_HOTBACKUP - ibool just_read_in, - /*!< in: TRUE if the i/o handler calls - this for a freshly read page */ -#endif /* !UNIV_HOTBACKUP */ - buf_block_t* block) /*!< in/out: buffer block */ +recv_recover_page(bool just_read_in, buf_block_t* block) { page_t* page; page_zip_des_t* page_zip; @@ -2427,13 +1871,11 @@ recv_recover_page_func( return; } -#ifndef UNIV_HOTBACKUP ut_ad(recv_needed_recovery); DBUG_PRINT("ib_log", ("Applying log to page %u:%u", recv_addr->space, recv_addr->page_no)); -#endif /* !UNIV_HOTBACKUP */ recv_addr->state = RECV_BEING_PROCESSED; @@ -2445,7 +1887,6 @@ recv_recover_page_func( page = block->frame; page_zip = buf_block_get_page_zip(block); -#ifndef UNIV_HOTBACKUP if (just_read_in) { /* Move the ownership of the x-latch on the page to this OS thread, so that we can acquire a second @@ -2461,12 +1902,10 @@ recv_recover_page_func( ut_a(success); buf_block_dbg_add_level(block, SYNC_NO_ORDER_CHECK); -#endif /* !UNIV_HOTBACKUP */ /* Read the newest modification lsn from the page */ page_lsn = mach_read_from_8(page + FIL_PAGE_LSN); -#ifndef UNIV_HOTBACKUP /* It may be that the page has been modified in the buffer pool: read the newest modification lsn there */ @@ -2476,10 +1915,6 @@ recv_recover_page_func( page_lsn = page_newest_lsn; } -#else /* !UNIV_HOTBACKUP */ - /* In recovery from a backup we do not really use the buffer pool */ - page_newest_lsn = 0; -#endif /* !UNIV_HOTBACKUP */ modification_to_page = FALSE; start_lsn = end_lsn = 0; @@ -2592,7 +2027,6 @@ recv_recover_page_func( } #endif /* UNIV_ZIP_DEBUG */ -#ifndef UNIV_HOTBACKUP if (modification_to_page) { ut_a(block); @@ -2600,9 +2034,6 @@ recv_recover_page_func( buf_flush_recv_note_modification(block, start_lsn, end_lsn); log_flush_order_mutex_exit(); } -#else /* !UNIV_HOTBACKUP */ - start_lsn = start_lsn; /* Silence compiler */ -#endif /* !UNIV_HOTBACKUP */ /* Make sure that committing mtr does not change the modification lsn values of page */ @@ -2626,7 +2057,6 @@ recv_recover_page_func( } -#ifndef UNIV_HOTBACKUP /** Reads in pages which have hashed log records, from an area around a given page number. @param[in] page_id page id @@ -2864,174 +2294,6 @@ recv_apply_hashed_log_recs( return err; } -#else /* !UNIV_HOTBACKUP */ -/*******************************************************************//** -Applies log records in the hash table to a backup. */ -void -recv_apply_log_recs_for_backup(void) -/*================================*/ -{ - recv_addr_t* recv_addr; - ulint n_hash_cells; - buf_block_t* block; - bool success; - ulint error; - ulint i; - fil_space_t* space = NULL; - page_id_t page_id; - recv_sys->apply_log_recs = TRUE; - recv_sys->apply_batch_on = TRUE; - - block = back_block1; - - ib::info() << "Starting an apply batch of log records to the" - " database...\n"; - - fputs("InnoDB: Progress in percent: ", stderr); - - n_hash_cells = hash_get_n_cells(recv_sys->addr_hash); - - for (i = 0; i < n_hash_cells; i++) { - /* The address hash table is externally chained */ - recv_addr = static_cast(hash_get_nth_cell( - recv_sys->addr_hash, i)->node); - - while (recv_addr != NULL) { - - ib::trace() << "recv_addr {State: " << recv_addr->state - << ", Space id: " << recv_addr->space - << "Page no: " << recv_addr->page_no - << ". index i: " << i << "\n"; - - bool found; - const page_size_t& page_size - = fil_space_get_page_size(recv_addr->space, - &found); - - if (!found) { -#if 0 - fprintf(stderr, - "InnoDB: Warning: cannot apply" - " log record to" - " tablespace %lu page %lu,\n" - "InnoDB: because tablespace with" - " that id does not exist.\n", - recv_addr->space, recv_addr->page_no); -#endif - recv_addr->state = RECV_DISCARDED; - - ut_a(recv_sys->n_addrs); - recv_sys->n_addrs--; - - goto skip_this_recv_addr; - } - - /* We simulate a page read made by the buffer pool, to - make sure the recovery apparatus works ok. We must init - the block. */ - - buf_page_init_for_backup_restore( - page_id_t(recv_addr->space, recv_addr->page_no), - page_size, block); - - /* Extend the tablespace's last file if the page_no - does not fall inside its bounds; we assume the last - file is auto-extending, and mysqlbackup copied the file - when it still was smaller */ - fil_space_t* space - = fil_space_get(recv_addr->space); - - success = fil_space_extend( - space, recv_addr->page_no + 1); - if (!success) { - ib::fatal() << "Cannot extend tablespace " - << recv_addr->space << " to hold " - << recv_addr->page_no << " pages"; - } - - /* Read the page from the tablespace file using the - fil0fil.cc routines */ - - const page_id_t page_id(recv_addr->space, - recv_addr->page_no); - - if (page_size.is_compressed()) { - - error = fil_io( - IORequestRead, true, - page_id, - page_size, 0, page_size.physical(), - block->page.zip.data, NULL); - - if (error == DB_SUCCESS - && !buf_zip_decompress(block, TRUE)) { - ut_error; - } - } else { - - error = fil_io( - IORequestRead, true, - page_id, page_size, 0, - page_size.logical(), - block->frame, NULL); - } - - if (error != DB_SUCCESS) { - ib::fatal() << "Cannot read from tablespace " - << recv_addr->space << " page number " - << recv_addr->page_no; - } - - /* Apply the log records to this page */ - recv_recover_page(FALSE, block); - - /* Write the page back to the tablespace file using the - fil0fil.cc routines */ - - buf_flush_init_for_writing( - block, block->frame, - buf_block_get_page_zip(block), - mach_read_from_8(block->frame + FIL_PAGE_LSN), - fsp_is_checksum_disabled( - block->page.id.space())); - - if (page_size.is_compressed()) { - - error = fil_io( - IORequestWrite, true, page_id, - page_size, 0, page_size.physical(), - block->page.zip.data, NULL); - } else { - error = fil_io( - IORequestWrite, true, page_id, - page_size, 0, page_size.logical(), - block->frame, NULL); - } -skip_this_recv_addr: - recv_addr = static_cast(HASH_GET_NEXT( - addr_hash, recv_addr)); - } - - if ((100 * i) / n_hash_cells - != (100 * (i + 1)) / n_hash_cells) { - fprintf(stderr, "%lu ", - (ulint) ((100 * i) / n_hash_cells)); - fflush(stderr); - sd_notifyf(0, "STATUS=Applying batch of log records for" - " backup InnoDB: Progress %lu", - (ulint) (100 * i) / n_hash_cells); - } - } - - sd_notify(0, "STATUS=InnoDB: Apply batch for backup completed"); - - /* write logs in next line */ - fprintf(stderr, "\n"); - recv_sys->apply_log_recs = FALSE; - recv_sys->apply_batch_on = FALSE; - recv_sys_empty_hash(); -} -#endif /* !UNIV_HOTBACKUP */ /** Tries to parse a single log record. @param[out] type log record type @@ -3197,12 +2459,10 @@ recv_report_corrupt_log( - recv_previous_parsed_rec_offset); putc('\n', stderr); -#ifndef UNIV_HOTBACKUP if (!srv_force_recovery) { ib::info() << "Set innodb_force_recovery to ignore this error."; return(false); } -#endif /* !UNIV_HOTBACKUP */ ib::warn() << "The log file may have been corrupt and it is possible" " that the log scan did not proceed far enough in recovery!" @@ -3348,9 +2608,7 @@ recv_parse_log_recs( } recv_sys->mlog_checkpoint_lsn = recv_sys->recovered_lsn; -#ifndef UNIV_HOTBACKUP return(true); -#endif /* !UNIV_HOTBACKUP */ } break; case MLOG_FILE_NAME: @@ -3631,11 +2889,9 @@ recv_sys_justify_left_parsing_buf(void) recv_sys->recovered_offset = 0; } -/*******************************************************//** -Scans log from a buffer and stores new log data to the parsing buffer. -Parses and hashes the log records if new data found. Unless -UNIV_HOTBACKUP is defined, this function will apply log records -automatically when the hash table becomes full. +/** Scan redo log from a buffer and stores new log data to the parsing buffer. +Parse and hash the log records if new data found. +Apply log records automatically when the hash table becomes full. @return true if not able to scan any more in this log group */ static bool @@ -3766,7 +3022,6 @@ recv_scan_log_recs( of startup type, we must initiate crash recovery environment before parsing these log records. */ -#ifndef UNIV_HOTBACKUP if (!recv_needed_recovery) { if (!srv_read_only_mode) { @@ -3783,7 +3038,6 @@ recv_scan_log_recs( return(true); } } -#endif /* !UNIV_HOTBACKUP */ /* We were able to find more log data: add it to the parsing buffer if parse_start_lsn is already @@ -3796,7 +3050,6 @@ recv_scan_log_recs( recv_sys->found_corrupt_log = true; -#ifndef UNIV_HOTBACKUP if (!srv_force_recovery) { ib::error() << "Set innodb_force_recovery" @@ -3804,8 +3057,6 @@ recv_scan_log_recs( *err = DB_ERROR; return(true); } -#endif /* !UNIV_HOTBACKUP */ - } else if (!recv_sys->found_corrupt_log) { more_data = recv_sys_add_to_parsing_buf( log_block, scanned_lsn); @@ -3827,8 +3078,7 @@ recv_scan_log_recs( *group_scanned_lsn = scanned_lsn; - if (recv_needed_recovery - || (recv_is_from_backup && !recv_is_making_a_backup)) { + if (recv_needed_recovery) { recv_scan_print_counter++; if (finished || (recv_scan_print_counter % 80 == 0)) { @@ -3877,7 +3127,6 @@ recv_scan_log_recs( return(finished); } -#ifndef UNIV_HOTBACKUP /** Scans log from a buffer and stores new log data to the parsing buffer. Parses and hashes the log records if new data found. @param[in,out] group log group @@ -4129,7 +3378,6 @@ recv_recovery_from_checkpoint_start( ib_uint64_t checkpoint_no; lsn_t contiguous_lsn; byte* buf; - byte log_hdr_buf[LOG_FILE_HDR_SIZE]; dberr_t err = DB_SUCCESS; /* Initialize red-black tree for fast insertions into the @@ -4166,49 +3414,6 @@ recv_recovery_from_checkpoint_start( checkpoint_lsn = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); checkpoint_no = mach_read_from_8(buf + LOG_CHECKPOINT_NO); - /* Read the first log file header to print a note if this is - a recovery from a restored InnoDB Hot Backup */ - - const page_id_t page_id(max_cp_group->space_id, 0); - - fil_io(IORequestLogRead, true, page_id, univ_page_size, 0, - LOG_FILE_HDR_SIZE, log_hdr_buf, max_cp_group, NULL); - - if (0 == ut_memcmp(log_hdr_buf + LOG_HEADER_CREATOR, - (byte*)"ibbackup", (sizeof "ibbackup") - 1)) { - - if (srv_read_only_mode) { - log_mutex_exit(); - - ib::error() << "Cannot restore from mysqlbackup," - " InnoDB running in read-only mode!"; - - return(DB_ERROR); - } - - /* This log file was created by mysqlbackup --restore: print - a note to the user about it */ - - ib::info() << "The log file was created by mysqlbackup" - " --apply-log at " - << log_hdr_buf + LOG_HEADER_CREATOR - << ". The following crash recovery is part of a" - " normal restore."; - - /* Replace the label. */ - ut_ad(LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR - >= sizeof LOG_HEADER_CREATOR_CURRENT); - memset(log_hdr_buf + LOG_HEADER_CREATOR, 0, - LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR); - strcpy(reinterpret_cast(log_hdr_buf) - + LOG_HEADER_CREATOR, LOG_HEADER_CREATOR_CURRENT); - - /* Write to the log file to wipe over the label */ - fil_io(IORequestLogWrite, true, page_id, - univ_page_size, 0, OS_FILE_LOG_BLOCK_SIZE, log_hdr_buf, - max_cp_group, NULL); - } - /* Start reading the log groups from the checkpoint lsn up. The variable contiguous_lsn contains an lsn up to which the log is known to be contiguously written to all log groups. */ @@ -4578,100 +3783,6 @@ recv_reset_logs( log_mutex_enter(); } -#endif /* !UNIV_HOTBACKUP */ - -#ifdef UNIV_HOTBACKUP -/******************************************************//** -Creates new log files after a backup has been restored. */ -void -recv_reset_log_files_for_backup( -/*============================*/ - const char* log_dir, /*!< in: log file directory path */ - ulint n_log_files, /*!< in: number of log files */ - lsn_t log_file_size, /*!< in: log file size */ - lsn_t lsn) /*!< in: new start lsn, must be - divisible by OS_FILE_LOG_BLOCK_SIZE */ -{ - os_file_t log_file; - bool success; - byte* buf; - ulint i; - ulint log_dir_len; - char name[5000]; - static const char ib_logfile_basename[] = "ib_logfile"; - - log_dir_len = strlen(log_dir); - /* full path name of ib_logfile consists of log dir path + basename - + number. This must fit in the name buffer. - */ - ut_a(log_dir_len + strlen(ib_logfile_basename) + 11 < sizeof(name)); - - buf = (byte*)ut_zalloc_nokey(LOG_FILE_HDR_SIZE + - OS_FILE_LOG_BLOCK_SIZE); - - for (i = 0; i < n_log_files; i++) { - - sprintf(name, "%s%s%lu", log_dir, - ib_logfile_basename, (ulint) i); - - log_file = os_file_create_simple(innodb_log_file_key, - name, OS_FILE_CREATE, - OS_FILE_READ_WRITE, - srv_read_only_mode, &success); - if (!success) { - ib::fatal() << "Cannot create " << name << ". Check that" - " the file does not exist yet."; - } - - ib::info() << "Setting log file size to " << log_file_size; - - success = os_file_set_size( - name, log_file, log_file_size, srv_read_only_mode); - - if (!success) { - ib::fatal() << "Cannot set " << name << " size to " - << (long long unsigned)log_file_size; - } - - os_file_flush(log_file); - os_file_close(log_file); - } - - /* We pretend there is a checkpoint at lsn + LOG_BLOCK_HDR_SIZE */ - - log_reset_first_header_and_checkpoint(buf, lsn); - - log_block_init(buf + LOG_FILE_HDR_SIZE, lsn); - log_block_set_first_rec_group(buf + LOG_FILE_HDR_SIZE, - LOG_BLOCK_HDR_SIZE); - log_block_set_checksum(buf + LOG_FILE_HDR_SIZE, - log_block_calc_checksum_crc32(buf + LOG_FILE_HDR_SIZE)); - - log_block_set_checksum(buf, log_block_calc_checksum_crc32(buf)); - sprintf(name, "%s%s%lu", log_dir, ib_logfile_basename, (ulint)0); - - log_file = os_file_create_simple(innodb_log_file_key, - name, OS_FILE_OPEN, - OS_FILE_READ_WRITE, - srv_read_only_mode, &success); - if (!success) { - ib::fatal() << "Cannot open " << name << "."; - } - - IORequest request(IORequest::WRITE); - - dberr_t err = os_file_write( - request, name, log_file, buf, 0, - LOG_FILE_HDR_SIZE + OS_FILE_LOG_BLOCK_SIZE); - - ut_a(err == DB_SUCCESS); - - os_file_flush(log_file); - os_file_close(log_file); - - ut_free(buf); -} -#endif /* UNIV_HOTBACKUP */ /** Find a doublewrite copy of a page. @param[in] space_id tablespace identifier diff --git a/storage/innobase/mem/mem0mem.cc b/storage/innobase/mem/mem0mem.cc index 41292ccf842c8..ff79382189515 100644 --- a/storage/innobase/mem/mem0mem.cc +++ b/storage/innobase/mem/mem0mem.cc @@ -277,9 +277,7 @@ mem_heap_create_block_func( ulint type) /*!< in: type of heap: MEM_HEAP_DYNAMIC or MEM_HEAP_BUFFER */ { -#ifndef UNIV_HOTBACKUP buf_block_t* buf_block = NULL; -#endif /* !UNIV_HOTBACKUP */ mem_block_t* block; ulint len; @@ -294,7 +292,6 @@ mem_heap_create_block_func( /* In dynamic allocation, calculate the size: block header + data. */ len = MEM_BLOCK_HEADER_SIZE + MEM_SPACE_NEEDED(n); -#ifndef UNIV_HOTBACKUP if (type == MEM_HEAP_DYNAMIC || len < UNIV_PAGE_SIZE / 2) { ut_ad(type == MEM_HEAP_DYNAMIC || n <= MEM_MAX_ALLOC_IN_BUF); @@ -329,11 +326,6 @@ mem_heap_create_block_func( block->buf_block = buf_block; block->free_block = NULL; -#else /* !UNIV_HOTBACKUP */ - len = MEM_BLOCK_HEADER_SIZE + MEM_SPACE_NEEDED(n); - block = ut_malloc_nokey(len); - ut_ad(block); -#endif /* !UNIV_HOTBACKUP */ block->magic_n = MEM_BLOCK_MAGIC_N; ut_d(ut_strlcpy_rev(block->file_name, file_name, @@ -428,11 +420,9 @@ mem_heap_block_free( { ulint type; ulint len; -#ifndef UNIV_HOTBACKUP buf_block_t* buf_block; buf_block = static_cast(block->buf_block); -#endif /* !UNIV_HOTBACKUP */ mem_block_validate(block); @@ -447,7 +437,6 @@ mem_heap_block_free( UNIV_MEM_ASSERT_W(block, len); -#ifndef UNIV_HOTBACKUP if (type == MEM_HEAP_DYNAMIC || len < UNIV_PAGE_SIZE / 2) { ut_ad(!buf_block); @@ -457,12 +446,8 @@ mem_heap_block_free( buf_block_free(buf_block); } -#else /* !UNIV_HOTBACKUP */ - ut_free(block); -#endif /* !UNIV_HOTBACKUP */ } -#ifndef UNIV_HOTBACKUP /******************************************************************//** Frees the free_block field from a memory heap. */ void @@ -477,4 +462,3 @@ mem_heap_free_block_free( heap->free_block = NULL; } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/mtr/mtr0log.cc b/storage/innobase/mtr/mtr0log.cc index d653b3851c3ab..a63ad40a3b0ea 100644 --- a/storage/innobase/mtr/mtr0log.cc +++ b/storage/innobase/mtr/mtr0log.cc @@ -34,9 +34,7 @@ Created 12/7/1995 Heikki Tuuri #include "log0recv.h" #include "page0page.h" #include "buf0dblwr.h" - -#ifndef UNIV_HOTBACKUP -# include "dict0boot.h" +#include "dict0boot.h" /********************************************************//** Catenates n bytes to the mtr log. */ @@ -85,7 +83,6 @@ mlog_write_initial_log_record( mlog_close(mtr, log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses an initial log record written by mlog_write_initial_log_record. @@ -307,7 +304,6 @@ mlog_write_ull( } } -#ifndef UNIV_HOTBACKUP /********************************************************//** Writes a string to a file page buffered in the buffer pool. Writes the corresponding log record to the mini-transaction log. */ @@ -362,7 +358,6 @@ mlog_log_string( mlog_catenate_string(mtr, ptr, len); } -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses a log record written by mlog_write_string. @@ -414,7 +409,6 @@ mlog_parse_string( return(ptr + len); } -#ifndef UNIV_HOTBACKUP /********************************************************//** Opens a buffer for mlog, writes the initial log record and, if needed, the field lengths of an index. @@ -534,7 +528,6 @@ mlog_open_and_write_index( } return(log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /********************************************************//** Parses a log record written by mlog_open_and_write_index. diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 8e9139026d0b2..5755707d710ef 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -54,17 +54,8 @@ Created 10/21/1995 Heikki Tuuri #ifdef HAVE_LINUX_UNISTD_H #include "unistd.h" #endif -#ifndef UNIV_HOTBACKUP -# include "os0event.h" -# include "os0thread.h" -#else /* !UNIV_HOTBACKUP */ -# ifdef _WIN32 -/* Add includes for the _stat() call to compile on Windows */ -# include -# include -# include -# endif /* _WIN32 */ -#endif /* !UNIV_HOTBACKUP */ +#include "os0event.h" +#include "os0thread.h" #include @@ -168,8 +159,6 @@ static DWORD fls_sync_io = FLS_OUT_OF_INDEXES; #define IOCP_SHUTDOWN_KEY (ULONG_PTR)-1 #endif /* _WIN32 */ -#ifndef UNIV_HOTBACKUP - /** In simulated aio, merge at most this many consecutive i/os */ static const ulint OS_AIO_MERGE_N_CONSECUTIVE = 64; @@ -749,7 +738,6 @@ static ulint os_aio_n_segments = ULINT_UNDEFINED; /** If the following is true, read i/o handler threads try to wait until a batch of new read requests have been posted */ static bool os_aio_recommend_sleep_for_read_threads = false; -#endif /* !UNIV_HOTBACKUP */ ulint os_n_file_reads = 0; ulint os_bytes_read_since_printout = 0; @@ -1453,7 +1441,6 @@ os_file_compress_page( #endif /* MYSQL_COMPRESSION */ #ifdef UNIV_DEBUG -# ifndef UNIV_HOTBACKUP /** Validates the consistency the aio system some of the time. @return true if ok or the check was skipped */ bool @@ -1479,16 +1466,12 @@ os_aio_validate_skip() os_aio_validate_count = OS_AIO_VALIDATE_SKIP; return(os_aio_validate()); } -# endif /* !UNIV_HOTBACKUP */ #endif /* UNIV_DEBUG */ #undef USE_FILE_LOCK -#define USE_FILE_LOCK -#if defined(UNIV_HOTBACKUP) || defined(_WIN32) -/* InnoDB Hot Backup does not lock the data files. - * On Windows, mandatory locking is used. - */ -# undef USE_FILE_LOCK +#ifndef _WIN32 +/* On Windows, mandatory locking is used */ +# define USE_FILE_LOCK #endif #ifdef USE_FILE_LOCK /** Obtain an exclusive lock on a file. @@ -1528,8 +1511,6 @@ os_file_lock( } #endif /* USE_FILE_LOCK */ -#ifndef UNIV_HOTBACKUP - /** Calculates local segment number and aio array from global segment number. @param[out] array aio wait array @param[in] segment global segment number @@ -1759,8 +1740,6 @@ os_file_io_complete( return(DB_SUCCESS); } -#endif /* !UNIV_HOTBACKUP */ - /** This function returns a new path name after replacing the basename in an old path with a new basename. The old_path is a full path name including the extension. The tablename is in the normal @@ -4067,18 +4046,6 @@ os_file_set_eof( return(!ftruncate(fileno(file), ftell(file))); } -#ifdef UNIV_HOTBACKUP -/** Closes a file handle. -@param[in] file Handle to a file -@return true if success */ -bool -os_file_close_no_error_handling( - os_file_t file) -{ - return(close(file) != -1); -} -#endif /* UNIV_HOTBACKUP */ - /** This function can be called if one wants to post a batch of reads and prefers an i/o-handler thread to handle them all at once later. You must call os_aio_simulated_wake_handler_threads later to ensure the threads @@ -4875,9 +4842,6 @@ os_file_create_func( DWORD attributes = 0; -#ifdef UNIV_HOTBACKUP - attributes |= FILE_FLAG_NO_BUFFERING; -#else if (purpose == OS_FILE_AIO) { #ifdef WIN_ASYNC_IO @@ -4917,7 +4881,6 @@ os_file_create_func( } #endif /* UNIV_NON_BUFFERED_IO */ -#endif /* UNIV_HOTBACKUP */ DWORD access = GENERIC_READ; if (!read_only) { @@ -5084,8 +5047,9 @@ os_file_delete_if_exists_func( } for (;;) { - /* In Windows, deleting an .ibd file may fail if ibbackup - is copying it */ + /* In Windows, deleting an .ibd file may fail if + the file is being accessed by an external program, + such as a backup tool. */ bool ret = DeleteFile((LPCTSTR) name); @@ -5136,8 +5100,9 @@ os_file_delete_func( ulint count = 0; for (;;) { - /* In Windows, deleting an .ibd file may fail if ibbackup - is copying it */ + /* In Windows, deleting an .ibd file may fail if + the file is being accessed by an external program, + such as a backup tool. */ BOOL ret = DeleteFile((LPCTSTR) name); @@ -5160,8 +5125,8 @@ os_file_delete_func( os_file_get_last_error(true); ib::warn() - << "Cannot delete file '" << name << "'. Are " - << "you running ibbackup to back up the file?"; + << "Cannot delete file '" << name << "'. Is " + << "another program accessing it?"; } /* sleep for a second */ @@ -5470,18 +5435,6 @@ os_file_set_eof( return(SetEndOfFile(h)); } -#ifdef UNIV_HOTBACKUP -/** Closes a file handle. -@param[in] file Handle to close -@return true if success */ -bool -os_file_close_no_error_handling( - os_file_t file) -{ - return(CloseHandle(file) ? true : false); -} -#endif /* UNIV_HOTBACKUP */ - /** This function can be called if one wants to post a batch of reads and prefers an i/o-handler thread to handle them all at once later. You must call os_aio_simulated_wake_handler_threads later to ensure the threads @@ -6136,21 +6089,8 @@ os_file_set_size( dberr_t err; IORequest request(IORequest::WRITE); -#ifdef UNIV_HOTBACKUP - err = os_file_write( request, name, file, buf, current_size, n_bytes); -#else - /* Using OS_AIO_SYNC mode on POSIX systems will result in - fall back to os_file_write/read. On Windows it will use - special mechanism to wait before it returns back. */ - - err = os_aio( - request, - OS_AIO_SYNC, name, - file, buf, current_size, n_bytes, - read_only, NULL, NULL, NULL); -#endif /* UNIV_HOTBACKUP */ if (err != DB_SUCCESS) { diff --git a/storage/innobase/os/os0thread.cc b/storage/innobase/os/os0thread.cc index 0ee789df7b923..fd88fafa69d65 100644 --- a/storage/innobase/os/os0thread.cc +++ b/storage/innobase/os/os0thread.cc @@ -32,10 +32,8 @@ Created 9/8/1995 Heikki Tuuri #include "os0thread.ic" #endif -#ifndef UNIV_HOTBACKUP #include "srv0srv.h" #include "os0event.h" - #include /** Mutex that tracks the thread count. Used by innorwlocktest.cc @@ -208,7 +206,6 @@ os_thread_yield(void) sched_yield(); #endif } -#endif /* !UNIV_HOTBACKUP */ /*****************************************************************//** The thread sleeps at least the time given in microseconds. */ diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 87c2887246242..b32e8deb6e6dc 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -35,7 +35,6 @@ Created 10/4/1994 Heikki Tuuri #include "btr0btr.h" #include "mtr0log.h" #include "log0recv.h" -#ifndef UNIV_HOTBACKUP #include "rem0cmp.h" #include "gis0rtree.h" @@ -1027,9 +1026,6 @@ page_cur_insert_rec_write_log( mlog_catenate_string(mtr, ins_ptr, rec_size); } } -#else /* !UNIV_HOTBACKUP */ -# define page_cur_insert_rec_write_log(ins_rec,size,cur,index,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a log record of a record insert on a page. @@ -1919,7 +1915,6 @@ page_cur_insert_rec_zip( return(insert_rec); } -#ifndef UNIV_HOTBACKUP /**********************************************************//** Writes a log record of copying a record list end to a new created page. @return 4-byte field where to write the log data length, or NULL if @@ -1947,7 +1942,6 @@ page_copy_rec_list_to_created_page_write_log( return(log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************//** Parses a log record of copying a record list end to a new created page. @@ -2007,7 +2001,6 @@ page_parse_copy_rec_list_to_created_page( return(rec_end); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Copies records from page to a newly created page, from a given record onward, including that record. Infimum and supremum records are not copied. @@ -2224,9 +2217,6 @@ page_cur_delete_rec_write_log( mlog_close(mtr, log_ptr + 2); } -#else /* !UNIV_HOTBACKUP */ -# define page_cur_delete_rec_write_log(rec,index,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses log record of a record delete on a page. diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index 9bcc28f598070..5f330b9e0711f 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -34,12 +34,10 @@ Created 2/2/1994 Heikki Tuuri #include "buf0buf.h" #include "btr0btr.h" #include "row0trunc.h" -#ifndef UNIV_HOTBACKUP -# include "srv0srv.h" -# include "lock0lock.h" -# include "fut0lst.h" -# include "btr0sea.h" -#endif /* !UNIV_HOTBACKUP */ +#include "srv0srv.h" +#include "lock0lock.h" +#include "fut0lst.h" +#include "btr0sea.h" /* THE INDEX PAGE ============== @@ -205,9 +203,7 @@ page_set_max_trx_id( mtr_t* mtr) /*!< in/out: mini-transaction, or NULL */ { page_t* page = buf_block_get_frame(block); -#ifndef UNIV_HOTBACKUP ut_ad(!mtr || mtr_memo_contains(mtr, block, MTR_MEMO_PAGE_X_FIX)); -#endif /* !UNIV_HOTBACKUP */ /* It is not necessary to write this change to the redo log, as during a database recovery we assume that the max trx id of every @@ -218,11 +214,9 @@ page_set_max_trx_id( page_zip_write_header(page_zip, page + (PAGE_HEADER + PAGE_MAX_TRX_ID), 8, mtr); -#ifndef UNIV_HOTBACKUP } else if (mtr) { mlog_write_ull(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), trx_id, mtr); -#endif /* !UNIV_HOTBACKUP */ } else { mach_write_to_8(page + (PAGE_HEADER + PAGE_MAX_TRX_ID), trx_id); } @@ -299,7 +293,6 @@ page_mem_alloc_heap( return(NULL); } -#ifndef UNIV_HOTBACKUP /**********************************************************//** Writes a log record of page creation. */ UNIV_INLINE @@ -323,9 +316,6 @@ page_create_write_log( mlog_write_initial_log_record(frame, type, mtr); } -#else /* !UNIV_HOTBACKUP */ -# define page_create_write_log(frame,mtr,comp,is_rtree) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /** The page infimum and supremum of an empty page in ROW_FORMAT=REDUNDANT */ static const byte infimum_supremum_redundant[] = { @@ -638,7 +628,6 @@ page_copy_rec_list_end_no_locks( } } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Copies records from page to new_page, from a given record onward, including that record. Infimum and supremum records are not copied. @@ -978,9 +967,6 @@ page_delete_rec_list_write_log( mlog_close(mtr, log_ptr + 2); } } -#else /* !UNIV_HOTBACKUP */ -# define page_delete_rec_list_write_log(rec,index,type,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /**********************************************************//** Parses a log record of a record list end or start deletion. @@ -1324,7 +1310,6 @@ page_delete_rec_list_start( mtr_set_log_mode(mtr, log_mode); } -#ifndef UNIV_HOTBACKUP /*************************************************************//** Moves record list end to another page. Moved records include split_rec. @@ -1415,7 +1400,6 @@ page_move_rec_list_start( return(TRUE); } -#endif /* !UNIV_HOTBACKUP */ /**************************************************************//** Used to delete n slots from the directory. This function updates @@ -1741,7 +1725,6 @@ page_rec_get_n_recs_before( return((ulint) n); } -#ifndef UNIV_HOTBACKUP /************************************************************//** Prints record contents including the data relevant only in the index page context. */ @@ -1767,7 +1750,7 @@ page_rec_print( rec_validate(rec, offsets); } -# ifdef UNIV_BTR_PRINT +#ifdef UNIV_BTR_PRINT /***************************************************************//** This is used to print the contents of the directory for debugging purposes. */ @@ -1924,8 +1907,7 @@ page_print( page_dir_print(page, dn); page_print_list(block, index, rn); } -# endif /* UNIV_BTR_PRINT */ -#endif /* !UNIV_HOTBACKUP */ +#endif /* UNIV_BTR_PRINT */ /***************************************************************//** The following is used to validate a record on a page. This function @@ -1972,7 +1954,6 @@ page_rec_validate( return(TRUE); } -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG /***************************************************************//** Checks that the first directory slot points to the infimum record and @@ -2005,7 +1986,6 @@ page_check_dir( } } #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** This function checks the consistency of an index page when we do not @@ -2504,7 +2484,6 @@ page_validate( goto func_exit; } -#ifndef UNIV_HOTBACKUP /* Check that the records are in the ascending order */ if (count >= PAGE_HEAP_NO_USER_LOW && !page_rec_is_supremum(rec)) { @@ -2547,7 +2526,6 @@ page_validate( goto func_exit; } } -#endif /* !UNIV_HOTBACKUP */ if (page_rec_is_user_rec(rec)) { @@ -2711,7 +2689,6 @@ page_validate( return(ret); } -#ifndef UNIV_HOTBACKUP /***************************************************************//** Looks in the page record list for a record with the given heap number. @return record, NULL if not found */ @@ -2757,7 +2734,6 @@ page_find_rec_with_heap_no( } } } -#endif /* !UNIV_HOTBACKUP */ /*******************************************************//** Removes the record from a leaf page. This function does not log diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 798aeeed74f0a..c47a1905b4bd5 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -49,32 +49,24 @@ const byte field_ref_zero[FIELD_REF_SIZE] = { #include "log0recv.h" #include "row0trunc.h" #include "zlib.h" -#ifndef UNIV_HOTBACKUP -# include "buf0buf.h" +#include "buf0buf.h" #include "buf0types.h" #include "buf0checksum.h" -# include "btr0sea.h" -# include "dict0boot.h" -# include "lock0lock.h" -# include "srv0srv.h" -# include "buf0lru.h" -# include "srv0mon.h" -# include "ut0crc32.h" -#else /* !UNIV_HOTBACKUP */ -# include "buf0checksum.h" -# define lock_move_reorganize_page(block, temp_block) ((void) 0) -# define buf_LRU_stat_inc_unzip() ((void) 0) -#endif /* !UNIV_HOTBACKUP */ +#include "btr0sea.h" +#include "dict0boot.h" +#include "lock0lock.h" +#include "srv0srv.h" +#include "buf0lru.h" +#include "srv0mon.h" +#include "ut0crc32.h" #include #include -#ifndef UNIV_HOTBACKUP /** Statistics on compression, indexed by page_zip_des_t::ssize - 1 */ page_zip_stat_t page_zip_stat[PAGE_ZIP_SSIZE_MAX]; /** Statistics on compression, indexed by index->id */ page_zip_stat_per_index_t page_zip_stat_per_index; -#endif /* !UNIV_HOTBACKUP */ /* Compression level to be used by zlib. Settable by user. */ uint page_zip_level = DEFAULT_COMPRESSION_LEVEL; @@ -156,7 +148,6 @@ page_zip_fail_func( # define page_zip_fail(fmt_args) /* empty */ #endif /* UNIV_DEBUG || UNIV_ZIP_DEBUG */ -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Determine the guaranteed free space on an empty page. @return minimum payload size on the page */ @@ -228,7 +219,6 @@ page_zip_is_too_big( return(false); } -#endif /* !UNIV_HOTBACKUP */ /*************************************************************//** Gets the number of elements in the dense page directory, @@ -384,7 +374,6 @@ page_zip_dir_get( - PAGE_ZIP_DIR_SLOT_SIZE * (slot + 1))); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Write a log record of compressing an index page. */ static @@ -448,7 +437,6 @@ page_zip_compress_write_log( mlog_catenate_string(mtr, page_zip->data + page_zip_get_size(page_zip) - trailer_size, trailer_size); } -#endif /* !UNIV_HOTBACKUP */ /******************************************************//** Determine how many externally stored columns are contained @@ -1290,9 +1278,7 @@ page_zip_compress( byte* storage; /* storage of uncompressed columns */ index_id_t ind_id; -#ifndef UNIV_HOTBACKUP uintmax_t usec = ut_time_us(NULL); -#endif /* !UNIV_HOTBACKUP */ #ifdef PAGE_ZIP_COMPRESS_DBG FILE* logfile = NULL; #endif @@ -1375,14 +1361,12 @@ page_zip_compress( } } #endif /* PAGE_ZIP_COMPRESS_DBG */ -#ifndef UNIV_HOTBACKUP page_zip_stat[page_zip->ssize - 1].compressed++; if (cmp_per_index_enabled) { mutex_enter(&page_zip_stat_per_index_mutex); page_zip_stat_per_index[ind_id].compressed++; mutex_exit(&page_zip_stat_per_index_mutex); } -#endif /* !UNIV_HOTBACKUP */ if (UNIV_UNLIKELY(n_dense * PAGE_ZIP_DIR_SLOT_SIZE >= page_zip_get_size(page_zip))) { @@ -1578,7 +1562,6 @@ page_zip_compress( fclose(logfile); } #endif /* PAGE_ZIP_COMPRESS_DBG */ -#ifndef UNIV_HOTBACKUP if (page_is_leaf(page) && index) { dict_index_zip_failure(index); } @@ -1592,7 +1575,6 @@ page_zip_compress( += time_diff; mutex_exit(&page_zip_stat_per_index_mutex); } -#endif /* !UNIV_HOTBACKUP */ return(FALSE); } @@ -1633,9 +1615,7 @@ page_zip_compress( #endif /* UNIV_ZIP_DEBUG */ if (mtr) { -#ifndef UNIV_HOTBACKUP page_zip_compress_write_log(page_zip, page, index, mtr); -#endif /* !UNIV_HOTBACKUP */ } UNIV_MEM_ASSERT_RW(page_zip->data, page_zip_get_size(page_zip)); @@ -1652,7 +1632,6 @@ page_zip_compress( fclose(logfile); } #endif /* PAGE_ZIP_COMPRESS_DBG */ -#ifndef UNIV_HOTBACKUP uintmax_t time_diff = ut_time_us(NULL) - usec; page_zip_stat[page_zip->ssize - 1].compressed_ok++; page_zip_stat[page_zip->ssize - 1].compressed_usec += time_diff; @@ -1666,7 +1645,6 @@ page_zip_compress( if (page_is_leaf(page) && !truncate_t::s_fix_up_active) { dict_index_zip_success(index); } -#endif /* !UNIV_HOTBACKUP */ return(TRUE); } @@ -3277,15 +3255,12 @@ page_zip_decompress( page header fields that should not change after page creation */ { -#ifndef UNIV_HOTBACKUP uintmax_t usec = ut_time_us(NULL); -#endif /* !UNIV_HOTBACKUP */ if (!page_zip_decompress_low(page_zip, page, all)) { return(FALSE); } -#ifndef UNIV_HOTBACKUP uintmax_t time_diff = ut_time_us(NULL) - usec; page_zip_stat[page_zip->ssize - 1].decompressed++; page_zip_stat[page_zip->ssize - 1].decompressed_usec += time_diff; @@ -3298,7 +3273,6 @@ page_zip_decompress( page_zip_stat_per_index[index_id].decompressed_usec += time_diff; mutex_exit(&page_zip_stat_per_index_mutex); } -#endif /* !UNIV_HOTBACKUP */ /* Update the stat counter for LRU policy. */ buf_LRU_stat_inc_unzip(); @@ -4013,7 +3987,6 @@ page_zip_write_blob_ptr( #endif /* UNIV_ZIP_DEBUG */ if (mtr) { -#ifndef UNIV_HOTBACKUP byte* log_ptr = mlog_open( mtr, 11 + 2 + 2 + BTR_EXTERN_FIELD_REF_SIZE); if (UNIV_UNLIKELY(!log_ptr)) { @@ -4029,7 +4002,6 @@ page_zip_write_blob_ptr( memcpy(log_ptr, externs, BTR_EXTERN_FIELD_REF_SIZE); log_ptr += BTR_EXTERN_FIELD_REF_SIZE; mlog_close(mtr, log_ptr); -#endif /* !UNIV_HOTBACKUP */ } } @@ -4153,7 +4125,6 @@ page_zip_write_node_ptr( memcpy(storage, field, REC_NODE_PTR_SIZE); if (mtr) { -#ifndef UNIV_HOTBACKUP byte* log_ptr = mlog_open(mtr, 11 + 2 + 2 + REC_NODE_PTR_SIZE); if (UNIV_UNLIKELY(!log_ptr)) { @@ -4169,7 +4140,6 @@ page_zip_write_node_ptr( memcpy(log_ptr, field, REC_NODE_PTR_SIZE); log_ptr += REC_NODE_PTR_SIZE; mlog_close(mtr, log_ptr); -#endif /* !UNIV_HOTBACKUP */ } } @@ -4654,7 +4624,6 @@ page_zip_parse_write_header( return(ptr + len); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Write a log record of writing to the uncompressed header portion of a page. */ void @@ -4688,7 +4657,6 @@ page_zip_write_header_log( mlog_catenate_string(mtr, data, length); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Reorganize and compress a page. This is a low-level operation for @@ -4711,9 +4679,7 @@ page_zip_reorganize( dict_index_t* index, /*!< in: index of the B-tree node */ mtr_t* mtr) /*!< in: mini-transaction */ { -#ifndef UNIV_HOTBACKUP buf_pool_t* buf_pool = buf_pool_from_block(block); -#endif /* !UNIV_HOTBACKUP */ page_zip_des_t* page_zip = buf_block_get_page_zip(block); page_t* page = buf_block_get_frame(block); buf_block_t* temp_block; @@ -4729,13 +4695,8 @@ page_zip_reorganize( /* Disable logging */ mtr_log_t log_mode = mtr_set_log_mode(mtr, MTR_LOG_NONE); -#ifndef UNIV_HOTBACKUP temp_block = buf_block_alloc(buf_pool); btr_search_drop_page_hash_index(block); -#else /* !UNIV_HOTBACKUP */ - ut_ad(block == back_block1); - temp_block = back_block2; -#endif /* !UNIV_HOTBACKUP */ temp_page = temp_block->frame; /* Copy the old page to temporary space */ @@ -4773,21 +4734,16 @@ page_zip_reorganize( if (!page_zip_compress(page_zip, page, index, page_zip_level, NULL, mtr)) { -#ifndef UNIV_HOTBACKUP buf_block_free(temp_block); -#endif /* !UNIV_HOTBACKUP */ return(FALSE); } lock_move_reorganize_page(block, temp_block); -#ifndef UNIV_HOTBACKUP buf_block_free(temp_block); -#endif /* !UNIV_HOTBACKUP */ return(TRUE); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Copy the records of a page byte for byte. Do not copy the page header or trailer, except those B-tree header fields that are directly @@ -4885,7 +4841,6 @@ page_zip_copy_recs( #endif /* UNIV_ZIP_DEBUG */ page_zip_compress_write_log(page_zip, page, index, mtr); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************************//** Parses a log record of compressing an index page. diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index d055aa5d84e74..920fdb62079a9 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -1503,7 +1503,6 @@ rec_convert_dtuple_to_rec( return(rec); } -#ifndef UNIV_HOTBACKUP /**********************************************************//** Determines the size of a data tuple prefix in ROW_FORMAT=COMPACT. @return total size */ @@ -1763,7 +1762,6 @@ rec_copy_prefix_to_buf( return(*buf + (rec - (lens + 1))); } -#endif /* UNIV_HOTBACKUP */ /***************************************************************//** Validates the consistency of an old-style physical record. @@ -1930,7 +1928,6 @@ rec_print_old( rec_validate_old(rec); } -#ifndef UNIV_HOTBACKUP /***************************************************************//** Prints a physical record in ROW_FORMAT=COMPACT. Ignores the record header. */ @@ -2268,7 +2265,7 @@ operator<<(std::ostream& o, const rec_offsets_print& r) return(o); } -# ifdef UNIV_DEBUG +#ifdef UNIV_DEBUG /************************************************************//** Reads the DB_TRX_ID of a clustered index record. @return the value of DB_TRX_ID */ @@ -2308,8 +2305,7 @@ rec_get_trx_id( return(trx_read_trx_id(trx_id)); } -# endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ +#endif /* UNIV_DEBUG */ /** Mark the nth field as externally stored. @param[in] offsets array returned by rec_get_offsets() diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 2a99090424289..3f7c795230a91 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -36,7 +36,6 @@ Created 12/27/1996 Heikki Tuuri #include "dict0mem.h" #include "trx0undo.h" #include "rem0rec.h" -#ifndef UNIV_HOTBACKUP #include "dict0boot.h" #include "dict0crea.h" #include "mach0data.h" @@ -58,7 +57,6 @@ Created 12/27/1996 Heikki Tuuri #include "fts0fts.h" #include "fts0types.h" #include - #include #include @@ -127,7 +125,6 @@ row_upd_changes_first_fields_binary( const upd_t* update, /*!< in: update vector for the row */ ulint n); /*!< in: how many first fields to check */ - /*********************************************************************//** Checks if index currently is mentioned as a referenced index in a foreign key constraint. @@ -454,7 +451,6 @@ upd_node_create( return(node); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Updates the trx id and roll ptr field in a clustered index record in database @@ -488,7 +484,6 @@ row_upd_rec_sys_fields_in_recovery( } } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Sets the trx id or roll ptr field of a clustered index entry. */ void @@ -633,7 +628,6 @@ row_upd_changes_disowned_external( return(false); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Replaces the new column values stored in the update vector to the @@ -689,7 +683,6 @@ row_upd_rec_in_place( } } -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Writes into the redo log the values of trx id and roll ptr and enough info to determine their positions within a clustered index record. @@ -718,7 +711,6 @@ row_upd_write_sys_vals_to_log( return(log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Parses the log data of system field values. @@ -752,7 +744,6 @@ row_upd_parse_sys_vals( return(const_cast(ptr)); } -#ifndef UNIV_HOTBACKUP /***********************************************************//** Writes to the redo log the new values of the fields occurring in the index. */ void @@ -830,7 +821,6 @@ row_upd_index_write_log( mlog_close(mtr, log_ptr); } -#endif /* !UNIV_HOTBACKUP */ /*********************************************************************//** Parses the log data written by row_upd_index_write_log. @@ -917,7 +907,6 @@ row_upd_index_parse( return(const_cast(ptr)); } -#ifndef UNIV_HOTBACKUP /***************************************************************//** Builds an update vector from those fields which in a secondary index entry differ from a record that has the equal ordering fields. NOTE: we compare @@ -2224,7 +2213,6 @@ srv_mbr_print(const byte* data) << ", " << d << "\n"; } - /***********************************************************//** Updates a secondary index entry of a row. @return DB_SUCCESS if operation successfully completed, else error @@ -2400,7 +2388,6 @@ row_upd_sec_index_entry( case ROW_FOUND: ut_ad(err == DB_SUCCESS); - /* Delete mark the old index record; it can already be delete marked if we return after a lock wait in row_ins_sec_index_entry() below */ @@ -3511,4 +3498,3 @@ void upd_node_t::dbug_trace() DBUG_VOID_RETURN; } #endif /* !DBUG_OFF */ -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/srv/srv0mon.cc b/storage/innobase/srv/srv0mon.cc index b2d0fc852e5ed..00b7bd14c98f9 100644 --- a/storage/innobase/srv/srv0mon.cc +++ b/storage/innobase/srv/srv0mon.cc @@ -25,7 +25,6 @@ Database monitor counter interfaces Created 12/9/2009 Jimmy Yang *******************************************************/ -#ifndef UNIV_HOTBACKUP #include "buf0buf.h" #include "dict0mem.h" #include "ibuf0ibuf.h" @@ -54,7 +53,6 @@ Created 12/9/2009 Jimmy Yang #define MONITOR_BUF_PAGE_WRITTEN(name, description, code) \ MONITOR_BUF_PAGE(name, description, code, "written", PAGE_WRITTEN) - /** This array defines basic static information of monitor counters, including each monitor's name, module it belongs to, a short description and its property/type and corresponding monitor_id. @@ -2250,4 +2248,3 @@ srv_mon_default_on(void) } } } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 73c8122753707..e6bf117c673b5 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -76,42 +76,39 @@ Created 2/16/1996 Heikki Tuuri #include "srv0start.h" #include "srv0srv.h" #include "btr0defragment.h" - #include "fsp0sysspace.h" #include "row0trunc.h" #include - -#ifndef UNIV_HOTBACKUP -# include "trx0rseg.h" -# include "os0proc.h" -# include "buf0flu.h" -# include "buf0rea.h" -# include "buf0mtflu.h" -# include "dict0boot.h" -# include "dict0load.h" -# include "dict0stats_bg.h" -# include "que0que.h" -# include "usr0sess.h" -# include "lock0lock.h" -# include "trx0roll.h" -# include "trx0purge.h" -# include "lock0lock.h" -# include "pars0pars.h" -# include "btr0sea.h" -# include "rem0cmp.h" -# include "dict0crea.h" -# include "row0ins.h" -# include "row0sel.h" -# include "row0upd.h" -# include "row0row.h" -# include "row0mysql.h" -# include "row0trunc.h" -# include "btr0pcur.h" -# include "os0event.h" -# include "zlib.h" -# include "ut0crc32.h" -# include "btr0scrub.h" -# include "ut0new.h" +#include "trx0rseg.h" +#include "os0proc.h" +#include "buf0flu.h" +#include "buf0rea.h" +#include "buf0mtflu.h" +#include "dict0boot.h" +#include "dict0load.h" +#include "dict0stats_bg.h" +#include "que0que.h" +#include "usr0sess.h" +#include "lock0lock.h" +#include "trx0roll.h" +#include "trx0purge.h" +#include "lock0lock.h" +#include "pars0pars.h" +#include "btr0sea.h" +#include "rem0cmp.h" +#include "dict0crea.h" +#include "row0ins.h" +#include "row0sel.h" +#include "row0upd.h" +#include "row0row.h" +#include "row0mysql.h" +#include "row0trunc.h" +#include "btr0pcur.h" +#include "os0event.h" +#include "zlib.h" +#include "ut0crc32.h" +#include "btr0scrub.h" +#include "ut0new.h" #ifdef HAVE_LZO1X #include @@ -190,7 +187,6 @@ static bool dict_stats_thread_started = false; static bool buf_flush_page_cleaner_thread_started = false; /** Name of srv_monitor_file */ static char* srv_monitor_file_name; -#endif /* !UNIV_HOTBACKUP */ /** Minimum expected tablespace size. (10M) */ static const ulint MIN_EXPECTED_TABLESPACE_SIZE = 5 * 1024 * 1024; @@ -282,7 +278,6 @@ srv_file_check_mode( return(true); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** I/o-handler thread function. @return OS_THREAD_DUMMY_RETURN */ @@ -341,9 +336,7 @@ DECLARE_THREAD(io_handler_thread)( OS_THREAD_DUMMY_RETURN; } -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP /*********************************************************************//** Creates a log file. @return DB_SUCCESS or error code */ @@ -1692,7 +1685,6 @@ innobase_start_or_create_for_mysql(void) ib::info() << ut_crc32_implementation; - if (!srv_read_only_mode) { mutex_create(LATCH_ID_SRV_MONITOR_FILE, @@ -2746,7 +2738,6 @@ srv_fts_close(void) } #endif - /****************************************************************//** Shuts down background threads that can generate undo pages. */ void @@ -2757,7 +2748,6 @@ srv_shutdown_bg_undo_sources(void) dict_stats_shutdown(); } - /****************************************************************//** Shuts down the InnoDB database. @return DB_SUCCESS or error code */ @@ -2884,8 +2874,6 @@ innobase_shutdown_for_mysql(void) return(DB_SUCCESS); } -#endif /* !UNIV_HOTBACKUP */ - /******************************************************************** Signal all per-table background threads to shutdown, and wait for them to do diff --git a/storage/innobase/trx/trx0rec.cc b/storage/innobase/trx/trx0rec.cc index b99ce22823b0c..25807e7f82544 100644 --- a/storage/innobase/trx/trx0rec.cc +++ b/storage/innobase/trx/trx0rec.cc @@ -33,7 +33,6 @@ Created 3/26/1996 Heikki Tuuri #include "mach0data.h" #include "trx0undo.h" #include "mtr0log.h" -#ifndef UNIV_HOTBACKUP #include "dict0dict.h" #include "ut0mem.h" #include "read0read.h" @@ -87,7 +86,6 @@ trx_undof_page_add_undo_rec_log( mlog_catenate_string(mtr, undo_page + old_free + 2, len); } } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses a redo log record of adding an undo log record. @@ -135,7 +133,6 @@ trx_undo_parse_add_undo_rec( return(ptr + len); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Calculates the free space left for extending an undo log record. @return bytes left */ @@ -1245,7 +1242,6 @@ trx_undo_page_report_modify( ut_a(prefix_len < sizeof ext_buf); - spatial_status = dict_col_get_spatial_status( col); @@ -1791,7 +1787,6 @@ trx_undo_rec_get_partial_row( return(const_cast(ptr)); } -#endif /* !UNIV_HOTBACKUP */ /***********************************************************************//** Erases the unused undo log page end. @@ -1838,7 +1833,6 @@ trx_undo_parse_erase_page_end( return(ptr); } -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Writes information to an undo log about an insert, update, or a delete marking of a clustered index record. This information is used in a rollback of the @@ -2223,8 +2217,6 @@ trx_undo_prev_version_build( /*!< in: status determine if it is going into this function by purge thread or not. And if we read "after image" of undo log */ - - { trx_undo_rec_t* undo_rec = NULL; dtuple_t* entry; @@ -2411,7 +2403,6 @@ trx_undo_prev_version_build( v_status & TRX_UNDO_PREV_IN_PURGE, NULL); } - return(true); } @@ -2495,4 +2486,3 @@ trx_undo_read_v_cols( ut_ad(ptr == end_ptr); } -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index 0dd5e0a633578..724139461d397 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -32,10 +32,6 @@ Created 3/26/1996 Heikki Tuuri #include "trx0sys.ic" #endif -#ifdef UNIV_HOTBACKUP -#include "fsp0types.h" - -#else /* !UNIV_HOTBACKUP */ #include "fsp0fsp.h" #include "mtr0log.h" #include "mtr0log.h" @@ -64,7 +60,6 @@ struct file_format_t { /** The transaction system */ trx_sys_t* trx_sys = NULL; -#endif /* !UNIV_HOTBACKUP */ /** List of animal names representing file format. */ static const char* file_format_name_map[] = { @@ -134,7 +129,6 @@ ReadView::check_trx_id_sanity( } } -#ifndef UNIV_HOTBACKUP #ifdef UNIV_DEBUG /* Flag to control TRX_RSEG_N_SLOTS behavior debugging. */ uint trx_rseg_n_slots_debug = 0; @@ -1076,208 +1070,6 @@ trx_sys_create_rsegs( return(n_used); } -#else /* !UNIV_HOTBACKUP */ -/*****************************************************************//** -Prints to stderr the MySQL binlog info in the system header if the -magic number shows it valid. */ -void -trx_sys_print_mysql_binlog_offset_from_page( -/*========================================*/ - const byte* page) /*!< in: buffer containing the trx - system header page, i.e., page number - TRX_SYS_PAGE_NO in the tablespace */ -{ - const trx_sysf_t* sys_header; - - sys_header = page + TRX_SYS; - - if (mach_read_from_4(sys_header + TRX_SYS_MYSQL_LOG_INFO - + TRX_SYS_MYSQL_LOG_MAGIC_N_FLD) - == TRX_SYS_MYSQL_LOG_MAGIC_N) { - - ib::info() << "mysqlbackup: Last MySQL binlog file position " - << mach_read_from_4( - sys_header + TRX_SYS_MYSQL_LOG_INFO - + TRX_SYS_MYSQL_LOG_OFFSET_HIGH) << " " - << mach_read_from_4( - sys_header + TRX_SYS_MYSQL_LOG_INFO - + TRX_SYS_MYSQL_LOG_OFFSET_LOW) - << ", file name " << sys_header - + TRX_SYS_MYSQL_LOG_INFO + TRX_SYS_MYSQL_LOG_NAME; - } -} - -/*****************************************************************//** -Reads the file format id from the first system table space file. -Even if the call succeeds and returns TRUE, the returned format id -may be ULINT_UNDEFINED signalling that the format id was not present -in the data file. -@return TRUE if call succeeds */ -ibool -trx_sys_read_file_format_id( -/*========================*/ - const char *pathname, /*!< in: pathname of the first system - table space file */ - ulint *format_id) /*!< out: file format of the system table - space */ -{ - os_file_t file; - bool success; - byte buf[UNIV_PAGE_SIZE * 2]; - page_t* page = ut_align(buf, UNIV_PAGE_SIZE); - const byte* ptr; - ib_id_t file_format_id; - - *format_id = ULINT_UNDEFINED; - - file = os_file_create_simple_no_error_handling( - innodb_data_file_key, - pathname, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - srv_read_only_mode, - &success - ); - if (!success) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "mysqlbackup: Error: trying to read system" - " tablespace file format, but could not open the" - " tablespace file " << pathname << "!"; - return(FALSE); - } - - /* Read the page on which file format is stored */ - - IORequest read_req(IORequest::READ) - - dberr_t err = os_file_read_no_error_handling( - read_req, file, page, TRX_SYS_PAGE_NO * UNIV_PAGE_SIZE, - UNIV_PAGE_SIZE, NULL); - - if (err != DB_SUCCESS) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "mysqlbackup: Error: trying to read system" - " tablespace file format, but failed to read the" - " tablespace file " << pathname << "!"; - - os_file_close(file); - return(FALSE); - } - os_file_close(file); - - /* get the file format from the page */ - ptr = page + TRX_SYS_FILE_FORMAT_TAG; - file_format_id = mach_read_from_8(ptr); - file_format_id -= TRX_SYS_FILE_FORMAT_TAG_MAGIC_N; - - if (file_format_id >= FILE_FORMAT_NAME_N) { - - /* Either it has never been tagged, or garbage in it. */ - return(TRUE); - } - - *format_id = (ulint) file_format_id; - - return(TRUE); -} - -/*****************************************************************//** -Reads the file format id from the given per-table data file. -@return TRUE if call succeeds */ -ibool -trx_sys_read_pertable_file_format_id( -/*=================================*/ - const char *pathname, /*!< in: pathname of a per-table - datafile */ - ulint *format_id) /*!< out: file format of the per-table - data file */ -{ - os_file_t file; - bool success; - byte buf[UNIV_PAGE_SIZE * 2]; - page_t* page = ut_align(buf, UNIV_PAGE_SIZE); - const byte* ptr; - ib_uint32_t flags; - - *format_id = ULINT_UNDEFINED; - - file = os_file_create_simple_no_error_handling( - innodb_data_file_key, - pathname, - OS_FILE_OPEN, - OS_FILE_READ_ONLY, - srv_read_only_mode, - &success - ); - if (!success) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "mysqlbackup: Error: trying to read per-table" - " tablespace format, but could not open the tablespace" - " file " << pathname << "!"; - - return(FALSE); - } - - IORequest read_req(IORequest::READ); - - /* Read the first page of the per-table datafile */ - - dberr_t err = os_file_read_no_error_handling( - read_req, file, page, 0, UNIV_PAGE_SIZE, NULL); - - if (err != DB_SUCCESS) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "mysqlbackup: Error: trying to per-table data" - " file format, but failed to read the tablespace file " - << pathname << "!"; - - os_file_close(file); - return(FALSE); - } - os_file_close(file); - - /* get the file format from the page */ - ptr = page + 54; - flags = mach_read_from_4(ptr); - - if (!fsp_flags_is_valid(flags) { - /* bad tablespace flags */ - return(FALSE); - } - - *format_id = FSP_FLAGS_GET_POST_ANTELOPE(flags); - - return(TRUE); -} - - -/*****************************************************************//** -Get the name representation of the file format from its id. -@return pointer to the name */ -const char* -trx_sys_file_format_id_to_name( -/*===========================*/ - const ulint id) /*!< in: id of the file format */ -{ - if (!(id < FILE_FORMAT_NAME_N)) { - /* unknown id */ - return("Unknown"); - } - - return(file_format_name_map[id]); -} - -#endif /* !UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP /********************************************************************* Shutdown/Close the transaction system. */ void @@ -1480,4 +1272,3 @@ trx_sys_validate_trx_list() return(true); } #endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index 932151b948d94..be60b9b4aaa7e 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -32,7 +32,6 @@ Created 3/26/1996 Heikki Tuuri #endif #include "fsp0fsp.h" -#ifndef UNIV_HOTBACKUP #include "mach0data.h" #include "mtr0log.h" #include "srv0mon.h" @@ -97,7 +96,6 @@ it until a truncate operation occurs, which can remove undo logs from the end of the list and release undo log segments. In stepping through the list, s-latches on the undo log pages are enough, but in a truncate, x-latches must be obtained on the rollback segment and individual pages. */ -#endif /* !UNIV_HOTBACKUP */ /********************************************************************//** Initializes the fields in an undo log segment page. */ @@ -109,7 +107,6 @@ trx_undo_page_init( ulint type, /*!< in: undo log segment type */ mtr_t* mtr); /*!< in: mtr */ -#ifndef UNIV_HOTBACKUP /********************************************************************//** Creates and initializes an undo log memory object. @return own: the undo log memory object */ @@ -126,7 +123,6 @@ trx_undo_mem_create( const XID* xid, /*!< in: X/Open XA transaction identification*/ ulint page_no,/*!< in: undo log header page number */ ulint offset);/*!< in: undo log header byte offset on page */ -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** Initializes a cached insert undo log header page for new use. NOTE that this function has its own log record type MLOG_UNDO_HDR_REUSE. You must NOT change @@ -150,7 +146,6 @@ trx_undo_discard_latest_update_undo( page_t* undo_page, /*!< in: header page of an undo log of size 1 */ mtr_t* mtr); /*!< in: mtr */ -#ifndef UNIV_HOTBACKUP /***********************************************************************//** Gets the previous record in an undo log from the previous page. @return undo log record, the page s-latched, NULL if none */ @@ -374,9 +369,6 @@ trx_undo_page_init_log( mlog_catenate_ulint_compressed(mtr, type); } -#else /* !UNIV_HOTBACKUP */ -# define trx_undo_page_init_log(undo_page,type,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses the redo log entry of an undo log page initialization. @@ -431,7 +423,6 @@ trx_undo_page_init( trx_undo_page_init_log(undo_page, type, mtr); } -#ifndef UNIV_HOTBACKUP /***************************************************************//** Creates a new undo log segment in file. @return DB_SUCCESS if page creation OK possible error codes are: @@ -543,9 +534,6 @@ trx_undo_header_create_log( mlog_catenate_ull_compressed(mtr, trx_id); } -#else /* !UNIV_HOTBACKUP */ -# define trx_undo_header_create_log(undo_page,trx_id,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***************************************************************//** Creates a new undo log header in file. NOTE that this function has its own @@ -621,7 +609,6 @@ trx_undo_header_create( return(free); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Write X/Open XA Transaction Identification (XID) to undo log header */ static @@ -722,9 +709,6 @@ trx_undo_insert_header_reuse_log( mlog_catenate_ull_compressed(mtr, trx_id); } -#else /* !UNIV_HOTBACKUP */ -# define trx_undo_insert_header_reuse_log(undo_page,trx_id,mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /** Parse the redo log entry of an undo log page header create or reuse. @param[in] type MLOG_UNDO_HDR_CREATE or MLOG_UNDO_HDR_REUSE @@ -820,7 +804,6 @@ trx_undo_insert_header_reuse( return(free); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Writes the redo log entry of an update undo log header discard. */ UNIV_INLINE @@ -832,9 +815,6 @@ trx_undo_discard_latest_log( { mlog_write_initial_log_record(undo_page, MLOG_UNDO_HDR_DISCARD, mtr); } -#else /* !UNIV_HOTBACKUP */ -# define trx_undo_discard_latest_log(undo_page, mtr) ((void) 0) -#endif /* !UNIV_HOTBACKUP */ /***********************************************************//** Parses the redo log entry of an undo log page header discard. @@ -898,7 +878,6 @@ trx_undo_discard_latest_update_undo( trx_undo_discard_latest_log(undo_page, mtr); } -#ifndef UNIV_HOTBACKUP /********************************************************************//** Tries to add a page to the undo log segment where the undo log is placed. @return X-latched block if success, else NULL */ @@ -2176,5 +2155,3 @@ trx_undo_truncate_tablespace( return(success); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/ut/ut0dbg.cc b/storage/innobase/ut/ut0dbg.cc index bcf9d5fd1b5b0..ed4a7778afa31 100644 --- a/storage/innobase/ut/ut0dbg.cc +++ b/storage/innobase/ut/ut0dbg.cc @@ -37,16 +37,8 @@ ut_dbg_assertion_failed( ulint line) /*!< in: line number of the assertion */ { ut_print_timestamp(stderr); -#ifdef UNIV_HOTBACKUP fprintf(stderr, " InnoDB: Assertion failure in file %s line %lu\n", file, line); -#else /* UNIV_HOTBACKUP */ - fprintf(stderr, - " InnoDB: Assertion failure in thread " ULINTPF - " in file %s line " ULINTPF "\n", - os_thread_pf(os_thread_get_curr_id()), - innobase_basename(file), line); -#endif /* UNIV_HOTBACKUP */ if (expr) { fprintf(stderr, "InnoDB: Failing assertion: %s\n", expr); diff --git a/storage/innobase/ut/ut0mem.cc b/storage/innobase/ut/ut0mem.cc index 0ad251be157f7..39b86568aa372 100644 --- a/storage/innobase/ut/ut0mem.cc +++ b/storage/innobase/ut/ut0mem.cc @@ -29,11 +29,9 @@ Created 5/11/1994 Heikki Tuuri #include "ut0mem.ic" #endif -#ifndef UNIV_HOTBACKUP -# include "os0thread.h" -# include "srv0srv.h" -# include -#endif /* !UNIV_HOTBACKUP */ +#include "os0thread.h" +#include "srv0srv.h" +#include /**********************************************************************//** Copies up to size - 1 characters from the NUL-terminated string src to @@ -81,7 +79,6 @@ ut_strlcpy_rev( return(src_size); } -#ifndef UNIV_HOTBACKUP /**********************************************************************//** Return the number of times s2 occurs in s1. Overlapping instances of s2 are only counted once. @@ -199,5 +196,3 @@ ut_strreplace( return(new_str); } - -#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index cc315148a903b..8dc6fb0bbe61f 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -30,23 +30,14 @@ Created 5/11/1994 Heikki Tuuri #endif #ifndef UNIV_INNOCHECKSUM - -#ifndef UNIV_HOTBACKUP -# include -#endif /* !UNIV_HOTBACKUP */ - +#include #include "os0thread.h" #include "ut0ut.h" - #ifdef UNIV_NONINL #include "ut0ut.ic" #endif - -#ifndef UNIV_HOTBACKUP -# include "trx0trx.h" -#endif /* !UNIV_HOTBACKUP */ - -# include +#include "trx0trx.h" +#include #include "log.h" /** A constant to prevent the compiler from optimizing ut_delay() away. */ @@ -113,7 +104,7 @@ ut_time(void) return(time(NULL)); } -#ifndef UNIV_HOTBACKUP + /**********************************************************//** Returns system time. Upon successful completion, the value 0 is returned; otherwise the @@ -193,7 +184,6 @@ ut_time_ms(void) return((ulint) tv.tv_sec * 1000 + tv.tv_usec / 1000); } -#endif /* !UNIV_HOTBACKUP */ /**********************************************************//** Returns the difference of two times in seconds. @@ -293,78 +283,6 @@ ut_sprintf_timestamp( #endif } -#ifdef UNIV_HOTBACKUP -/**********************************************************//** -Sprintfs a timestamp to a buffer with no spaces and with ':' characters -replaced by '_'. */ -void -ut_sprintf_timestamp_without_extra_chars( -/*=====================================*/ - char* buf) /*!< in: buffer where to sprintf */ -{ -#ifdef _WIN32 - SYSTEMTIME cal_tm; - - GetLocalTime(&cal_tm); - - sprintf(buf, "%02d%02d%02d_%2d_%02d_%02d", - (int) cal_tm.wYear % 100, - (int) cal_tm.wMonth, - (int) cal_tm.wDay, - (int) cal_tm.wHour, - (int) cal_tm.wMinute, - (int) cal_tm.wSecond); -#else - struct tm* cal_tm_ptr; - time_t tm; - - struct tm cal_tm; - time(&tm); - localtime_r(&tm, &cal_tm); - cal_tm_ptr = &cal_tm; - sprintf(buf, "%02d%02d%02d_%2d_%02d_%02d", - cal_tm_ptr->tm_year % 100, - cal_tm_ptr->tm_mon + 1, - cal_tm_ptr->tm_mday, - cal_tm_ptr->tm_hour, - cal_tm_ptr->tm_min, - cal_tm_ptr->tm_sec); -#endif -} - -/**********************************************************//** -Returns current year, month, day. */ -void -ut_get_year_month_day( -/*==================*/ - ulint* year, /*!< out: current year */ - ulint* month, /*!< out: month */ - ulint* day) /*!< out: day */ -{ -#ifdef _WIN32 - SYSTEMTIME cal_tm; - - GetLocalTime(&cal_tm); - - *year = (ulint) cal_tm.wYear; - *month = (ulint) cal_tm.wMonth; - *day = (ulint) cal_tm.wDay; -#else - struct tm* cal_tm_ptr; - time_t tm; - - struct tm cal_tm; - time(&tm); - localtime_r(&tm, &cal_tm); - cal_tm_ptr = &cal_tm; - *year = (ulint) cal_tm_ptr->tm_year + 1900; - *month = (ulint) cal_tm_ptr->tm_mon + 1; - *day = (ulint) cal_tm_ptr->tm_mday; -#endif -} - -#else /* UNIV_HOTBACKUP */ - /*************************************************************//** Runs an idle loop on CPU. The argument gives the desired delay in microseconds on 100 MHz Pentium + Visual C++. @@ -390,7 +308,6 @@ ut_delay( return(j); } -#endif /* UNIV_HOTBACKUP */ /*************************************************************//** Prints the contents of a memory buffer in hex and ascii. */ @@ -495,7 +412,6 @@ ut_2_power_up( return(res); } -#ifndef UNIV_HOTBACKUP /** Get a fixed-length string, quoted as an SQL identifier. If the string contains a slash '/', the string will be output as two identifiers separated by a period (.), @@ -614,7 +530,6 @@ ut_copy_file( } } while (len > 0); } -#endif /* !UNIV_HOTBACKUP */ #ifdef _WIN32 # include From b727213de27582a366a12fe87d491d91ad6987ad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 30 Dec 2016 16:14:33 +0200 Subject: [PATCH 05/74] MDEV-11687 innodb_use_fallocate has no effect Deprecate the variable in MariaDB 10.2, saying that it will be removed in 10.3. --- mysql-test/suite/sys_vars/r/sysvars_innodb.result | 14 ++++++++++++++ storage/innobase/handler/ha_innodb.cc | 14 ++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 320d34fc63b23..87e000faf025f 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -2595,6 +2595,20 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY YES COMMAND_LINE_ARGUMENT NONE +VARIABLE_NAME INNODB_USE_FALLOCATE +SESSION_VALUE NULL +GLOBAL_VALUE OFF +GLOBAL_VALUE_ORIGIN COMPILE-TIME +DEFAULT_VALUE OFF +VARIABLE_SCOPE GLOBAL +VARIABLE_TYPE BOOLEAN +VARIABLE_COMMENT Use posix_fallocate() to allocate files. DEPRECATED, has no effect. +NUMERIC_MIN_VALUE NULL +NUMERIC_MAX_VALUE NULL +NUMERIC_BLOCK_SIZE NULL +ENUM_VALUE_LIST OFF,ON +READ_ONLY YES +COMMAND_LINE_ARGUMENT NONE VARIABLE_NAME INNODB_USE_MTFLUSH SESSION_VALUE NULL GLOBAL_VALUE OFF diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 9ed7855b8dab6..6a6a049d5a6af 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -250,6 +250,7 @@ values */ static ulong innobase_fast_shutdown = 1; static my_bool innobase_file_format_check = TRUE; static my_bool innobase_use_atomic_writes = FALSE; +static my_bool innobase_use_fallocate; static my_bool innobase_use_doublewrite = TRUE; static my_bool innobase_use_checksums = TRUE; static my_bool innobase_locks_unsafe_for_binlog = FALSE; @@ -4637,6 +4638,13 @@ innobase_init( data_mysql_default_charset_coll = (ulint) default_charset_info->number; innobase_commit_concurrency_init_default(); + + if (innobase_use_fallocate) { + ib::warn() << "innodb_use_fallocate is DEPRECATED" + " and has no effect in MariaDB 10.2." + " It will be removed in MariaDB 10.3."; + } + srv_use_atomic_writes = (ibool) innobase_use_atomic_writes; if (innobase_use_atomic_writes) { fprintf(stderr, "InnoDB: using atomic writes.\n"); @@ -21947,6 +21955,11 @@ static MYSQL_SYSVAR_BOOL(use_atomic_writes, innobase_use_atomic_writes, "on Linux only with FusionIO device, and directFS filesystem.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate, + PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, + "Use posix_fallocate() to allocate files. DEPRECATED, has no effect.", + NULL, NULL, FALSE); + static MYSQL_SYSVAR_ULONG(io_capacity, srv_io_capacity, PLUGIN_VAR_RQCMDARG, "Number of IOPs the server can do. Tunes the background IO rate", @@ -23178,6 +23191,7 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(data_home_dir), MYSQL_SYSVAR(doublewrite), MYSQL_SYSVAR(use_atomic_writes), + MYSQL_SYSVAR(use_fallocate), MYSQL_SYSVAR(api_enable_binlog), MYSQL_SYSVAR(api_enable_mdl), MYSQL_SYSVAR(api_disable_rowlock), From 2f5670dc269ce19f14132d54e54060c91cc028e3 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 27 Dec 2016 14:13:32 +0530 Subject: [PATCH 06/74] MDEV-11636 Extra persistent columns on slave always gets NULL in RBR Problem:- In replication if slave has extra persistent column then these column are not computed while applying write-set from master. Solution:- While applying row events from server, we will generate values for extra persistent columns. --- .../rpl/r/rpl_alter_extra_persistent.result | 220 ++++++++++++++++++ .../rpl/t/rpl_alter_extra_persistent.test | 106 +++++++++ sql/rpl_record.cc | 32 +++ sql/rpl_record.h | 1 + 4 files changed, 359 insertions(+) create mode 100644 mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result create mode 100644 mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test diff --git a/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result new file mode 100644 index 0000000000000..84915073d87fc --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result @@ -0,0 +1,220 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +connection slave; +select * from t1 order by a; +a +1 +2 +3 +4 +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +connection master; +insert into t1 values(5); +insert into t1 values(6); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#UPDATE query +connection master; +update t1 set a = a+10; +select * from t1 order by a; +a +11 +12 +13 +14 +15 +16 +connection slave; +select * from t1 order by a; +a z1 z2 +11 12 13 +12 13 14 +13 14 15 +14 15 16 +15 16 17 +16 17 18 +connection master; +update t1 set a = a-10; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#DELETE quert +connection master; +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +#REPLACE query +connection master; +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#SELECT query +connection master; +select * from t1 where a > 2 and a < 4; +a +3 +connection slave; +select * from t1 where a > 2 and a < 4; +a z1 z2 +3 4 5 +#UPDATE with SELECT query +connection master; +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +13 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +13 14 15 +connection master; +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; +connection master; +#entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +connection slave; +include/wait_for_slave_sql_error.inc [errno=1062] +connection slave; +connection slave; +select * from t1 order by a; +a z1 z2 z4 +1 2 3 1 +2 3 4 2 +3 4 5 3 +4 5 6 4 +5 6 7 5 +6 7 8 0 +alter table t1 drop column z4; +start slave; +include/wait_for_slave_sql_to_start.inc +connection slave; +connection slave; +connection master; +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +7 8 9 +connection master; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +drop table t1; +include/rpl_end.inc +connection server_2; +connection server_2; +connection server_2; +connection server_2; +connection server_1; +connection server_1; +connection server_1; +connection server_2; +connection server_1; +connection server_2; +connection server_2; +connection server_1; +connection server_1; diff --git a/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test new file mode 100644 index 0000000000000..3b2fff1cb1350 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test @@ -0,0 +1,106 @@ +--source include/master-slave.inc +--source include/have_binlog_format_row.inc + +--enable_connect_log +--connection master +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); + +--sync_slave_with_master +select * from t1 order by a; +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; + +--connection master +insert into t1 values(5); +insert into t1 values(6); + +--sync_slave_with_master +select * from t1 order by a; + + +--echo #UPDATE query + +--connection master +update t1 set a = a+10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a-10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #DELETE quert +--connection master +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #REPLACE query +--connection master +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); + +--sync_slave_with_master +select * from t1 order by a; + +--echo #SELECT query +--connection master +select * from t1 where a > 2 and a < 4; + +--connection slave +select * from t1 where a > 2 and a < 4; + +--echo #UPDATE with SELECT query +--connection master +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; + +--connection master + +--echo #entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; + +--connection slave +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc +select * from t1 order by a; +alter table t1 drop column z4; +start slave; + +--source include/wait_for_slave_sql_to_start.inc + +--connection master +--sync_slave_with_master +select * from t1 order by a; + +--connection master +select * from t1 order by a; +drop table t1; + +--source include/rpl_end.inc diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index f82c5a3982af6..e62f722675d84 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -416,6 +416,13 @@ unpack_row(rpl_group_info *rgi, } } + /* + Add Extra slave persistent columns + */ + int error= 0; + if ((error= fill_extra_persistent_columns(table, cols->n_bits))) + DBUG_RETURN(error); + /* We should now have read all the null bytes, otherwise something is really wrong. @@ -489,5 +496,30 @@ int prepare_record(TABLE *const table, const uint skip, const bool check) DBUG_RETURN(0); } +/** + Fills @c table->record[0] with computed values of extra persistent column which are present on slave but not on master. + @param table Table whose record[0] buffer is prepared. + @param master_cols No of columns on master + @returns 0 on success + */ +int fill_extra_persistent_columns(TABLE *table, int master_cols) +{ + int error= 0; + Field **vfield_ptr, *vfield; + if (!table->vfield) + return 0; + for (vfield_ptr= table->vfield; *vfield_ptr; ++vfield_ptr) + { + vfield= *vfield_ptr; + if (vfield->field_index >= master_cols && vfield->stored_in_db) + { + /*Set bitmap for writing*/ + bitmap_set_bit(table->vcol_set, vfield->field_index); + error= vfield->vcol_info->expr_item->save_in_field(vfield,0); + bitmap_clear_bit(table->vcol_set, vfield->field_index); + } + } + return error; +} #endif // HAVE_REPLICATION diff --git a/sql/rpl_record.h b/sql/rpl_record.h index c10eb8225b089..be69716d9d557 100644 --- a/sql/rpl_record.h +++ b/sql/rpl_record.h @@ -38,6 +38,7 @@ int unpack_row(rpl_group_info *rgi, // Fill table's record[0] with default values. int prepare_record(TABLE *const table, const uint skip, const bool check); +int fill_extra_persistent_columns(TABLE *table, int master_cols); #endif #endif From d02a77bc5fbaf2e9504035e98f132fdf7145fd94 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 27 Dec 2016 14:13:32 +0530 Subject: [PATCH 07/74] MDEV-11636 Extra persistent columns on slave always gets NULL in RBR Problem:- In replication if slave has extra persistent column then these column are not computed while applying write-set from master. Solution:- While applying row events from server, we will generate values for extra persistent columns. --- .../rpl/r/rpl_alter_extra_persistent.result | 220 ++++++++++++++++++ .../rpl/t/rpl_alter_extra_persistent.test | 106 +++++++++ sql/rpl_record.cc | 31 +++ sql/rpl_record.h | 1 + 4 files changed, 358 insertions(+) create mode 100644 mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result create mode 100644 mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test diff --git a/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result new file mode 100644 index 0000000000000..84915073d87fc --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result @@ -0,0 +1,220 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +connection slave; +select * from t1 order by a; +a +1 +2 +3 +4 +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +connection master; +insert into t1 values(5); +insert into t1 values(6); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#UPDATE query +connection master; +update t1 set a = a+10; +select * from t1 order by a; +a +11 +12 +13 +14 +15 +16 +connection slave; +select * from t1 order by a; +a z1 z2 +11 12 13 +12 13 14 +13 14 15 +14 15 16 +15 16 17 +16 17 18 +connection master; +update t1 set a = a-10; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#DELETE quert +connection master; +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +#REPLACE query +connection master; +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#SELECT query +connection master; +select * from t1 where a > 2 and a < 4; +a +3 +connection slave; +select * from t1 where a > 2 and a < 4; +a z1 z2 +3 4 5 +#UPDATE with SELECT query +connection master; +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +13 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +13 14 15 +connection master; +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; +connection master; +#entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +connection slave; +include/wait_for_slave_sql_error.inc [errno=1062] +connection slave; +connection slave; +select * from t1 order by a; +a z1 z2 z4 +1 2 3 1 +2 3 4 2 +3 4 5 3 +4 5 6 4 +5 6 7 5 +6 7 8 0 +alter table t1 drop column z4; +start slave; +include/wait_for_slave_sql_to_start.inc +connection slave; +connection slave; +connection master; +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +7 8 9 +connection master; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +drop table t1; +include/rpl_end.inc +connection server_2; +connection server_2; +connection server_2; +connection server_2; +connection server_1; +connection server_1; +connection server_1; +connection server_2; +connection server_1; +connection server_2; +connection server_2; +connection server_1; +connection server_1; diff --git a/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test new file mode 100644 index 0000000000000..3b2fff1cb1350 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test @@ -0,0 +1,106 @@ +--source include/master-slave.inc +--source include/have_binlog_format_row.inc + +--enable_connect_log +--connection master +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); + +--sync_slave_with_master +select * from t1 order by a; +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; + +--connection master +insert into t1 values(5); +insert into t1 values(6); + +--sync_slave_with_master +select * from t1 order by a; + + +--echo #UPDATE query + +--connection master +update t1 set a = a+10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a-10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #DELETE quert +--connection master +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #REPLACE query +--connection master +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); + +--sync_slave_with_master +select * from t1 order by a; + +--echo #SELECT query +--connection master +select * from t1 where a > 2 and a < 4; + +--connection slave +select * from t1 where a > 2 and a < 4; + +--echo #UPDATE with SELECT query +--connection master +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; + +--connection master + +--echo #entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; + +--connection slave +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc +select * from t1 order by a; +alter table t1 drop column z4; +start slave; + +--source include/wait_for_slave_sql_to_start.inc + +--connection master +--sync_slave_with_master +select * from t1 order by a; + +--connection master +select * from t1 order by a; +drop table t1; + +--source include/rpl_end.inc diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index 183248ad1b8ce..5e48cfb02e57f 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -389,6 +389,12 @@ unpack_row(rpl_group_info *rgi, } } + /* + Add Extra slave persistent columns + */ + if ((error= fill_extra_persistent_columns(table, cols->n_bits))) + DBUG_RETURN(error); + /* We should now have read all the null bytes, otherwise something is really wrong. @@ -461,5 +467,30 @@ int prepare_record(TABLE *const table, const uint skip, const bool check) DBUG_RETURN(0); } +/** + Fills @c table->record[0] with computed values of extra persistent column which are present on slave but not on master. + @param table Table whose record[0] buffer is prepared. + @param master_cols No of columns on master + @returns 0 on success + */ +int fill_extra_persistent_columns(TABLE *table, int master_cols) +{ + int error= 0; + Field **vfield_ptr, *vfield; + if (!table->vfield) + return 0; + for (vfield_ptr= table->vfield; *vfield_ptr; ++vfield_ptr) + { + vfield= *vfield_ptr; + if (vfield->field_index >= master_cols && vfield->stored_in_db) + { + /*Set bitmap for writing*/ + bitmap_set_bit(table->vcol_set, vfield->field_index); + error= vfield->vcol_info->expr_item->save_in_field(vfield,0); + bitmap_clear_bit(table->vcol_set, vfield->field_index); + } + } + return error; +} #endif // HAVE_REPLICATION diff --git a/sql/rpl_record.h b/sql/rpl_record.h index c10eb8225b089..be69716d9d557 100644 --- a/sql/rpl_record.h +++ b/sql/rpl_record.h @@ -38,6 +38,7 @@ int unpack_row(rpl_group_info *rgi, // Fill table's record[0] with default values. int prepare_record(TABLE *const table, const uint skip, const bool check); +int fill_extra_persistent_columns(TABLE *table, int master_cols); #endif #endif From efcd0935f77c1080cf00f86e70f28b8a74762475 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 27 Dec 2016 14:13:32 +0530 Subject: [PATCH 08/74] MDEV-11636 Extra persistent columns on slave always gets NULL in RBR Problem:- In replication if slave has extra persistent column then these column are not computed while applying write-set from master. Solution:- While applying row events from server, we will generate values for extra persistent columns. --- .../rpl/r/rpl_alter_extra_persistent.result | 203 ++++++++++++++++++ .../rpl/t/rpl_alter_extra_persistent.test | 106 +++++++++ sql/rpl_record.cc | 32 +++ sql/rpl_record.h | 1 + 4 files changed, 342 insertions(+) create mode 100644 mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result create mode 100644 mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test diff --git a/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result new file mode 100644 index 0000000000000..96df87d8ad434 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_alter_extra_persistent.result @@ -0,0 +1,203 @@ +include/master-slave.inc +[connection master] +connection master; +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); +connection slave; +select * from t1 order by a; +a +1 +2 +3 +4 +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +connection master; +insert into t1 values(5); +insert into t1 values(6); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#UPDATE query +connection master; +update t1 set a = a+10; +select * from t1 order by a; +a +11 +12 +13 +14 +15 +16 +connection slave; +select * from t1 order by a; +a z1 z2 +11 12 13 +12 13 14 +13 14 15 +14 15 16 +15 16 17 +16 17 18 +connection master; +update t1 set a = a-10; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#DELETE quert +connection master; +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +#REPLACE query +connection master; +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#SELECT query +connection master; +select * from t1 where a > 2 and a < 4; +a +3 +connection slave; +select * from t1 where a > 2 and a < 4; +a z1 z2 +3 4 5 +#UPDATE with SELECT query +connection master; +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; +a +1 +2 +4 +5 +6 +13 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +4 5 6 +5 6 7 +6 7 8 +13 14 15 +connection master; +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +#Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; +connection master; +#entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +connection slave; +include/wait_for_slave_sql_error.inc [errno=1062] +select * from t1 order by a; +a z1 z2 z4 +1 2 3 1 +2 3 4 2 +3 4 5 3 +4 5 6 4 +5 6 7 5 +6 7 8 0 +alter table t1 drop column z4; +start slave; +include/wait_for_slave_sql_to_start.inc +connection master; +connection slave; +select * from t1 order by a; +a z1 z2 +1 2 3 +2 3 4 +3 4 5 +4 5 6 +5 6 7 +6 7 8 +7 8 9 +connection master; +select * from t1 order by a; +a +1 +2 +3 +4 +5 +6 +7 +drop table t1; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test new file mode 100644 index 0000000000000..3b2fff1cb1350 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_alter_extra_persistent.test @@ -0,0 +1,106 @@ +--source include/master-slave.inc +--source include/have_binlog_format_row.inc + +--enable_connect_log +--connection master +create table t1(a int primary key); +insert into t1 values(1); +insert into t1 values(2); +insert into t1 values(3); +insert into t1 values(4); + +--sync_slave_with_master +select * from t1 order by a; +alter table t1 add column z1 int as(a+1) virtual, add column z2 int as (a+2) persistent; +select * from t1 order by a; + +--connection master +insert into t1 values(5); +insert into t1 values(6); + +--sync_slave_with_master +select * from t1 order by a; + + +--echo #UPDATE query + +--connection master +update t1 set a = a+10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a-10; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #DELETE quert +--connection master +delete from t1 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #REPLACE query +--connection master +replace into t1 values(1); +replace into t1 values(3); +replace into t1 values(1); + +--sync_slave_with_master +select * from t1 order by a; + +--echo #SELECT query +--connection master +select * from t1 where a > 2 and a < 4; + +--connection slave +select * from t1 where a > 2 and a < 4; + +--echo #UPDATE with SELECT query +--connection master +update t1 set a = a + 10 where a > 2 and a < 4; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--connection master +update t1 set a = a - 10 where a = 13; +select * from t1 order by a; + +--sync_slave_with_master +select * from t1 order by a; + +--echo #Break Unique Constraint +alter table t1 add column z4 int as (a % 6) persistent unique; + +--connection master + +--echo #entering duplicate value for slave persistent column +insert into t1 values(7); +select * from t1 order by a; + +--connection slave +--let $slave_sql_errno= 1062 +--source include/wait_for_slave_sql_error.inc +select * from t1 order by a; +alter table t1 drop column z4; +start slave; + +--source include/wait_for_slave_sql_to_start.inc + +--connection master +--sync_slave_with_master +select * from t1 order by a; + +--connection master +select * from t1 order by a; +drop table t1; + +--source include/rpl_end.inc diff --git a/sql/rpl_record.cc b/sql/rpl_record.cc index f82c5a3982af6..c753f76c9303d 100644 --- a/sql/rpl_record.cc +++ b/sql/rpl_record.cc @@ -416,6 +416,13 @@ unpack_row(rpl_group_info *rgi, } } + /* + Add Extra slave persistent columns + */ + int error= 0; + if ((error= fill_extra_persistent_columns(table, cols->n_bits))) + DBUG_RETURN(error); + /* We should now have read all the null bytes, otherwise something is really wrong. @@ -489,5 +496,30 @@ int prepare_record(TABLE *const table, const uint skip, const bool check) DBUG_RETURN(0); } +/** + Fills @c table->record[0] with computed values of extra persistent column which are present on slave but not on master. + @param table Table whose record[0] buffer is prepared. + @param master_cols No of columns on master + @returns 0 on success + */ +int fill_extra_persistent_columns(TABLE *table, int master_cols) +{ + int error= 0; + Field **vfield_ptr, *vfield; + if (!table->vfield) + return 0; + for (vfield_ptr= table->vfield; *vfield_ptr; ++vfield_ptr) + { + vfield= *vfield_ptr; + if (vfield->field_index >= master_cols && vfield->stored_in_db()) + { + /*Set bitmap for writing*/ + bitmap_set_bit(table->vcol_set, vfield->field_index); + error= vfield->vcol_info->expr->save_in_field(vfield,0); + bitmap_clear_bit(table->vcol_set, vfield->field_index); + } + } + return error; +} #endif // HAVE_REPLICATION diff --git a/sql/rpl_record.h b/sql/rpl_record.h index c10eb8225b089..be69716d9d557 100644 --- a/sql/rpl_record.h +++ b/sql/rpl_record.h @@ -38,6 +38,7 @@ int unpack_row(rpl_group_info *rgi, // Fill table's record[0] with default values. int prepare_record(TABLE *const table, const uint skip, const bool check); +int fill_extra_persistent_columns(TABLE *table, int master_cols); #endif #endif From 3871477c40efc826805f4c4e35b006c2c233dd26 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 1 Jan 2017 20:06:03 +0200 Subject: [PATCH 09/74] MDEV-10100 main.pool_of_threads fails sporadically in buildbot The patch fixes two test failures: - on slow builders, sometimes a connection attempt which should fail due to the exceeded number of thread_pool_max_threads actually succeeds; - on even slow builders, MTR sometimes cannot establish the initial connection, and check-testcase fails prior to the test start The problem with check-testcase was caused by connect-timeout=2 which was set for all clients in the test config file. On slow builders it might be not enough. There is no way to override it for the pre-test check, so it needed to be substantially increased or removed. The other problem was caused by a race condition between sleeps that the test performs in existing connections and the connect timeout for the connection attempt which was expected to fail. If sleeps finished before the connect-timeout was exceeded, it would allow the connection to succeed. To solve each problem without making the other one worse, connect-timeout should be configured dynamically during the test. Due to the nature of the test (all connections must be busy at the moment when we need to change the timeout, and cannot execute SET GLOBAL ...), it needs to be done independently from the server. The solution: - recognize 'connect_timeout' as a connection option in mysqltest's "connect" command; - remove connect-timeout from the test configuration file; - use the new connect_timeout option for those connections which are expected to fail; - re-arrange the test flow to allow running a huge SLEEP without affecting the test execution time (because it would be interrupted after the main test flow is finished). The test is still subject to false negatives, e.g. if the connection fails due to timeout rather than due to the exceeded number of allowed threads, or if the connection on extra port succeeds due to a race condition and not because the special logic for the extra port. But those false negatives have always been possible there on slow builders, they should not be critical because faster builders should catch such failures if they appear. --- client/mysqltest.cc | 12 ++++++++ mysql-test/r/pool_of_threads.result | 21 ++++++------- mysql-test/t/pool_of_threads.cnf | 3 -- mysql-test/t/pool_of_threads.test | 47 +++++++++++++++-------------- 4 files changed, 47 insertions(+), 36 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index ed401b6e4a2f3..15db5759ac93c 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -5945,6 +5945,7 @@ void do_connect(struct st_command *command) my_bool con_shm __attribute__ ((unused))= 0; int read_timeout= 0; int write_timeout= 0; + int connect_timeout= 0; struct st_connection* con_slot; static DYNAMIC_STRING ds_connection_name; @@ -6051,6 +6052,11 @@ void do_connect(struct st_command *command) { write_timeout= atoi(con_options + sizeof("write_timeout=")-1); } + else if (strncasecmp(con_options, "connect_timeout=", + sizeof("connect_timeout=")-1) == 0) + { + connect_timeout= atoi(con_options + sizeof("connect_timeout=")-1); + } else die("Illegal option to connect: %.*s", (int) (end - con_options), con_options); @@ -6135,6 +6141,12 @@ void do_connect(struct st_command *command) (char*)&write_timeout); } + if (connect_timeout) + { + mysql_options(con_slot->mysql, MYSQL_OPT_CONNECT_TIMEOUT, + (char*)&connect_timeout); + } + #ifdef HAVE_SMEM if (con_shm) { diff --git a/mysql-test/r/pool_of_threads.result b/mysql-test/r/pool_of_threads.result index 9611d7ff43b97..ddc451516e229 100644 --- a/mysql-test/r/pool_of_threads.result +++ b/mysql-test/r/pool_of_threads.result @@ -2157,23 +2157,22 @@ Warnings: Warning 1052 Column 'kundentyp' in group statement is ambiguous drop table t1; SET optimizer_switch=@save_optimizer_switch; -SELECT sleep(5.5); -SELECT sleep(5); +SELECT sleep(50); +SELECT sleep(50); # -- Success: more than --thread_pool_max_threads normal connections not possible -sleep(5.5) -0 -sleep(5) -0 -SELECT sleep(5); -SELECT sleep(5); SELECT 'Connection on extra port ok'; Connection on extra port ok Connection on extra port ok +SELECT sleep(5.5); SELECT 'Connection on extra port 2 ok'; Connection on extra port 2 ok Connection on extra port 2 ok # -- Success: more than --extra-max-connections + 1 normal connections not possible -sleep(5) -0 -sleep(5) +KILL QUERY ; +KILL QUERY ; +sleep(50) +1 +sleep(50) +1 +sleep(5.5) 0 diff --git a/mysql-test/t/pool_of_threads.cnf b/mysql-test/t/pool_of_threads.cnf index c03e1da64502e..f6651c878deb9 100644 --- a/mysql-test/t/pool_of_threads.cnf +++ b/mysql-test/t/pool_of_threads.cnf @@ -7,8 +7,5 @@ loose-thread_pool_max_threads= 2 extra-port= @ENV.MASTER_EXTRA_PORT extra-max-connections=1 -[client] -connect-timeout= 2 - [ENV] MASTER_EXTRA_PORT= @OPT.port diff --git a/mysql-test/t/pool_of_threads.test b/mysql-test/t/pool_of_threads.test index 24e0218db626f..f13a096985cf4 100644 --- a/mysql-test/t/pool_of_threads.test +++ b/mysql-test/t/pool_of_threads.test @@ -15,23 +15,26 @@ SET optimizer_switch=@save_optimizer_switch; # connections on the extra port. # First set two connections running, and check that extra connection -# on normal port fails due to--thread-pool-max_threads=2 +# on normal port fails due to --thread-pool-max-threads=2. +# We can afford using a really long sleep, because we won't wait +# till it ends, we'll interrupt it as soon as we don't need it anymore + connection default; +--let $con1_id= `SELECT CONNECTION_ID()` -# Sleep for slightly longer than 5 sec to trigger MDEV-4566 -# (abort in interruptible wait connection check) -send SELECT sleep(5.5); +send SELECT sleep(50); --sleep 1 connect(con2,localhost,root,,); -connection con2; -send SELECT sleep(5); +--let $con2_id= `SELECT CONNECTION_ID()` + +send SELECT sleep(50); --sleep 0.5 --disable_abort_on_error --disable_result_log --disable_query_log -connect(con3,localhost,root,,); +connect(con3,localhost,root,,,,,connect_timeout=2); --enable_query_log --enable_result_log --enable_abort_on_error @@ -45,24 +48,15 @@ if ($error) --echo # -- Success: more than --thread_pool_max_threads normal connections not possible } -connection default; ---reap -connection con2; ---reap - -# Now try again, but this time use the extra port to successfully connect. - -connection default; -send SELECT sleep(5); - -connection con2; -send SELECT sleep(5); ---sleep 1 - connect(extracon,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); connection extracon; SELECT 'Connection on extra port ok'; +# Here, sleep just for slightly longer than 5 sec to trigger MDEV-4566 +# (abort in interruptible wait connection check). +send SELECT sleep(5.5); + + connect(extracon2,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); connection extracon2; SELECT 'Connection on extra port 2 ok'; @@ -70,7 +64,7 @@ SELECT 'Connection on extra port 2 ok'; --disable_abort_on_error --disable_result_log --disable_query_log -connect(extracon3,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,); +connect(extracon3,127.0.0.1,root,,test,$MASTER_EXTRA_PORT,,connect_timeout=2); --enable_query_log --enable_result_log --enable_abort_on_error @@ -84,7 +78,16 @@ if ($error) --echo # -- Success: more than --extra-max-connections + 1 normal connections not possible } +connection extracon2; +--replace_result $con1_id +eval KILL QUERY $con1_id; +--replace_result $con2_id +eval KILL QUERY $con2_id; + connection default; --reap connection con2; --reap + +connection extracon; +--reap From d9a1a201aae87a655cdf3e5a344b2265912a94a7 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 3 Jan 2017 10:10:58 +0530 Subject: [PATCH 10/74] MDEV-11016 wsrep_node_is_ready() check is too strict Problem:- The condition that checks for node readiness is too strict as it does not allow SELECTs even if these selects do not access any tables. For example,if we run SELECT 1; OR SELECT @@max_allowed_packet; Solution:- We need not to report this error when all_tables(lex->query_tables) is NULL: --- mysql-test/suite/galera/r/MW-284.result | 2 ++ mysql-test/suite/galera/r/galera_var_dirty_reads.result | 9 +++++++++ mysql-test/suite/galera/t/MW-284.test | 5 +++++ mysql-test/suite/galera/t/galera_var_dirty_reads.test | 5 +++++ sql/sql_parse.cc | 5 ++++- 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/galera/r/MW-284.result b/mysql-test/suite/galera/r/MW-284.result index 3ff131674ea62..9465370c25b4e 100644 --- a/mysql-test/suite/galera/r/MW-284.result +++ b/mysql-test/suite/galera/r/MW-284.result @@ -2,12 +2,14 @@ CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; SET SESSION wsrep_on = OFF; SET SESSION wsrep_on = ON; +SET global wsrep_sync_wait=0; START SLAVE; include/wait_for_slave_param.inc [Slave_IO_Running] SET GLOBAL wsrep_provider_options='gmcast.isolate=0'; include/wait_for_slave_to_start.inc INSERT INTO t1 VALUES (1); DROP TABLE t1; +SET global wsrep_sync_wait=7; STOP SLAVE; RESET SLAVE ALL; CALL mtr.add_suppression('failed registering on master'); diff --git a/mysql-test/suite/galera/r/galera_var_dirty_reads.result b/mysql-test/suite/galera/r/galera_var_dirty_reads.result index 8a3175912c70b..52729395b1927 100644 --- a/mysql-test/suite/galera/r/galera_var_dirty_reads.result +++ b/mysql-test/suite/galera/r/galera_var_dirty_reads.result @@ -18,6 +18,15 @@ Variable_name Value wsrep_cluster_status non-Primary SELECT * FROM t1; ERROR 08S01: WSREP has not yet prepared node for application use +SELECT @@wsrep_dirty_reads; +@@wsrep_dirty_reads +0 +SELECT 2; +2 +2 +SELECT 2+2 FROM DUAL; +2+2 +4 SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; i diff --git a/mysql-test/suite/galera/t/MW-284.test b/mysql-test/suite/galera/t/MW-284.test index f3ce1b0dc9171..7add82f122796 100644 --- a/mysql-test/suite/galera/t/MW-284.test +++ b/mysql-test/suite/galera/t/MW-284.test @@ -11,12 +11,16 @@ --enable_query_log --connection node_1 +--let $wsrep_sync_wait_state= `SELECT @@global.wsrep_sync_wait;` CREATE TABLE t1 (f1 INTEGER) ENGINE=InnoDB; SET GLOBAL wsrep_provider_options='gmcast.isolate=1'; SET SESSION wsrep_on = OFF; --let $wait_condition = SELECT VARIABLE_VALUE = 'non-Primary' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_status' --source include/wait_condition.inc SET SESSION wsrep_on = ON; +#wsrep_sync_wait is set to zero because when slave tries to connect it it ask for queries like SELECT UNIX_TIMESTAMP() on node 1 which will fail, causing +#a warning in slave error log. +SET global wsrep_sync_wait=0; --connection node_3 START SLAVE; @@ -47,6 +51,7 @@ INSERT INTO t1 VALUES (1); --connection node_1 DROP TABLE t1; +--eval SET global wsrep_sync_wait=$wsrep_sync_wait_state --connection node_3 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 't1' --source include/wait_condition.inc diff --git a/mysql-test/suite/galera/t/galera_var_dirty_reads.test b/mysql-test/suite/galera/t/galera_var_dirty_reads.test index bcdb1574a3dd5..1fd30c69a7bc8 100644 --- a/mysql-test/suite/galera/t/galera_var_dirty_reads.test +++ b/mysql-test/suite/galera/t/galera_var_dirty_reads.test @@ -37,6 +37,11 @@ SHOW STATUS LIKE 'wsrep_cluster_status'; --error ER_UNKNOWN_COM_ERROR SELECT * FROM t1; +#Select query which does not access table should be allowed MDEV-11016 +SELECT @@wsrep_dirty_reads; +SELECT 2; +SELECT 2+2 FROM DUAL; + SET @@session.wsrep_dirty_reads=ON; SELECT * FROM t1; diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index b214a1759be12..5717c3dd6c423 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -288,6 +288,7 @@ void init_update_queries(void) server_command_flags[COM_STMT_RESET]= CF_SKIP_QUESTIONS | CF_SKIP_WSREP_CHECK; server_command_flags[COM_STMT_EXECUTE]= CF_SKIP_WSREP_CHECK; server_command_flags[COM_STMT_SEND_LONG_DATA]= CF_SKIP_WSREP_CHECK; + server_command_flags[COM_REGISTER_SLAVE]= CF_SKIP_WSREP_CHECK; /* Initialize the sql command flags array. */ memset(sql_command_flags, 0, sizeof(sql_command_flags)); @@ -2651,7 +2652,7 @@ mysql_execute_command(THD *thd) /* Bail out if DB snapshot has not been installed. SET and SHOW commands, however, are always allowed. - + Select query is also allowed if it does not access any table. We additionally allow all other commands that do not change data in case wsrep_dirty_reads is enabled. */ @@ -2659,6 +2660,8 @@ mysql_execute_command(THD *thd) !wsrep_is_show_query(lex->sql_command) && !(thd->variables.wsrep_dirty_reads && !is_update_query(lex->sql_command)) && + !(lex->sql_command == SQLCOM_SELECT && + !all_tables) && !wsrep_node_is_ready(thd)) goto error; } From b4616c40be00c5c8a2a73d537d676d8ddb7c84cf Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Tue, 3 Jan 2017 10:45:55 +0530 Subject: [PATCH 11/74] MDEV-7955 WSREP() appears on radar in OLTP RO This commit is for optimizing WSREP(thd) macro. #define WSREP(thd) \ (WSREP_ON && wsrep && (thd && thd->variables.wsrep_on)) In this we can safely remove wsrep and thd. We are not removing WSREP_ON because this will change WSREP(thd) behaviour. Patch Credit:- Nirbhay Choubay, Sergey Vojtovich --- sql/sql_parse.cc | 2 +- sql/wsrep_mysqld.h | 2 +- storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/trx/trx0trx.cc | 2 +- storage/xtradb/handler/ha_innodb.cc | 2 +- storage/xtradb/trx/trx0trx.cc | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 5717c3dd6c423..4f630b6826425 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2624,7 +2624,7 @@ mysql_execute_command(THD *thd) } /* endif unlikely slave */ #endif #ifdef WITH_WSREP - if (WSREP(thd)) + if (wsrep && WSREP(thd)) { /* change LOCK TABLE WRITE to transaction diff --git a/sql/wsrep_mysqld.h b/sql/wsrep_mysqld.h index cf549ffe544c9..fc4b2a452fe55 100644 --- a/sql/wsrep_mysqld.h +++ b/sql/wsrep_mysqld.h @@ -179,7 +179,7 @@ extern wsrep_seqno_t wsrep_locked_seqno; strcmp(wsrep_provider, WSREP_NONE)) #define WSREP(thd) \ - (WSREP_ON && wsrep && (thd && thd->variables.wsrep_on)) + (WSREP_ON && thd->variables.wsrep_on) #define WSREP_CLIENT(thd) \ (WSREP(thd) && thd->wsrep_client_thread) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 3713dd959d45d..b283bbda7f49a 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4103,7 +4103,7 @@ innobase_commit_low( #ifdef WITH_WSREP THD* thd = (THD*)trx->mysql_thd; const char* tmp = 0; - if (wsrep_on(thd)) { + if (thd && wsrep_on(thd)) { #ifdef WSREP_PROC_INFO char info[64]; info[sizeof(info) - 1] = '\0'; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index cde3244876937..11025b4c21cf6 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -1349,7 +1349,7 @@ trx_commit_in_memory( ut_ad(!trx->in_rw_trx_list); #ifdef WITH_WSREP - if (wsrep_on(trx->mysql_thd)) { + if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { trx->lock.was_chosen_as_deadlock_victim = FALSE; } #endif diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 8d564df2bb3eb..8f51b9310383b 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4742,7 +4742,7 @@ innobase_commit_low( #ifdef WITH_WSREP THD* thd = (THD*)trx->mysql_thd; const char* tmp = 0; - if (wsrep_on(thd)) { + if (thd && wsrep_on(thd)) { #ifdef WSREP_PROC_INFO char info[64]; info[sizeof(info) - 1] = '\0'; diff --git a/storage/xtradb/trx/trx0trx.cc b/storage/xtradb/trx/trx0trx.cc index ec57a8e5c5424..bdf407ff7fb0a 100644 --- a/storage/xtradb/trx/trx0trx.cc +++ b/storage/xtradb/trx/trx0trx.cc @@ -1572,7 +1572,7 @@ trx_commit_in_memory( ut_ad(!trx->in_rw_trx_list); #ifdef WITH_WSREP - if (wsrep_on(trx->mysql_thd)) { + if (trx->mysql_thd && wsrep_on(trx->mysql_thd)) { trx->lock.was_chosen_as_deadlock_victim = FALSE; } #endif From 4c610d10d400938481e418d8798279d61ff41d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 09:44:44 +0200 Subject: [PATCH 12/74] Post-fix for MDEV-11195 NUMA does not get enabled even when checks are passed The C preprocessor symbol WITH_NUMA is never defined. Instead, the symbol HAVE_LIBNUMA is used for checking if the feature is to be used. If cmake -DWITH_NUMA=OFF is specified, HAVE_LIBNUMA will not be defined at compilation time even if the library is available. If cmake -DWITH_NUMA=ON is specified but the library is not available at configuration time, the compilation will be aborted. --- mysql-test/suite/sys_vars/r/sysvars_innodb.result | 1 + .../suite/sys_vars/t/innodb_numa_interleave_basic.test | 1 - mysql-test/suite/sys_vars/t/sysvars_innodb.test | 1 + storage/innobase/buf/buf0buf.cc | 10 +++++----- storage/innobase/handler/ha_innodb.cc | 10 +++++----- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 87e000faf025f..ad6dcc1bb6438 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -2,6 +2,7 @@ select * from information_schema.system_variables where variable_name like 'innodb%' and variable_name not in ( 'innodb_disallow_writes', # only available WITH_WSREP +'innodb_numa_interleave', # only available WITH_NUMA 'innodb_sched_priority_cleaner', # linux only 'innodb_use_native_aio') # default value depends on OS order by variable_name; diff --git a/mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test b/mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test index 518b5ebba1774..de89784f7e4a6 100644 --- a/mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test @@ -1,6 +1,5 @@ --source include/have_innodb.inc --source include/have_numa.inc ---source include/have_64bit.inc SELECT @@GLOBAL.innodb_numa_interleave; diff --git a/mysql-test/suite/sys_vars/t/sysvars_innodb.test b/mysql-test/suite/sys_vars/t/sysvars_innodb.test index bd8442b6a4438..38f248cb61138 100644 --- a/mysql-test/suite/sys_vars/t/sysvars_innodb.test +++ b/mysql-test/suite/sys_vars/t/sysvars_innodb.test @@ -9,6 +9,7 @@ select * from information_schema.system_variables where variable_name like 'innodb%' and variable_name not in ( 'innodb_disallow_writes', # only available WITH_WSREP + 'innodb_numa_interleave', # only available WITH_NUMA 'innodb_sched_priority_cleaner', # linux only 'innodb_use_native_aio') # default value depends on OS order by variable_name; diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 935b7fadafa54..638126f8de186 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -85,7 +85,7 @@ Created 11/5/1995 Heikki Tuuri #include "lzo/lzo1x.h" #endif -#if defined(HAVE_LIBNUMA) && defined(WITH_NUMA) +#ifdef HAVE_LIBNUMA #include #include struct set_numa_interleave_t @@ -126,7 +126,7 @@ struct set_numa_interleave_t #define NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE set_numa_interleave_t scoped_numa #else #define NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE -#endif /* HAVE_LIBNUMA && WITH_NUMA */ +#endif /* HAVE_LIBNUMA */ /* Enable this for checksum error messages. */ //#ifdef UNIV_DEBUG @@ -1608,7 +1608,7 @@ buf_chunk_init( return(NULL); } -#if defined(HAVE_LIBNUMA) && defined(WITH_NUMA) +#ifdef HAVE_LIBNUMA if (srv_numa_interleave) { struct bitmask *numa_mems_allowed = numa_get_mems_allowed(); int st = mbind(chunk->mem, chunk->mem_size(), @@ -1622,7 +1622,7 @@ buf_chunk_init( " (error: " << strerror(errno) << ")."; } } -#endif /* HAVE_LIBNUMA && WITH_NUMA */ +#endif /* HAVE_LIBNUMA */ /* Allocate the block descriptors from diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 6a6a049d5a6af..9a278143945e2 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -22724,12 +22724,12 @@ static MYSQL_SYSVAR_BOOL(use_native_aio, srv_use_native_aio, "Use native AIO if supported on this platform.", NULL, NULL, TRUE); -#if defined(HAVE_LIBNUMA) && defined(WITH_NUMA) +#ifdef HAVE_LIBNUMA static MYSQL_SYSVAR_BOOL(numa_interleave, srv_numa_interleave, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, "Use NUMA interleave memory policy to allocate InnoDB buffer pool.", NULL, NULL, FALSE); -#endif /* HAVE_LIBNUMA && WITH_NUMA */ +#endif /* HAVE_LIBNUMA */ static MYSQL_SYSVAR_BOOL(api_enable_binlog, ib_binlog_enabled, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, @@ -23276,9 +23276,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(autoinc_lock_mode), MYSQL_SYSVAR(version), MYSQL_SYSVAR(use_native_aio), -#if defined(HAVE_LIBNUMA) && defined(WITH_NUMA) +#ifdef HAVE_LIBNUMA MYSQL_SYSVAR(numa_interleave), -#endif /* HAVE_LIBNUMA && WITH_NUMA */ +#endif /* HAVE_LIBNUMA */ MYSQL_SYSVAR(change_buffering), MYSQL_SYSVAR(change_buffer_max_size), #if defined UNIV_DEBUG || defined UNIV_IBUF_DEBUG From 403f6e96070a3f35745d4d59d12e222d21ac4f38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 3 Jan 2017 11:22:09 +0200 Subject: [PATCH 13/74] MDEV-11705: InnoDB: Failing assertion: (&log_sys->mutex)->is_owned() if server started with innodb-scrub-log Problem was that log_scrub function did not take required log_sys mutex. Background: Unused space in log blocks are padded with MLOG_DUMMY_RECORD if innodb-scrub-log is enabled. As log files are written on circular fashion old log blocks can be reused later for new redo-log entries. Scrubbing pads unused space in log blocks to avoid visibility of the possible old redo-log contents. log_scrub(): Take log_sys mutex log_pad_current_log_block(): Increase srv_stats.n_log_scrubs if padding is done. srv0srv.cc: Set srv_stats.n_log_scrubs to export vars innodb_scrub_log ha_innodb.cc: Export innodb_scrub_log to global status. --- .../suite/encryption/r/innodb-scrub-log.result | 3 +++ mysql-test/suite/encryption/t/innodb-scrub-log.opt | 1 + mysql-test/suite/encryption/t/innodb-scrub-log.test | 13 +++++++++++++ storage/innobase/handler/ha_innodb.cc | 3 +++ storage/innobase/include/srv0srv.h | 6 +++++- storage/innobase/log/log0log.cc | 8 +++++++- storage/innobase/srv/srv0srv.cc | 1 + 7 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/encryption/r/innodb-scrub-log.result create mode 100644 mysql-test/suite/encryption/t/innodb-scrub-log.opt create mode 100644 mysql-test/suite/encryption/t/innodb-scrub-log.test diff --git a/mysql-test/suite/encryption/r/innodb-scrub-log.result b/mysql-test/suite/encryption/r/innodb-scrub-log.result new file mode 100644 index 0000000000000..8ea7500036834 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-scrub-log.result @@ -0,0 +1,3 @@ +create table t1(a int not null primary key auto_increment, +b varchar(200), c char(100), d varchar(150)) engine=innodb; +DROP TABLE t1; diff --git a/mysql-test/suite/encryption/t/innodb-scrub-log.opt b/mysql-test/suite/encryption/t/innodb-scrub-log.opt new file mode 100644 index 0000000000000..0078065bbf724 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-scrub-log.opt @@ -0,0 +1 @@ +--loose-innodb-scrub-log=on diff --git a/mysql-test/suite/encryption/t/innodb-scrub-log.test b/mysql-test/suite/encryption/t/innodb-scrub-log.test new file mode 100644 index 0000000000000..36ecc88b99a10 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-scrub-log.test @@ -0,0 +1,13 @@ +--source include/have_innodb.inc + +# +# MDEV-11705: InnoDB: Failing assertion: (&log_sys->mutex)->is_owned() if server started with innodb-scrub-log +# + +create table t1(a int not null primary key auto_increment, +b varchar(200), c char(100), d varchar(150)) engine=innodb; + +let $wait_condition= SELECT variable_value FROM information_schema.global_status WHERE variable_name = 'innodb_scrub_log'; +--source include/wait_condition.inc + +DROP TABLE t1; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 9a278143945e2..aadc7155327bb 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1273,6 +1273,9 @@ static SHOW_VAR innodb_status_variables[]= { {"scrub_background_page_split_failures_unknown", (char*) &export_vars.innodb_scrub_page_split_failures_unknown, SHOW_LONG}, + {"scrub_log", + (char*) &export_vars.innodb_scrub_log, + SHOW_LONGLONG}, {"encryption_num_key_requests", (char*) &export_vars.innodb_encryption_key_requests, SHOW_LONGLONG}, diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 92572a0f8b35e..e04726ea837a8 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -194,6 +194,9 @@ struct srv_stats_t { /** Number of encryption_get_latest_key_version calls */ ulint_ctr_64_t n_key_requests; + + /** Number of log scrub operations */ + ulint_ctr_64_t n_log_scrubs; }; extern const char* srv_main_thread_op_info; @@ -1118,6 +1121,7 @@ struct export_var_t{ ulint innodb_scrub_page_split_failures_out_of_filespace; ulint innodb_scrub_page_split_failures_missing_index; ulint innodb_scrub_page_split_failures_unknown; + int64_t innodb_scrub_log; }; /** Thread slot in the thread table. */ diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 979a40e22fe99..c9ea1b10b06fe 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2552,6 +2552,10 @@ log_pad_current_log_block(void) pad_length = 0; } + if (pad_length) { + srv_stats.n_log_scrubs.inc(); + } + for (i = 0; i < pad_length; i++) { log_write_low(&b, 1); } @@ -2571,6 +2575,7 @@ void log_scrub() /*=========*/ { + log_mutex_enter(); ulint cur_lbn = log_block_convert_lsn_to_no(log_sys->lsn); if (next_lbn_to_pad == cur_lbn) @@ -2579,6 +2584,7 @@ log_scrub() } next_lbn_to_pad = log_block_convert_lsn_to_no(log_sys->lsn); + log_mutex_exit(); } /* log scrubbing speed, in bytes/sec */ diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 0eda03f7fd7a2..3df86e25b552c 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -1705,6 +1705,7 @@ srv_export_innodb_status(void) scrub_stat.page_split_failures_missing_index; export_vars.innodb_scrub_page_split_failures_unknown = scrub_stat.page_split_failures_unknown; + export_vars.innodb_scrub_log = srv_stats.n_log_scrubs; mutex_exit(&srv_innodb_monitor_mutex); } From 509e7773eccfcca2577e980b543f38fe4f0ed4cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 12:09:14 +0200 Subject: [PATCH 14/74] MDEV-11695 Define a reasonable upper limit for innodb_spin_wait_delay The upper limit of innodb_spin_wait_delay was ~0UL. It does not make any sense to wait more than a few dozens of microseconds between attempts to acquire a busy mutex. Make the new upper limit 6000. ut_delay(6000) could correspond to several milliseconds even today. --- storage/innobase/handler/ha_innodb.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index aadc7155327bb..9da1e6bcd05b7 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -22615,7 +22615,7 @@ static MYSQL_SYSVAR_ULONG(sync_spin_loops, srv_n_spin_wait_rounds, static MYSQL_SYSVAR_ULONG(spin_wait_delay, srv_spin_wait_delay, PLUGIN_VAR_OPCMDARG, "Maximum delay between polling for a spin lock (6 by default)", - NULL, NULL, 6L, 0L, ~0UL, 0); + NULL, NULL, 6, 0, 6000, 0); static MYSQL_SYSVAR_ULONG(thread_concurrency, srv_thread_concurrency, PLUGIN_VAR_RQCMDARG, From fc779252aec56e01c5924d80378cafc8b6ec80b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 13:18:47 +0200 Subject: [PATCH 15/74] MDEV-11688 fil_crypt_threads_end() tries to create threads after aborted InnoDB startup This bug was repeatable by starting MariaDB 10.2 with an invalid option, such as --innodb-flush-method=foo. It is not repeatable in MariaDB 10.1 in the same way, but the problem exists already there. --- storage/innobase/fil/fil0crypt.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 6483cdc5a5396..6482956eb0a35 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2465,8 +2465,9 @@ void fil_crypt_threads_end() /*===================*/ { - /* stop threads */ - fil_crypt_set_thread_cnt(0); + if (fil_crypt_threads_inited) { + fil_crypt_set_thread_cnt(0); + } } /********************************************************************* From 8a04b8cadedb4b7388379a30b7d7211a2b607bfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 13:18:47 +0200 Subject: [PATCH 16/74] MDEV-11688 fil_crypt_threads_end() tries to create threads after aborted InnoDB startup This bug was repeatable by starting MariaDB 10.2 with an invalid option, such as --innodb-flush-method=foo. It is not repeatable in MariaDB 10.1 in the same way, but the problem exists already there. --- storage/innobase/fil/fil0crypt.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index fd1cc24c92a45..708dbd0fd01f5 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2458,8 +2458,9 @@ void fil_crypt_threads_end() /*===================*/ { - /* stop threads */ - fil_crypt_set_thread_cnt(0); + if (fil_crypt_threads_inited) { + fil_crypt_set_thread_cnt(0); + } } /********************************************************************* From a0d396fd3f5f86e09f602ac9f76b8d9f2e739221 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 3 Jan 2017 14:35:08 +0200 Subject: [PATCH 17/74] MDEV-11684: post-10.1-merge fixes 10.1 is merged into 10.2 now. Two issues are left to fix: (1) encryption.innochecksum test (2) read_page0 vs page_0_crypt_read (1) innochecksum tool did not compile after merge because buf_page_is_corrupted uses fil_crypt_t that has been changed. extra/CMakeLists.txt: Added fil/fil0crypt.cc as dependency as we need to use fil_crypt_verify_checksum for encrypted pages. innochecksum.cc: If we think page is encrypted i.e. FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION != 0 we call fil_crypt_verify_checksum() function to compare calculated checksum to stored checksum calculated after encryption (this is stored on different offset i.e. FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4). If checksum does not match we call normal buf_page_is_corrupted to compare calculated checksum to stored checksum. fil0crypt.cc: add #ifdef UNIV_INNOCHECKSUM to be able to compile this file for innochecksum tool. (2) read_page0 is not needed and thus removed. --- extra/CMakeLists.txt | 1 + extra/innochecksum.cc | 42 +++++- mysql-test/suite/encryption/disabled.def | 2 +- storage/innobase/buf/buf0buf.cc | 8 +- storage/innobase/fil/fil0crypt.cc | 173 +++++++++++++---------- storage/innobase/fil/fil0fil.cc | 5 +- storage/innobase/include/fil0fil.h | 5 +- 7 files changed, 146 insertions(+), 90 deletions(-) diff --git a/extra/CMakeLists.txt b/extra/CMakeLists.txt index ee696c156f936..480cd6e7fbb9e 100644 --- a/extra/CMakeLists.txt +++ b/extra/CMakeLists.txt @@ -80,6 +80,7 @@ IF(WITH_INNOBASE_STORAGE_ENGINE OR WITH_XTRADB_STORAGE_ENGINE) ../storage/innobase/buf/buf0buf.cc ../storage/innobase/page/page0zip.cc ../storage/innobase/os/os0file.cc + ../storage/innobase/fil/fil0crypt.cc ) diff --git a/extra/innochecksum.cc b/extra/innochecksum.cc index ce631c139527b..9d02a545cc5db 100644 --- a/extra/innochecksum.cc +++ b/extra/innochecksum.cc @@ -1,6 +1,6 @@ /* Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved. - Copyright (c) 2014, 2016, MariaDB Corporation. + Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -70,6 +70,24 @@ The parts not included are excluded by #ifndef UNIV_INNOCHECKSUM. */ #define PRIuMAX "llu" #endif +/********************************************************************* +Verify checksum for a page (iff it's encrypted) +NOTE: currently this function can only be run in single threaded mode +as it modifies srv_checksum_algorithm (temporarily) +@param[in] src_fame page to verify +@param[in] page_size page_size +@param[in] page_no page number of given read_buf +@param[in] strict_check true if strict-check option is enabled +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( +/*============================*/ + const byte* src_frame, /*!< in: page the verify */ + const page_size_t& page_size /*!< in: page size */ + ,uintmax_t page_no, + bool strict_check); + /* Global variables */ static bool verbose; static bool just_count; @@ -564,9 +582,25 @@ is_page_corrupted( } } - is_corrupted = buf_page_is_corrupted( - true, buf, page_size, false, cur_page_num, strict_verify, - is_log_enabled, log_file); + /* If page is encrypted, use different checksum calculation + as innochecksum can't decrypt pages. Note that some old InnoDB + versions did not initialize FIL_PAGE_FILE_FLUSH_LSN field + so if crypt checksum does not match we verify checksum using + normal method. + */ + if (mach_read_from_4(buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0) { + is_corrupted = fil_space_verify_crypt_checksum(buf, page_size, + cur_page_num, strict_verify); + } else { + is_corrupted = true; + } + + if (is_corrupted) { + is_corrupted = buf_page_is_corrupted( + true, buf, page_size, false, + cur_page_num, strict_verify, + is_log_enabled, log_file); + } return(is_corrupted); } diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def index f8d7fb518655c..c4394b269471f 100644 --- a/mysql-test/suite/encryption/disabled.def +++ b/mysql-test/suite/encryption/disabled.def @@ -13,4 +13,4 @@ innodb_scrub : MDEV-8139 innodb_scrub_compressed : MDEV-8139 innodb_scrub_background : MDEV-8139 -innochecksum: see buf_page_is_corrupted() + diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index 638126f8de186..59b3ee4ca7eaa 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -846,7 +846,7 @@ buf_page_is_corrupted( ulint checksum_field2; bool page_encrypted = false; -#ifndef UNIV_INNOCHECKSUM // FIXME see also encryption.innochecksum test +#ifndef UNIV_INNOCHECKSUM ulint space_id = mach_read_from_4( read_buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); fil_space_crypt_t* crypt_data = fil_space_get_crypt_data(space_id); @@ -859,6 +859,12 @@ buf_page_is_corrupted( fil_page_is_encrypted(read_buf)) { page_encrypted = true; } +#else + if (mach_read_from_4(read_buf+FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION) != 0 + || mach_read_from_2(read_buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_COMPRESSED_ENCRYPTED) { + page_encrypted = true; + } + #endif DBUG_EXECUTE_IF("buf_page_is_corrupt_failure", return(TRUE); ); diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 708dbd0fd01f5..ff30b3a09e986 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -24,14 +24,16 @@ Modified Jan Lindström jan.lindstrom@mariadb.com *******************************************************/ #include "fil0fil.h" +#include "mach0data.h" +#include "page0size.h" +#include "page0zip.h" +#ifndef UNIV_INNOCHECKSUM #include "fil0crypt.h" #include "srv0srv.h" #include "srv0start.h" -#include "mach0data.h" #include "log0recv.h" #include "mtr0mtr.h" #include "mtr0log.h" -#include "page0zip.h" #include "ut0ut.h" #include "btr0scrub.h" #include "fsp0fsp.h" @@ -910,81 +912,6 @@ fil_crypt_calculate_checksum( return checksum; } -/********************************************************************* -Verify checksum for a page (iff it's encrypted) -NOTE: currently this function can only be run in single threaded mode -as it modifies srv_checksum_algorithm (temporarily) -@return true if page is encrypted AND OK, false otherwise */ -UNIV_INTERN -bool -fil_space_verify_crypt_checksum( -/*============================*/ - const byte* src_frame, /*!< in: page the verify */ - const page_size_t& page_size) /*!< in: page size */ -{ - // key version - uint key_version = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); - - if (key_version == 0) { - return false; // unencrypted page - } - - /* "trick" the normal checksum routines by storing the post-encryption - * checksum into the normal checksum field allowing for reuse of - * the normal routines */ - - // post encryption checksum - ib_uint32_t stored_post_encryption = mach_read_from_4( - src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); - - // save pre encryption checksum for restore in end of this function - ib_uint32_t stored_pre_encryption = mach_read_from_4( - src_frame + FIL_PAGE_SPACE_OR_CHKSUM); - - ib_uint32_t checksum_field2 = mach_read_from_4( - src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); - - /** prepare frame for usage of normal checksum routines */ - mach_write_to_4(const_cast(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM, - stored_post_encryption); - - /* NOTE: this function is (currently) only run when restoring - * dblwr-buffer, server is single threaded so it's safe to modify - * srv_checksum_algorithm */ - srv_checksum_algorithm_t save_checksum_algorithm = - (srv_checksum_algorithm_t)srv_checksum_algorithm; - - if (!page_size.is_compressed() && - (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || - save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) { - /* handle ALGORITHM_INNODB specially, - * "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC - * checksum_field2 is sort of pointless anyway... - */ - srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; - mach_write_to_4(const_cast(src_frame) + - UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, - BUF_NO_CHECKSUM_MAGIC); - } - - /* verify checksums */ - ibool corrupted = buf_page_is_corrupted(false, src_frame, page_size, false); - - /** restore frame & algorithm */ - srv_checksum_algorithm = save_checksum_algorithm; - - mach_write_to_4(const_cast(src_frame) + - FIL_PAGE_SPACE_OR_CHKSUM, - stored_pre_encryption); - - mach_write_to_4(const_cast(src_frame) + - UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, - checksum_field2); - - return (!corrupted); -} - /***********************************************************************/ /** A copy of global key state */ @@ -2671,3 +2598,93 @@ fil_space_get_scrub_status( return crypt_data == NULL ? 1 : 0; } +#endif /* UNIV_INNOCHECKSUM */ + +/********************************************************************* +Verify checksum for a page (iff it's encrypted) +NOTE: currently this function can only be run in single threaded mode +as it modifies srv_checksum_algorithm (temporarily) +@param[in] src_fame page to verify +@param[in] page_size page_size +@param[in] page_no page number of given read_buf +@param[in] strict_check true if strict-check option is enabled +@return true if page is encrypted AND OK, false otherwise */ +UNIV_INTERN +bool +fil_space_verify_crypt_checksum( +/*============================*/ + const byte* src_frame, /*!< in: page the verify */ + const page_size_t& page_size /*!< in: page size */ +#ifdef UNIV_INNOCHECKSUM + ,uintmax_t page_no, + bool strict_check +#endif /* UNIV_INNOCHECKSUM */ +) +{ + // key version + uint key_version = mach_read_from_4( + src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION); + + if (key_version == 0) { + return false; // unencrypted page + } + + /* "trick" the normal checksum routines by storing the post-encryption + * checksum into the normal checksum field allowing for reuse of + * the normal routines */ + + // post encryption checksum + ib_uint32_t stored_post_encryption = mach_read_from_4( + src_frame + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + 4); + + // save pre encryption checksum for restore in end of this function + ib_uint32_t stored_pre_encryption = mach_read_from_4( + src_frame + FIL_PAGE_SPACE_OR_CHKSUM); + + ib_uint32_t checksum_field2 = mach_read_from_4( + src_frame + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM); + + /** prepare frame for usage of normal checksum routines */ + mach_write_to_4(const_cast(src_frame) + FIL_PAGE_SPACE_OR_CHKSUM, + stored_post_encryption); + + /* NOTE: this function is (currently) only run when restoring + * dblwr-buffer, server is single threaded so it's safe to modify + * srv_checksum_algorithm */ + srv_checksum_algorithm_t save_checksum_algorithm = + (srv_checksum_algorithm_t)srv_checksum_algorithm; + + if (!page_size.is_compressed() && + (save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB || + save_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB)) { + /* handle ALGORITHM_INNODB specially, + * "downgrade" to ALGORITHM_INNODB and store BUF_NO_CHECKSUM_MAGIC + * checksum_field2 is sort of pointless anyway... + */ + srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; + mach_write_to_4(const_cast(src_frame) + + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, + BUF_NO_CHECKSUM_MAGIC); + } + + /* verify checksums */ + bool corrupted = buf_page_is_corrupted(false, src_frame, + page_size, false +#ifdef UNIV_INNOCHECKSUM + ,page_no, strict_check, false, NULL +#endif /* UNIV_INNOCHECKSUM */ + ); + + /** restore frame & algorithm */ + srv_checksum_algorithm = save_checksum_algorithm; + + mach_write_to_4(const_cast(src_frame) + + FIL_PAGE_SPACE_OR_CHKSUM, + stored_pre_encryption); + + mach_write_to_4(const_cast(src_frame) + + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM, + checksum_field2); + + return (!corrupted); +} diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 60ed4e014224e..43b3100230698 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1718,13 +1718,14 @@ fil_space_create( fil_system->max_assigned_id = id; } +#ifdef UNIV_DEBUG if (crypt_data) { - space->read_page0 = true; /* If table could be encrypted print info */ ib::info() << "Tablespace ID " << id << " name " << space->name << ":" << fil_crypt_get_mode(crypt_data) << " " << fil_crypt_get_type(crypt_data); } +#endif mutex_exit(&fil_system->mutex); diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 1e07c4f8fb534..30d79a52b4d8f 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -190,9 +190,6 @@ struct fil_space_t { /** True if we have already printed compression failure */ bool printed_compression_failure; - /** True if page 0 of tablespace is read */ - bool read_page0; - /** Release the reserved free extents. @param[in] n_reserved number of reserved extents */ void release_free_extents(ulint n_reserved); From ba8198a34cf651cce8a1f59f61ba7f31dbe41579 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 15:44:44 +0200 Subject: [PATCH 18/74] Post-fix for MDEV-11688 fil_crypt_threads_end() tries to create threads fil_crypt_threads_cleanup(): Do nothing if nothing was initialized. --- storage/innobase/fil/fil0crypt.cc | 3 +++ storage/xtradb/fil/fil0crypt.cc | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 6482956eb0a35..1eaabf6867888 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2477,6 +2477,9 @@ void fil_crypt_threads_cleanup() /*=======================*/ { + if (!fil_crypt_threads_inited) { + return; + } os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 6483cdc5a5396..1eaabf6867888 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -2465,8 +2465,9 @@ void fil_crypt_threads_end() /*===================*/ { - /* stop threads */ - fil_crypt_set_thread_cnt(0); + if (fil_crypt_threads_inited) { + fil_crypt_set_thread_cnt(0); + } } /********************************************************************* @@ -2476,6 +2477,9 @@ void fil_crypt_threads_cleanup() /*=======================*/ { + if (!fil_crypt_threads_inited) { + return; + } os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); From a758479c103f513077a0b90a9285b17229ee8289 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 15:44:44 +0200 Subject: [PATCH 19/74] Post-fix for MDEV-11688 fil_crypt_threads_end() tries to create threads fil_crypt_threads_cleanup(): Do nothing if nothing was initialized. --- storage/innobase/fil/fil0crypt.cc | 3 +++ storage/xtradb/fil/fil0crypt.cc | 8 ++++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index ff30b3a09e986..a0c9fbf57ac8a 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2397,6 +2397,9 @@ void fil_crypt_threads_cleanup() /*=======================*/ { + if (!fil_crypt_threads_inited) { + return; + } os_event_destroy(fil_crypt_event); os_event_destroy(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 3670f0fac4f66..6252a5470bdc3 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -2465,8 +2465,9 @@ void fil_crypt_threads_end() /*===================*/ { - /* stop threads */ - fil_crypt_set_thread_cnt(0); + if (fil_crypt_threads_inited) { + fil_crypt_set_thread_cnt(0); + } } /********************************************************************* @@ -2476,6 +2477,9 @@ void fil_crypt_threads_cleanup() /*=======================*/ { + if (!fil_crypt_threads_inited) { + return; + } os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); From 80d5d1452a4a6b7bd1627116f2a5a950003fc3cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 19:32:47 +0200 Subject: [PATCH 20/74] MDEV-11694 InnoDB tries to create unused table SYS_ZIP_DICT MariaDB Server 10.0.28 and 10.1.19 merged code from Percona XtraDB that introduced support for compressed columns. Much but not all of this code was disabled by placing #ifdef HAVE_PERCONA_COMPRESSED_COLUMNS around it. Among the unused but not disabled code is code to access some new system tables related to compressed columns. The creation of these system tables SYS_ZIP_DICT and SYS_ZIP_DICT_COLS would cause a crash in --innodb-read-only mode when upgrading from an earlier version to 10.0.28 or 10.1.19. Let us remove all the dead code related to compressed columns. Users who already upgraded to 10.0.28 and 10.1.19 will have the two above mentioned empty tables in their InnoDB system tablespace. Subsequent versions of MariaDB Server will completely ignore those tables. --- storage/xtradb/dict/dict0boot.cc | 5 +- storage/xtradb/dict/dict0crea.cc | 583 +----------------------- storage/xtradb/dict/dict0dict.cc | 160 +------ storage/xtradb/dict/dict0load.cc | 160 +------ storage/xtradb/handler/ha_innodb.cc | 415 +---------------- storage/xtradb/handler/ha_innodb.h | 38 +- storage/xtradb/handler/handler0alter.cc | 62 +-- storage/xtradb/handler/xtradb_i_s.cc | 330 +------------- storage/xtradb/handler/xtradb_i_s.h | 3 +- storage/xtradb/include/data0type.h | 15 +- storage/xtradb/include/data0type.ic | 16 +- storage/xtradb/include/dict0boot.h | 33 +- storage/xtradb/include/dict0boot.ic | 3 +- storage/xtradb/include/dict0crea.h | 90 +--- storage/xtradb/include/dict0dict.h | 48 +- storage/xtradb/include/dict0load.h | 29 +- storage/xtradb/include/row0mysql.h | 85 +--- storage/xtradb/rem/rem0rec.cc | 18 +- storage/xtradb/row/row0merge.cc | 7 +- storage/xtradb/row/row0mysql.cc | 583 +----------------------- storage/xtradb/row/row0sel.cc | 39 +- storage/xtradb/srv/srv0start.cc | 7 +- 22 files changed, 60 insertions(+), 2669 deletions(-) diff --git a/storage/xtradb/dict/dict0boot.cc b/storage/xtradb/dict/dict0boot.cc index c0bb0298bea72..20ab5d752059e 100644 --- a/storage/xtradb/dict/dict0boot.cc +++ b/storage/xtradb/dict/dict0boot.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -272,10 +273,6 @@ dict_boot(void) ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_FOR_NAME == 2); ut_ad(DICT_NUM_COLS__SYS_FOREIGN_COLS == 4); ut_ad(DICT_NUM_FIELDS__SYS_FOREIGN_COLS == 6); - ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT == 3); - ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT == 5); - ut_ad(DICT_NUM_COLS__SYS_ZIP_DICT_COLS == 3); - ut_ad(DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS == 5); mtr_start(&mtr); diff --git a/storage/xtradb/dict/dict0crea.cc b/storage/xtradb/dict/dict0crea.cc index db0ca638de4da..9a4040421d7fb 100644 --- a/storage/xtradb/dict/dict0crea.cc +++ b/storage/xtradb/dict/dict0crea.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1932,135 +1933,6 @@ dict_create_or_check_sys_tablespace(void) return(err); } -/** Creates the zip_dict system table inside InnoDB -at server bootstrap or server start if it is not found or is -not of the right form. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -dict_create_or_check_sys_zip_dict(void) -{ - trx_t* trx; - my_bool srv_file_per_table_backup; - dberr_t err; - dberr_t sys_zip_dict_err; - dberr_t sys_zip_dict_cols_err; - - ut_a(srv_get_active_thread_type() == SRV_NONE); - - /* Note: The master thread has not been started at this point. */ - - sys_zip_dict_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); - sys_zip_dict_cols_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT_COLS", DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, - 1); - - if (sys_zip_dict_err == DB_SUCCESS && - sys_zip_dict_cols_err == DB_SUCCESS) - return (DB_SUCCESS); - - trx = trx_allocate_for_mysql(); - - trx_set_dict_operation(trx, TRX_DICT_OP_TABLE); - - trx->op_info = "creating zip_dict and zip_dict_cols sys tables"; - - row_mysql_lock_data_dictionary(trx); - - /* Check which incomplete table definition to drop. */ - - if (sys_zip_dict_err == DB_CORRUPTION) { - ib_logf(IB_LOG_LEVEL_WARN, - "Dropping incompletely created " - "SYS_ZIP_DICT table."); - row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE, TRUE); - } - if (sys_zip_dict_cols_err == DB_CORRUPTION) { - ib_logf(IB_LOG_LEVEL_WARN, - "Dropping incompletely created " - "SYS_ZIP_DICT_COLS table."); - row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE, TRUE); - } - - ib_logf(IB_LOG_LEVEL_INFO, - "Creating zip_dict and zip_dict_cols system tables."); - - /* We always want SYSTEM tables to be created inside the system - tablespace. */ - srv_file_per_table_backup = srv_file_per_table; - srv_file_per_table = 0; - - err = que_eval_sql( - NULL, - "PROCEDURE CREATE_SYS_ZIP_DICT_PROC () IS\n" - "BEGIN\n" - "CREATE TABLE SYS_ZIP_DICT(\n" - " ID INT UNSIGNED NOT NULL,\n" - " NAME CHAR(" - STRINGIFY_ARG(ZIP_DICT_MAX_NAME_LENGTH) - ") NOT NULL,\n" - " DATA BLOB NOT NULL\n" - ");\n" - "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_ID" - " ON SYS_ZIP_DICT (ID);\n" - "CREATE UNIQUE INDEX SYS_ZIP_DICT_NAME" - " ON SYS_ZIP_DICT (NAME);\n" - "CREATE TABLE SYS_ZIP_DICT_COLS(\n" - " TABLE_ID INT UNSIGNED NOT NULL,\n" - " COLUMN_POS INT UNSIGNED NOT NULL,\n" - " DICT_ID INT UNSIGNED NOT NULL\n" - ");\n" - "CREATE UNIQUE CLUSTERED INDEX SYS_ZIP_DICT_COLS_COMPOSITE" - " ON SYS_ZIP_DICT_COLS (TABLE_ID, COLUMN_POS);\n" - "END;\n", - FALSE, trx); - - if (err != DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_ERROR, - "Creation of SYS_ZIP_DICT and SYS_ZIP_DICT_COLS" - "has failed with error %lu. Tablespace is full. " - "Dropping incompletely created tables.", - (ulong) err); - - ut_a(err == DB_OUT_OF_FILE_SPACE - || err == DB_TOO_MANY_CONCURRENT_TRXS); - - row_drop_table_for_mysql("SYS_ZIP_DICT", trx, TRUE, TRUE); - row_drop_table_for_mysql("SYS_ZIP_DICT_COLS", trx, TRUE, TRUE); - - if (err == DB_OUT_OF_FILE_SPACE) { - err = DB_MUST_GET_MORE_FILE_SPACE; - } - } - - trx_commit_for_mysql(trx); - - row_mysql_unlock_data_dictionary(trx); - - trx_free_for_mysql(trx); - - srv_file_per_table = srv_file_per_table_backup; - - if (err == DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_INFO, - "zip_dict and zip_dict_cols system tables created."); - } - - /* Note: The master thread has not been started at this point. */ - /* Confirm and move to the non-LRU part of the table LRU list. */ - - sys_zip_dict_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT", DICT_NUM_FIELDS__SYS_ZIP_DICT + 1, 2); - ut_a(sys_zip_dict_err == DB_SUCCESS); - sys_zip_dict_cols_err = dict_check_if_system_table_exists( - "SYS_ZIP_DICT_COLS", - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS + 1, 1); - ut_a(sys_zip_dict_cols_err == DB_SUCCESS); - - return(err); -} - /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -2114,456 +1986,3 @@ dict_create_add_tablespace_to_dictionary( return(error); } - -/** Add a single compression dictionary definition to the SYS_ZIP_DICT -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - ut_ad(data); - - pars_info_t* info = pars_info_create(); - - pars_info_add_literal(info, "name", name, name_len, - DATA_VARCHAR, DATA_ENGLISH); - pars_info_add_literal(info, "data", data, data_len, - DATA_BLOB, DATA_BINARY_TYPE | DATA_NOT_NULL); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - " max_id INT;\n" - "DECLARE CURSOR cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " ORDER BY ID DESC;\n" - "BEGIN\n" - " max_id := 0;\n" - " OPEN cur;\n" - " FETCH cur INTO max_id;\n" - " IF (cur % NOTFOUND) THEN\n" - " max_id := 0;\n" - " END IF;\n" - " CLOSE cur;\n" - " INSERT INTO SYS_ZIP_DICT VALUES" - " (max_id + 1, :name, :data);\n" - "END;\n", - FALSE, trx); - - return error; -} - -/** Fetch callback, just stores extracted zip_dict id in the external -variable. -@return TRUE if all OK */ -static -ibool -dict_create_extract_int_aux( - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: int32 id */ -{ - sel_node_t* node = static_cast(row); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - ut_a(len == sizeof(ib_uint32_t)); - - memcpy(user_arg, dfield_get_data(dfield), sizeof(ib_uint32_t)); - - return(TRUE); -} - -/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "table_id", table_id); - pars_info_add_int4_literal(info, "column_pos", column_pos); - pars_info_add_int4_literal(info, "dict_id", dict_id); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - " INSERT INTO SYS_ZIP_DICT_COLS VALUES" - " (:table_id, :column_pos, :dict_id);\n" - "END;\n", - FALSE, trx); - return error; -} - -/** Get a single compression dictionary id for the given -(table id, column pos) pair. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(dict_id); - - pars_info_t* info = pars_info_create(); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast(&dict_id_buf ), - ULINT32_UNDEFINED); - - pars_info_add_int4_literal(info, "table_id", table_id); - pars_info_add_int4_literal(info, "column_pos", column_pos); - pars_info_bind_function( - info, "my_func", dict_create_extract_int_aux, &dict_id_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT DICT_ID FROM SYS_ZIP_DICT_COLS\n" - " WHERE TABLE_ID = :table_id AND\n" - " COLUMN_POS = :column_pos;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) - error = DB_RECORD_NOT_FOUND; - else - *dict_id = local_dict_id; - } - return error; -} - -/** Get compression dictionary id for the given name. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_name( - const char* dict_name, /*!< in: dict name */ - ulint dict_name_len, /*!< in: dict name length */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(dict_name); - ut_ad(dict_name_len); - ut_ad(dict_id); - - pars_info_t* info = pars_info_create(); - - pars_info_add_literal(info, "dict_name", dict_name, dict_name_len, - DATA_VARCHAR, DATA_ENGLISH); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast(&dict_id_buf), - ULINT32_UNDEFINED); - pars_info_bind_function( - info, "my_func", dict_create_extract_int_aux, &dict_id_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " WHERE NAME = :dict_name;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) - error = DB_RECORD_NOT_FOUND; - else - *dict_id = local_dict_id; - } - return error; -} - -/** Auxiliary enum used to indicate zip dict data extraction result code */ -enum zip_dict_info_aux_code { - zip_dict_info_success, /*!< success */ - zip_dict_info_not_found, /*!< zip dict record not found */ - zip_dict_info_oom, /*!< out of memory */ - zip_dict_info_corrupted_name, /*!< corrupted zip dict name */ - zip_dict_info_corrupted_data /*!< corrupted zip dict data */ -}; - -/** Auxiliary struct used to return zip dict info aling with result code */ -struct zip_dict_info_aux { - LEX_STRING name; /*!< zip dict name */ - LEX_STRING data; /*!< zip dict data */ - int code; /*!< result code (0 - success) */ -}; - -/** Fetch callback, just stores extracted zip_dict data in the external -variable. -@return always returns TRUE */ -static -ibool -dict_create_get_zip_dict_info_by_id_aux( - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to zip_dict_info_aux* */ -{ - sel_node_t* node = static_cast(row); - zip_dict_info_aux* result = - static_cast(user_arg); - - result->code = zip_dict_info_success; - result->name.str = 0; - result->name.length = 0; - result->data.str = 0; - result->data.length = 0; - - /* NAME field */ - que_node_t* exp = node->select_list; - ut_a(exp != 0); - - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - - if (len == UNIV_SQL_NULL) { - result->code = zip_dict_info_corrupted_name; - } - else { - result->name.str = - static_cast(my_malloc(len + 1, MYF(0))); - if (result->name.str == 0) { - result->code = zip_dict_info_oom; - } - else { - memcpy(result->name.str, data, len); - result->name.str[len] = '\0'; - result->name.length = len; - } - } - - /* DATA field */ - exp = que_node_get_next(exp); - ut_a(exp != 0); - - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_a(dtype_get_mtype(type) == DATA_BLOB); - - len = dfield_get_len(dfield); - data = dfield_get_data(dfield); - - if (len == UNIV_SQL_NULL) { - result->code = zip_dict_info_corrupted_data; - } - else { - result->data.str = - static_cast(my_malloc( - len == 0 ? 1 : len, MYF(0))); - if (result->data.str == 0) { - result->code = zip_dict_info_oom; - } - else { - memcpy(result->data.str, data, len); - result->data.length = len; - } - } - - ut_ad(que_node_get_next(exp) == 0); - - if (result->code != zip_dict_info_success) { - if (result->name.str == 0) { - mem_free(result->name.str); - result->name.str = 0; - result->name.length = 0; - } - if (result->data.str == 0) { - mem_free(result->data.str); - result->data.str = 0; - result->data.length = 0; - } - } - - return TRUE; -} - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory for name and data on success. -Must be freed with mem_free(). -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_info_by_id( - ulint dict_id, /*!< in: dict id */ - char** name, /*!< out: dict name */ - ulint* name_len, /*!< out: dict name length*/ - char** data, /*!< out: dict data */ - ulint* data_len, /*!< out: dict data length*/ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - ut_ad(data); - - zip_dict_info_aux rec; - rec.code = zip_dict_info_not_found; - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "id", dict_id); - pars_info_bind_function( - info, "my_func", dict_create_get_zip_dict_info_by_id_aux, - &rec); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR cur IS\n" - " SELECT NAME, DATA FROM SYS_ZIP_DICT\n" - " WHERE ID = :id;\n" - "BEGIN\n" - " OPEN cur;\n" - " FETCH cur INTO my_func();\n" - " CLOSE cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - switch (rec.code) { - case zip_dict_info_success: - *name = rec.name.str; - *name_len = rec.name.length; - *data = rec.data.str; - *data_len = rec.data.length; - break; - case zip_dict_info_not_found: - error = DB_RECORD_NOT_FOUND; - break; - case zip_dict_info_oom: - error = DB_OUT_OF_MEMORY; - break; - case zip_dict_info_corrupted_name: - case zip_dict_info_corrupted_data: - error = DB_INVALID_NULL; - break; - default: - ut_error; - } - } - return error; -} - -/** Remove a single compression dictionary from the data dictionary -tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - trx_t* trx) /*!< in/out: transaction */ -{ - ut_ad(name); - - pars_info_t* info = pars_info_create(); - - ib_uint32_t dict_id_buf; - mach_write_to_4(reinterpret_cast(&dict_id_buf), - ULINT32_UNDEFINED); - ib_uint32_t counter_buf; - mach_write_to_4(reinterpret_cast(&counter_buf), - ULINT32_UNDEFINED); - - pars_info_add_literal(info, "name", name, name_len, - DATA_VARCHAR, DATA_ENGLISH); - pars_info_bind_int4_literal(info, "dict_id", &dict_id_buf); - pars_info_bind_function(info, "find_dict_func", - dict_create_extract_int_aux, &dict_id_buf); - pars_info_bind_function(info, "count_func", - dict_create_extract_int_aux, &counter_buf); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "DECLARE FUNCTION find_dict_func;\n" - "DECLARE FUNCTION count_func;\n" - "DECLARE CURSOR dict_cur IS\n" - " SELECT ID FROM SYS_ZIP_DICT\n" - " WHERE NAME = :name\n" - " FOR UPDATE;\n" - "DECLARE CURSOR ref_cur IS\n" - " SELECT 1 FROM SYS_ZIP_DICT_COLS\n" - " WHERE DICT_ID = :dict_id;\n" - "BEGIN\n" - " OPEN dict_cur;\n" - " FETCH dict_cur INTO find_dict_func();\n" - " IF NOT (SQL % NOTFOUND) THEN\n" - " OPEN ref_cur;\n" - " FETCH ref_cur INTO count_func();\n" - " IF SQL % NOTFOUND THEN\n" - " DELETE FROM SYS_ZIP_DICT WHERE CURRENT OF dict_cur;\n" - " END IF;\n" - " CLOSE ref_cur;\n" - " END IF;\n" - " CLOSE dict_cur;\n" - "END;\n", - FALSE, trx); - if (error == DB_SUCCESS) { - ib_uint32_t local_dict_id = mach_read_from_4( - reinterpret_cast(&dict_id_buf)); - if (local_dict_id == ULINT32_UNDEFINED) { - error = DB_RECORD_NOT_FOUND; - } - else { - ib_uint32_t local_counter = mach_read_from_4( - reinterpret_cast(&counter_buf)); - if (local_counter != ULINT32_UNDEFINED) - error = DB_ROW_IS_REFERENCED; - } - } - return error; -} - -/** Remove all compression dictionary references for the given table ID from -the data dictionary tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict_references_for_table( - ulint table_id, /*!< in: table id */ - trx_t* trx) /*!< in/out: transaction */ -{ - pars_info_t* info = pars_info_create(); - - pars_info_add_int4_literal(info, "table_id", table_id); - - dberr_t error = que_eval_sql(info, - "PROCEDURE P () IS\n" - "BEGIN\n" - " DELETE FROM SYS_ZIP_DICT_COLS\n" - " WHERE TABLE_ID = :table_id;\n" - "END;\n", - FALSE, trx); - return error; -} diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index 3845138c8bf60..339e5615bea41 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -7288,161 +7288,3 @@ dict_tf_to_row_format_string( return(0); } #endif /* !UNIV_HOTBACKUP */ - -/** Insert a records into SYS_ZIP_DICT. -@retval DB_SUCCESS if OK -@retval dberr_t if the insert failed */ -UNIV_INTERN -dberr_t -dict_create_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len, /*!< in: zip_dict name length*/ - const char* data, /*!< in: zip_dict data */ - ulint data_len) /*!< in: zip_dict data length */ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - ut_ad(name); - ut_ad(data); - - rw_lock_x_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "insert zip_dict"; - trx->dict_operation_lock_mode = RW_X_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_add_zip_dict(name, name_len, data, data_len, trx); - - if (err == DB_SUCCESS) { - trx_commit_for_mysql(trx); - } - else { - trx->op_info = "rollback of internal trx on zip_dict table"; - trx_rollback_to_savepoint(trx, NULL); - ut_a(trx->error_state == DB_SUCCESS); - } - trx->op_info = ""; - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_x_unlock(&dict_operation_lock); - - return err; -} -/** Get single compression dictionary id for the given -(table id, column pos) pair. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_id_by_key( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id) /*!< out: zip_dict id */ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - rw_lock_s_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "get zip dict id by composite key"; - trx->dict_operation_lock_mode = RW_S_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_get_zip_dict_id_by_reference(table_id, column_pos, - dict_id, trx); - - trx_commit_for_mysql(trx); - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_s_unlock(&dict_operation_lock); - - return err; -} -/** Get compression dictionary info (name and data) for the given id. -Allocates memory in name->str and data->str on success. -Must be freed with mem_free(). -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_info_by_id( - ulint dict_id, /*!< in: table name */ - char** name, /*!< out: dictionary name */ - ulint* name_len, /*!< out: dictionary name length*/ - char** data, /*!< out: dictionary data */ - ulint* data_len) /*!< out: dictionary data length*/ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - rw_lock_s_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "get zip dict name and data by id"; - trx->dict_operation_lock_mode = RW_S_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_get_zip_dict_info_by_id(dict_id, name, name_len, - data, data_len, trx); - - trx_commit_for_mysql(trx); - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_s_unlock(&dict_operation_lock); - - return err; -} -/** Delete a record in SYS_ZIP_DICT with the given name. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found -@retval DB_ROW_IS_REFERENCED if in use */ -UNIV_INTERN -dberr_t -dict_drop_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len) /*!< in: zip_dict name length*/ -{ - dberr_t err = DB_SUCCESS; - trx_t* trx; - - ut_ad(name); - - rw_lock_x_lock(&dict_operation_lock); - dict_mutex_enter_for_mysql(); - - trx = trx_allocate_for_background(); - trx->op_info = "delete zip_dict"; - trx->dict_operation_lock_mode = RW_X_LATCH; - trx_start_if_not_started(trx); - - err = dict_create_remove_zip_dict(name, name_len, trx); - - if (err == DB_SUCCESS) { - trx_commit_for_mysql(trx); - } - else { - trx->op_info = "rollback of internal trx on zip_dict table"; - trx_rollback_to_savepoint(trx, NULL); - ut_a(trx->error_state == DB_SUCCESS); - } - trx->op_info = ""; - trx->dict_operation_lock_mode = 0; - trx_free_for_background(trx); - - dict_mutex_exit_for_mysql(); - rw_lock_x_unlock(&dict_operation_lock); - - return err; -} diff --git a/storage/xtradb/dict/dict0load.cc b/storage/xtradb/dict/dict0load.cc index 9ea4f4d873aac..6cbd0a3d488c8 100644 --- a/storage/xtradb/dict/dict0load.cc +++ b/storage/xtradb/dict/dict0load.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,9 +57,7 @@ static const char* SYSTEM_TABLE_NAME[] = { "SYS_FOREIGN", "SYS_FOREIGN_COLS", "SYS_TABLESPACES", - "SYS_DATAFILES", - "SYS_ZIP_DICT", - "SYS_ZIP_DICT_COLS" + "SYS_DATAFILES" }; /* If this flag is TRUE, then we will load the cluster index's (and tables') @@ -730,161 +729,6 @@ dict_process_sys_datafiles( return(NULL); } -/** This function parses a SYS_ZIP_DICT record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict( - mem_heap_t* heap, /*!< in/out: heap memory */ - ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* id, /*!< out: dict id */ - const char** name, /*!< out: dict name */ - const char** data, /*!< out: dict data */ - ulint* data_len) /*!< out: dict data length */ -{ - ulint len; - const byte* field; - - /* Initialize the output values */ - *id = ULINT_UNDEFINED; - *name = NULL; - *data = NULL; - *data_len = 0; - - if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { - return("delete-marked record in SYS_ZIP_DICT"); - } - - if (UNIV_UNLIKELY( - rec_get_n_fields_old(rec)!= DICT_NUM_FIELDS__SYS_ZIP_DICT)) { - return("wrong number of columns in SYS_ZIP_DICT record"); - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *id = mach_read_from_4(field); - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID, &len); - if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR, &len); - if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__NAME, &len); - if (UNIV_UNLIKELY(len == 0 || len == UNIV_SQL_NULL)) { - goto err_len; - } - *name = mem_heap_strdupl(heap, (char*) field, len); - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT__DATA, &len); - if (UNIV_UNLIKELY(len == UNIV_SQL_NULL)) { - goto err_len; - } - - if (rec_get_1byte_offs_flag(rec) == 0 && - rec_2_is_field_extern(rec, DICT_FLD__SYS_ZIP_DICT__DATA)) { - ut_a(len >= BTR_EXTERN_FIELD_REF_SIZE); - - if (UNIV_UNLIKELY - (!memcmp(field + len - BTR_EXTERN_FIELD_REF_SIZE, - field_ref_zero, - BTR_EXTERN_FIELD_REF_SIZE))) { - goto err_len; - } - *data = reinterpret_cast( - btr_copy_externally_stored_field(data_len, field, - zip_size, len, heap, 0)); - } - else { - *data_len = len; - *data = static_cast(mem_heap_dup(heap, field, len)); - } - - return(NULL); - -err_len: - return("incorrect column length in SYS_ZIP_DICT"); -} - -/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict_cols( - mem_heap_t* heap, /*!< in/out: heap memory */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* table_id, /*!< out: table id */ - ulint* column_pos, /*!< out: column position */ - ulint* dict_id) /*!< out: dict id */ -{ - ulint len; - const byte* field; - - /* Initialize the output values */ - *table_id = ULINT_UNDEFINED; - *column_pos = ULINT_UNDEFINED; - *dict_id = ULINT_UNDEFINED; - - if (UNIV_UNLIKELY(rec_get_deleted_flag(rec, 0))) { - return("delete-marked record in SYS_ZIP_DICT_COLS"); - } - - if (UNIV_UNLIKELY(rec_get_n_fields_old(rec) != - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS)) { - return("wrong number of columns in SYS_ZIP_DICT_COLS" - " record"); - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { -err_len: - return("incorrect column length in SYS_ZIP_DICT_COLS"); - } - *table_id = mach_read_from_4(field); - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *column_pos = mach_read_from_4(field); - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID, &len); - if (UNIV_UNLIKELY(len != DATA_TRX_ID_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - rec_get_nth_field_offs_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR, &len); - if (UNIV_UNLIKELY(len != DATA_ROLL_PTR_LEN && len != UNIV_SQL_NULL)) { - goto err_len; - } - - field = rec_get_nth_field_old( - rec, DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID, &len); - if (UNIV_UNLIKELY(len != DICT_FLD_LEN_SPACE)) { - goto err_len; - } - *dict_id = mach_read_from_4(field); - - return(NULL); -} /********************************************************************//** Determine the flags of a table as stored in SYS_TABLES.TYPE and N_COLS. @return ULINT_UNDEFINED if error, else a valid dict_table_t::flags. */ diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index d072e61a23799..ff4f218b55759 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4,7 +4,7 @@ Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -107,14 +107,6 @@ this program; if not, write to the Free Software Foundation, Inc., #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) -#ifndef HAVE_PERCONA_COMPRESSED_COLUMNS -#define COLUMN_FORMAT_TYPE_COMPRESSED 0xBADF00D -#define SQLCOM_CREATE_COMPRESSION_DICTIONARY 0xDECAF -#define SQLCOM_DROP_COMPRESSION_DICTIONARY 0xC0FFEE -#define ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST 0xDEADFACE -const static LEX_CSTRING null_lex_cstr={0,0}; -#endif - #ifdef MYSQL_DYNAMIC_PLUGIN #define tc_size 400 #define tdc_size 400 @@ -1476,30 +1468,6 @@ normalize_table_name_low( ibool set_lower_case); /* in: TRUE if we want to set name to lower case */ -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/** Creates a new compression dictionary. */ -static -handler_create_zip_dict_result -innobase_create_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len, - /*!< in/out: zip dictionary name length */ - const char* data, /*!< in: zip dictionary data */ - ulint* data_len); - /*!< in/out: zip dictionary data length */ - -/** Drops a existing compression dictionary. */ -static -handler_drop_zip_dict_result -innobase_drop_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len); - /*!< in/out: zip dictionary name length */ -#endif /*************************************************************//** Checks if buffer pool is big enough to enable backoff algorithm. InnoDB empty free list algorithm backoff requires free pages @@ -3591,10 +3559,6 @@ innobase_init( if (srv_file_per_table) innobase_hton->tablefile_extensions = ha_innobase_exts; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - innobase_hton->create_zip_dict = innobase_create_zip_dict; - innobase_hton->drop_zip_dict = innobase_drop_zip_dict; -#endif ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); #ifndef DBUG_OFF @@ -4288,90 +4252,6 @@ innobase_purge_changed_page_bitmaps( return (my_bool)log_online_purge_changed_page_bitmaps(lsn); } -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/** Creates a new compression dictionary. */ -static -handler_create_zip_dict_result -innobase_create_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len, - /*!< in/out: zip dictionary name length */ - const char* data, /*!< in: zip dictionary data */ - ulint* data_len) - /*!< in/out: zip dictionary data length */ -{ - handler_create_zip_dict_result result = - HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; - - DBUG_ENTER("innobase_create_zip_dict"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (UNIV_UNLIKELY(high_level_read_only)) { - DBUG_RETURN(HA_CREATE_ZIP_DICT_READ_ONLY); - } - - if (UNIV_UNLIKELY(*name_len > ZIP_DICT_MAX_NAME_LENGTH)) { - *name_len = ZIP_DICT_MAX_NAME_LENGTH; - DBUG_RETURN(HA_CREATE_ZIP_DICT_NAME_TOO_LONG); - } - - if (UNIV_UNLIKELY(*data_len > ZIP_DICT_MAX_DATA_LENGTH)) { - *data_len = ZIP_DICT_MAX_DATA_LENGTH; - DBUG_RETURN(HA_CREATE_ZIP_DICT_DATA_TOO_LONG); - } - - switch (dict_create_zip_dict(name, *name_len, data, *data_len)) { - case DB_SUCCESS: - result = HA_CREATE_ZIP_DICT_OK; - break; - case DB_DUPLICATE_KEY: - result = HA_CREATE_ZIP_DICT_ALREADY_EXISTS; - break; - default: - ut_ad(0); - result = HA_CREATE_ZIP_DICT_UNKNOWN_ERROR; - } - DBUG_RETURN(result); -} - -/** Drops a existing compression dictionary. */ -static -handler_drop_zip_dict_result -innobase_drop_zip_dict( - handlerton* hton, /*!< in: innobase handlerton */ - THD* thd, /*!< in: handle to the MySQL thread */ - const char* name, /*!< in: zip dictionary name */ - ulint* name_len) - /*!< in/out: zip dictionary name length */ -{ - handler_drop_zip_dict_result result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; - - DBUG_ENTER("innobase_drop_zip_dict"); - DBUG_ASSERT(hton == innodb_hton_ptr); - - if (UNIV_UNLIKELY(high_level_read_only)) { - DBUG_RETURN(HA_DROP_ZIP_DICT_READ_ONLY); - } - - switch (dict_drop_zip_dict(name, *name_len)) { - case DB_SUCCESS: - result = HA_DROP_ZIP_DICT_OK; - break; - case DB_RECORD_NOT_FOUND: - result = HA_DROP_ZIP_DICT_DOES_NOT_EXIST; - break; - case DB_ROW_IS_REFERENCED: - result = HA_DROP_ZIP_DICT_IS_REFERENCED; - break; - default: - ut_ad(0); - result = HA_DROP_ZIP_DICT_UNKNOWN_ERROR; - } - DBUG_RETURN(result); -} -#endif /*****************************************************************//** Check whether this is a fake change transaction. @return TRUE if a fake change transaction */ @@ -5941,88 +5821,6 @@ innobase_build_index_translation( DBUG_RETURN(ret); } -/** This function checks if all the compression dictionaries referenced -in table->fields exist in SYS_ZIP_DICT InnoDB system table. -@return true if all referenced dictionaries exist */ -UNIV_INTERN -bool -innobase_check_zip_dicts( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - ulint* dict_ids, /*!< out: identified zip dict ids - (at least n_fields long) */ - trx_t* trx, /*!< in: transaction */ - const char** err_dict_name) /*!< out: the name of the - zip_dict which does not exist. */ -{ - DBUG_ENTER("innobase_check_zip_dicts"); - - bool res = true; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - dberr_t err = DB_SUCCESS; - const size_t n_fields = table->s->fields; - - Field* field_ptr; - for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; - ++field_idx) - { - field_ptr = table->field[field_idx]; - if (field_ptr->has_associated_compression_dictionary()) { - err = dict_create_get_zip_dict_id_by_name( - field_ptr->zip_dict_name.str, - field_ptr->zip_dict_name.length, - &dict_ids[field_idx], - trx); - ut_a(err == DB_SUCCESS || err == DB_RECORD_NOT_FOUND); - } - else { - dict_ids[field_idx] = ULINT_UNDEFINED; - } - - } - - if (err != DB_SUCCESS) { - res = false; - *err_dict_name = field_ptr->zip_dict_name.str; - } - -#endif - DBUG_RETURN(res); -} - -/** This function creates compression dictionary references in -SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info -in table->fields and provided zip dict ids. */ -UNIV_INTERN -void -innobase_create_zip_dict_references( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - table_id_t ib_table_id, /*!< in: table ID in Innodb data - dictionary */ - ulint* zip_dict_ids, /*!< in: zip dict ids - (at least n_fields long) */ - trx_t* trx) /*!< in: transaction */ -{ - DBUG_ENTER("innobase_create_zip_dict_references"); - - dberr_t err = DB_SUCCESS; - const size_t n_fields = table->s->fields; - - for (size_t field_idx = 0; err == DB_SUCCESS && field_idx < n_fields; - ++field_idx) - { - if (zip_dict_ids[field_idx] != ULINT_UNDEFINED) { - err = dict_create_add_zip_dict_reference(ib_table_id, - table->field[field_idx]->field_index, - zip_dict_ids[field_idx], trx); - ut_a(err == DB_SUCCESS); - } - } - - DBUG_VOID_RETURN; -} - /*******************************************************************//** This function uses index translation table to quickly locate the requested index structure. @@ -7289,16 +7087,7 @@ ha_innobase::store_key_val_for_row( blob_data = row_mysql_read_blob_ref(&blob_len, (byte*) (record + (ulint) get_field_offset(table, field)), - (ulint) field->pack_length(), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast( - field->zip_dict_data.str), - field->zip_dict_data.length, prebuilt); -#else - 0, 0, 0, prebuilt); -#endif + (ulint) field->pack_length()); true_len = blob_len; @@ -7613,14 +7402,6 @@ build_template_field( templ->mbminlen = dict_col_get_mbminlen(col); templ->mbmaxlen = dict_col_get_mbmaxlen(col); templ->is_unsigned = col->prtype & DATA_UNSIGNED; -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - templ->compressed = (field->column_format() - == COLUMN_FORMAT_TYPE_COMPRESSED); - templ->zip_dict_data = field->zip_dict_data; -#else - templ->compressed = 0; - templ->zip_dict_data = null_lex_cstr; -#endif if (!dict_index_is_clust(index) && templ->rec_field_no == ULINT_UNDEFINED) { @@ -8409,10 +8190,8 @@ calc_row_difference( case DATA_BLOB: /* Do not compress blob column while comparing*/ - o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len, - false, 0, 0, prebuilt); - n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len, - false, 0, 0, prebuilt); + o_ptr = row_mysql_read_blob_ref(&o_len, o_ptr, o_len); + n_ptr = row_mysql_read_blob_ref(&n_len, n_ptr, n_len); break; @@ -8482,17 +8261,7 @@ calc_row_difference( TRUE, new_mysql_row_col, col_pack_len, - dict_table_is_comp(prebuilt->table), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast( - field->zip_dict_data.str), - field->zip_dict_data.length, -#else - 0, 0, 0, -#endif - prebuilt); + dict_table_is_comp(prebuilt->table)); dfield_copy(&ufield->new_val, &dfield); } else { dfield_set_null(&ufield->new_val); @@ -10179,7 +9948,6 @@ create_table_def( ulint unsigned_type; ulint binary_type; ulint long_true_varchar; - ulint compressed; ulint charset_no; ulint i; ulint doc_id_col = 0; @@ -10329,13 +10097,6 @@ create_table_def( } } - /* Check if the the field has COMPRESSED attribute */ - compressed = 0; - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - compressed = DATA_COMPRESSED; - } - /* First check whether the column to be added has a system reserved name. */ if (dict_col_name_is_reserved(field->field_name)){ @@ -10356,8 +10117,7 @@ create_table_def( dtype_form_prtype( (ulint) field->type() | nulls_allowed | unsigned_type - | binary_type | long_true_varchar - | compressed, + | binary_type | long_true_varchar, charset_no), col_len); } @@ -11187,9 +10947,6 @@ ha_innobase::create( const char* stmt; size_t stmt_len; - mem_heap_t* heap = 0; - ulint* zip_dict_ids = 0; - DBUG_ENTER("ha_innobase::create"); DBUG_ASSERT(thd != NULL); @@ -11280,25 +11037,6 @@ ha_innobase::create( row_mysql_lock_data_dictionary(trx); - heap = mem_heap_create(form->s->fields * sizeof(ulint)); - zip_dict_ids = static_cast( - mem_heap_alloc(heap, form->s->fields * sizeof(ulint))); - - /* This is currently required for valgrind because MariaDB does - not currently support compressed columns. */ - for (size_t field_idx = 0; field_idx < form->s->fields; ++field_idx) { - zip_dict_ids[field_idx] = ULINT_UNDEFINED; - } - - const char* err_zip_dict_name = 0; - if (!innobase_check_zip_dicts(form, zip_dict_ids, - trx, &err_zip_dict_name)) { - error = -1; - my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, - MYF(0), err_zip_dict_name); - goto cleanup; - } - error = create_table_def(trx, form, norm_name, temp_path, remote_path, flags, flags2); if (error) { @@ -11406,22 +11144,6 @@ ha_innobase::create( dict_table_get_all_fts_indexes(innobase_table, fts->indexes); } - /* - Adding compression dictionary <-> compressed table column links - to the SYS_ZIP_DICT_COLS table. - */ - ut_a(zip_dict_ids != 0); - { - dict_table_t* local_table = dict_table_open_on_name( - norm_name, TRUE, FALSE, DICT_ERR_IGNORE_NONE); - - ut_a(local_table); - table_id_t table_id = local_table->id; - dict_table_close(local_table, TRUE, FALSE); - innobase_create_zip_dict_references(form, - table_id, zip_dict_ids, trx); - } - stmt = innobase_get_stmt(thd, &stmt_len); if (stmt) { @@ -11538,9 +11260,6 @@ ha_innobase::create( trx_free_for_mysql(trx); - if (heap != 0) - mem_heap_free(heap); - DBUG_RETURN(0); cleanup: @@ -11550,9 +11269,6 @@ ha_innobase::create( trx_free_for_mysql(trx); - if (heap != 0) - mem_heap_free(heap); - DBUG_RETURN(error); } @@ -13834,10 +13550,6 @@ ha_innobase::extra( row_mysql_prebuilt_free_blob_heap(prebuilt); } - if (prebuilt->compress_heap) { - row_mysql_prebuilt_free_compress_heap(prebuilt); - } - break; case HA_EXTRA_RESET_STATE: reset_template(); @@ -13889,10 +13601,6 @@ ha_innobase::reset() row_mysql_prebuilt_free_blob_heap(prebuilt); } - if (prebuilt->compress_heap) { - row_mysql_prebuilt_free_compress_heap(prebuilt); - } - reset_template(); ds_mrr.dsmrr_close(); @@ -14092,11 +13800,7 @@ ha_innobase::external_lock( && lock_type == F_WRLCK) || thd_sql_command(thd) == SQLCOM_CREATE_INDEX || thd_sql_command(thd) == SQLCOM_DROP_INDEX - || thd_sql_command(thd) == SQLCOM_DELETE - || thd_sql_command(thd) == - SQLCOM_CREATE_COMPRESSION_DICTIONARY - || thd_sql_command(thd) == - SQLCOM_DROP_COMPRESSION_DICTIONARY)) { + || thd_sql_command(thd) == SQLCOM_DELETE)) { if (thd_sql_command(thd) == SQLCOM_CREATE_TABLE) { @@ -14864,9 +14568,7 @@ ha_innobase::store_lock( && lock_type <= TL_WRITE)) || sql_command == SQLCOM_CREATE_INDEX || sql_command == SQLCOM_DROP_INDEX - || sql_command == SQLCOM_DELETE - || sql_command == SQLCOM_CREATE_COMPRESSION_DICTIONARY - || sql_command == SQLCOM_DROP_COMPRESSION_DICTIONARY)) { + || sql_command == SQLCOM_DELETE)) { ib_senderrf(trx->mysql_thd, IB_LOG_LEVEL_WARN, ER_READ_ONLY_MODE); @@ -15807,84 +15509,6 @@ ha_innobase::check_if_incompatible_data( return(COMPATIBLE_DATA_YES); } -/** This function reads zip dict-related info from SYS_ZIP_DICT -and SYS_ZIP_DICT_COLS for all columns marked with -COLUMN_FORMAT_TYPE_COMPRESSED flag and updates -zip_dict_name / zip_dict_data for those which have associated -compression dictionaries. -*/ -UNIV_INTERN -void -ha_innobase::update_field_defs_with_zip_dict_info() -{ - DBUG_ENTER("update_field_defs_with_zip_dict_info"); - ut_ad(!mutex_own(&dict_sys->mutex)); - - char norm_name[FN_REFLEN]; - normalize_table_name(norm_name, table_share->normalized_path.str); - - dict_table_t* ib_table = dict_table_open_on_name( - norm_name, FALSE, FALSE, DICT_ERR_IGNORE_NONE); - - /* if dict_table_open_on_name() returns NULL, then it means that - TABLE_SHARE is populated for a table being created and we can - skip filling zip dict info here */ - if (ib_table == 0) - DBUG_VOID_RETURN; - -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - table_id_t ib_table_id = ib_table->id; - dict_table_close(ib_table, FALSE, FALSE); - Field* field; - for (uint i = 0; i < table_share->fields; ++i) { - field = table_share->field[i]; - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - bool reference_found = false; - ulint dict_id = 0; - switch (dict_get_dictionary_id_by_key(ib_table_id, i, - &dict_id)) { - case DB_SUCCESS: - reference_found = true; - break; - case DB_RECORD_NOT_FOUND: - reference_found = false; - break; - default: - ut_error; - } - if (reference_found) { - char* local_name = 0; - ulint local_name_len = 0; - char* local_data = 0; - ulint local_data_len = 0; - if (dict_get_dictionary_info_by_id(dict_id, - &local_name, &local_name_len, - &local_data, &local_data_len) != - DB_SUCCESS) { - ut_error; - } - else { - field->zip_dict_name.str = - local_name; - field->zip_dict_name.length = - local_name_len; - field->zip_dict_data.str = - local_data; - field->zip_dict_data.length = - local_data_len; - } - } - else { - field->zip_dict_name = null_lex_cstr; - field->zip_dict_data = null_lex_cstr; - } - } - } -#endif - DBUG_VOID_RETURN; -} - /****************************************************************//** Update the system variable innodb_io_capacity_max using the "saved" value. This function is registered as a callback with MySQL. */ @@ -19276,21 +18900,6 @@ static MYSQL_SYSVAR_BOOL(use_stacktrace, srv_use_stacktrace, "Print stacktrace on long semaphore wait (off by default supported only on linux)", NULL, NULL, FALSE); -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -static MYSQL_SYSVAR_UINT(compressed_columns_zip_level, - srv_compressed_columns_zip_level, - PLUGIN_VAR_RQCMDARG, - "Compression level used for compressed columns. 0 is no compression" - ", 1 is fastest and 9 is best compression. Default is 6.", - NULL, NULL, DEFAULT_COMPRESSION_LEVEL, 0, 9, 0); - -static MYSQL_SYSVAR_ULONG(compressed_columns_threshold, - srv_compressed_columns_threshold, - PLUGIN_VAR_RQCMDARG, - "Compress column data if its length exceeds this value. Default is 96", - NULL, NULL, 96, 1, ~0UL, 0); -#endif - static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_block_size), MYSQL_SYSVAR(additional_mem_pool_size), @@ -19494,10 +19103,6 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(tmpdir), MYSQL_SYSVAR(use_stacktrace), MYSQL_SYSVAR(simulate_comp_failures), -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - MYSQL_SYSVAR(compressed_columns_zip_level), - MYSQL_SYSVAR(compressed_columns_threshold), -#endif NULL }; @@ -19520,10 +19125,6 @@ maria_declare_plugin(xtradb) i_s_xtradb_read_view, i_s_xtradb_internal_hash_tables, i_s_xtradb_rseg, -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -i_s_xtradb_zip_dict, -i_s_xtradb_zip_dict_cols, -#endif i_s_innodb_trx, i_s_innodb_locks, i_s_innodb_lock_waits, diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index f01faaf41d823..f4c73fda89df5 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -290,14 +290,6 @@ class ha_innobase: public handler bool check_if_supported_virtual_columns(void) { return TRUE; } - /** This function reads zip dict-related info from SYS_ZIP_DICT - and SYS_ZIP_DICT_COLS for all columns marked with - COLUMN_FORMAT_TYPE_COMPRESSED flag and updates - zip_dict_name / zip_dict_data for those which have associated - compression dictionaries. - */ - virtual void update_field_defs_with_zip_dict_info(); - private: /** Builds a 'template' to the prebuilt struct. @@ -691,31 +683,3 @@ ib_push_frm_error( TABLE* table, /*!< in: MySQL table */ ulint n_keys, /*!< in: InnoDB #keys */ bool push_warning); /*!< in: print warning ? */ - -/** This function checks if all the compression dictionaries referenced -in table->fields exist in SYS_ZIP_DICT InnoDB system table. -@return true if all referenced dictionaries exist */ -UNIV_INTERN -bool -innobase_check_zip_dicts( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - ulint* dict_ids, /*!< out: identified zip dict ids - (at least n_fields long) */ - trx_t* trx, /*!< in: transaction */ - const char** err_dict_name); /*!< out: the name of the - zip_dict which does not exist. */ - -/** This function creates compression dictionary references in -SYS_ZIP_DICT_COLS InnoDB system table for table_id based on info -in table->fields and provided zip dict ids. */ -UNIV_INTERN -void -innobase_create_zip_dict_references( - const TABLE* table, /*!< in: table in MySQL data - dictionary */ - table_id_t ib_table_id, /*!< in: table ID in Innodb data - dictionary */ - ulint* zip_dict_ids, /*!< in: zip dict ids - (at least n_fields long) */ - trx_t* trx); /*!< in: transaction */ diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 362ac9c1b89e8..c5ac48dc4e376 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -21,11 +22,6 @@ this program; if not, write to the Free Software Foundation, Inc., Smart ALTER TABLE *******************************************************/ -#ifndef HAVE_PERCONA_COMPRESSED_COLUMNS -#define COLUMN_FORMAT_TYPE_COMPRESSED 0xBADF00D -#define ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST 0xDEADFACE -#endif - #include #include #include @@ -1185,15 +1181,6 @@ innobase_col_to_mysql( field->reset(); if (field->type() == MYSQL_TYPE_VARCHAR) { - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - /* Skip compressed varchar column when - reporting an erroneous row - during index creation or table rebuild. */ - field->set_null(); - break; - } - /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2492,14 +2479,7 @@ innobase_build_col_map_add( byte* buf = static_cast(mem_heap_alloc(heap, size)); row_mysql_store_col_in_innobase_format( - dfield, buf, TRUE, field->ptr, size, comp, -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - field->column_format() == COLUMN_FORMAT_TYPE_COMPRESSED, - reinterpret_cast(field->zip_dict_data.str), - field->zip_dict_data.length, prebuilt); -#else - 0,0,0, prebuilt); -#endif + dfield, buf, TRUE, field->ptr, size, comp); } /** Construct the translation table for reordering, dropping or @@ -2753,7 +2733,6 @@ prepare_inplace_alter_table_dict( ulint num_fts_index; ha_innobase_inplace_ctx*ctx; uint sql_idx; - ulint* zip_dict_ids = 0; DBUG_ENTER("prepare_inplace_alter_table_dict"); @@ -2890,26 +2869,6 @@ prepare_inplace_alter_table_dict( ulint n_cols; dtuple_t* add_cols; - zip_dict_ids = static_cast( - mem_heap_alloc(ctx->heap, - altered_table->s->fields * sizeof(ulint))); - - /* This is currently required for valgrind because MariaDB does - not currently support compressed columns. */ - for (size_t field_idx = 0; - field_idx < altered_table->s->fields; - ++field_idx) { - zip_dict_ids[field_idx] = ULINT_UNDEFINED; - } - - const char* err_zip_dict_name = 0; - if (!innobase_check_zip_dicts(altered_table, zip_dict_ids, - ctx->trx, &err_zip_dict_name)) { - my_error(ER_COMPRESSION_DICTIONARY_DOES_NOT_EXIST, - MYF(0), err_zip_dict_name); - goto new_clustered_failed; - } - if (innobase_check_foreigns( ha_alter_info, altered_table, old_table, user_table, ctx->drop_fk, ctx->num_to_drop_fk)) { @@ -3016,12 +2975,6 @@ prepare_inplace_alter_table_dict( } } - if (field->column_format() == - COLUMN_FORMAT_TYPE_COMPRESSED) { - field_type |= DATA_COMPRESSED; - } - - if (dict_col_name_is_reserved(field->field_name)) { dict_mem_table_free(ctx->new_table); my_error(ER_WRONG_COLUMN_NAME, MYF(0), @@ -3279,17 +3232,6 @@ prepare_inplace_alter_table_dict( DBUG_ASSERT(error == DB_SUCCESS); -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS - /* - Adding compression dictionary <-> compressed table column links - to the SYS_ZIP_DICT_COLS table. - */ - if (zip_dict_ids != 0) { - innobase_create_zip_dict_references(altered_table, - ctx->trx->table_id, zip_dict_ids, ctx->trx); - } -#endif - /* Commit the data dictionary transaction in order to release the table locks on the system tables. This means that if MySQL crashes while creating a new primary key inside diff --git a/storage/xtradb/handler/xtradb_i_s.cc b/storage/xtradb/handler/xtradb_i_s.cc index eb6637dad0397..84c1e5852bc77 100644 --- a/storage/xtradb/handler/xtradb_i_s.cc +++ b/storage/xtradb/handler/xtradb_i_s.cc @@ -2,6 +2,7 @@ Copyright (c) 2007, 2012, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2010-2012, Percona Inc. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -36,7 +37,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include /* btr_search_sys */ #include /* recv_sys */ #include -#include /* for ZIP_DICT_MAX_* constants */ /* for XTRADB_RSEG table */ #include "trx0trx.h" /* for TRX_QUE_STATE_STR_MAX_LEN */ @@ -629,331 +629,3 @@ UNIV_INTERN struct st_mysql_plugin i_s_xtradb_rseg = STRUCT_FLD(version_info, INNODB_VERSION_STR), STRUCT_FLD(maturity, MariaDB_PLUGIN_MATURITY_STABLE), }; - - -#ifdef HAVE_PERCONA_COMPRESSED_COLUMNS -/************************************************************************/ -enum zip_dict_field_type -{ - zip_dict_field_id, - zip_dict_field_name, - zip_dict_field_zip_dict -}; - -static ST_FIELD_INFO xtradb_sys_zip_dict_fields_info[] = -{ - { STRUCT_FLD(field_name, "id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "name"), - STRUCT_FLD(field_length, ZIP_DICT_MAX_NAME_LENGTH), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, 0), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "zip_dict"), - STRUCT_FLD(field_length, ZIP_DICT_MAX_DATA_LENGTH), - STRUCT_FLD(field_type, MYSQL_TYPE_BLOB), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, 0), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - END_OF_ST_FIELD_INFO -}; - -/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT with information -collected by scanning SYS_ZIP_DICT table. -@return 0 on success */ -static -int -xtradb_i_s_dict_fill_sys_zip_dict( - THD* thd, /*!< in: thread */ - ulint id, /*!< in: dict ID */ - const char* name, /*!< in: dict name */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - TABLE* table_to_fill) /*!< in/out: fill this table */ -{ - DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict"); - - Field** fields = table_to_fill->field; - - OK(field_store_ulint(fields[zip_dict_field_id], id)); - OK(field_store_string(fields[zip_dict_field_name], name)); - OK(field_store_blob(fields[zip_dict_field_zip_dict], data, - data_len)); - - OK(schema_table_store_record(thd, table_to_fill)); - - DBUG_RETURN(0); -} - -/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. -Loop through each record in SYS_ZIP_DICT, and extract the column -information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT table. -@return 0 on success */ -static -int -xtradb_i_s_sys_zip_dict_fill_table( - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (not used) */ -{ - btr_pcur_t pcur; - const rec_t* rec; - mem_heap_t* heap; - mtr_t mtr; - - DBUG_ENTER("xtradb_i_s_sys_zip_dict_fill_table"); - RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); - - /* deny access to user without SUPER_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { - DBUG_RETURN(0); - } - - heap = mem_heap_create(1000); - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - - rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT); - ulint zip_size = dict_table_zip_size(pcur.btr_cur.index->table); - - while (rec) { - const char* err_msg; - ulint id; - const char* name; - const char* data; - ulint data_len; - - /* Extract necessary information from a SYS_ZIP_DICT row */ - err_msg = dict_process_sys_zip_dict( - heap, zip_size, rec, &id, &name, &data, &data_len); - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - - if (!err_msg) { - xtradb_i_s_dict_fill_sys_zip_dict( - thd, id, name, data, data_len, - tables->table); - } else { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); - } - - mem_heap_empty(heap); - - /* Get the next record */ - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - rec = dict_getnext_system(&pcur, &mtr); - } - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - mem_heap_free(heap); - - DBUG_RETURN(0); -} - -static int i_s_xtradb_zip_dict_init(void* p) -{ - DBUG_ENTER("i_s_xtradb_zip_dict_init"); - - ST_SCHEMA_TABLE* schema = static_cast(p); - - schema->fields_info = xtradb_sys_zip_dict_fields_info; - schema->fill_table = xtradb_i_s_sys_zip_dict_fill_table; - - DBUG_RETURN(0); -} - -UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict = -{ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - STRUCT_FLD(info, &i_s_info), - STRUCT_FLD(name, "XTRADB_ZIP_DICT"), - STRUCT_FLD(author, PLUGIN_AUTHOR), - STRUCT_FLD(descr, "InnoDB compression dictionaries information"), - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - STRUCT_FLD(init, i_s_xtradb_zip_dict_init), - STRUCT_FLD(deinit, i_s_common_deinit), - STRUCT_FLD(version, INNODB_VERSION_SHORT), - STRUCT_FLD(status_vars, NULL), - STRUCT_FLD(system_vars, NULL), - STRUCT_FLD(__reserved1, NULL), - STRUCT_FLD(flags, 0UL), -}; - -enum zip_dict_cols_field_type -{ - zip_dict_cols_field_table_id, - zip_dict_cols_field_column_pos, - zip_dict_cols_field_dict_id -}; - -static ST_FIELD_INFO xtradb_sys_zip_dict_cols_fields_info[] = -{ - { STRUCT_FLD(field_name, "table_id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "column_pos"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - { STRUCT_FLD(field_name, "dict_id"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE) }, - - END_OF_ST_FIELD_INFO -}; - -/** Function to fill INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS with information -collected by scanning SYS_ZIP_DICT_COLS table. -@return 0 on success */ -static -int -xtradb_i_s_dict_fill_sys_zip_dict_cols( - THD* thd, /*!< in: thread */ - ulint table_id, /*!< in: table ID */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict ID */ - TABLE* table_to_fill) /*!< in/out: fill this table */ -{ - DBUG_ENTER("xtradb_i_s_dict_fill_sys_zip_dict_cols"); - - Field** fields = table_to_fill->field; - - OK(field_store_ulint(fields[zip_dict_cols_field_table_id], - table_id)); - OK(field_store_ulint(fields[zip_dict_cols_field_column_pos], - column_pos)); - OK(field_store_ulint(fields[zip_dict_cols_field_dict_id], - dict_id)); - - OK(schema_table_store_record(thd, table_to_fill)); - - DBUG_RETURN(0); -} - -/** Function to populate INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. -Loop through each record in SYS_ZIP_DICT_COLS, and extract the column -information and fill the INFORMATION_SCHEMA.XTRADB_ZIP_DICT_COLS table. -@return 0 on success */ -static -int -xtradb_i_s_sys_zip_dict_cols_fill_table( - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (not used) */ -{ - btr_pcur_t pcur; - const rec_t* rec; - mem_heap_t* heap; - mtr_t mtr; - - DBUG_ENTER("xtradb_i_s_sys_zip_dict_cols_fill_table"); - RETURN_IF_INNODB_NOT_STARTED(tables->schema_table_name); - - /* deny access to user without SUPER_ACL privilege */ - if (check_global_access(thd, SUPER_ACL)) { - DBUG_RETURN(0); - } - - heap = mem_heap_create(1000); - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - - rec = dict_startscan_system(&pcur, &mtr, SYS_ZIP_DICT_COLS); - - while (rec) { - const char* err_msg; - ulint table_id; - ulint column_pos; - ulint dict_id; - - /* Extract necessary information from a SYS_ZIP_DICT_COLS - row */ - err_msg = dict_process_sys_zip_dict_cols( - heap, rec, &table_id, &column_pos, &dict_id); - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - - if (!err_msg) { - xtradb_i_s_dict_fill_sys_zip_dict_cols( - thd, table_id, column_pos, dict_id, - tables->table); - } else { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - ER_CANT_FIND_SYSTEM_REC, "%s", err_msg); - } - - mem_heap_empty(heap); - - /* Get the next record */ - mutex_enter(&dict_sys->mutex); - mtr_start(&mtr); - rec = dict_getnext_system(&pcur, &mtr); - } - - mtr_commit(&mtr); - mutex_exit(&dict_sys->mutex); - mem_heap_free(heap); - - DBUG_RETURN(0); -} - -static int i_s_xtradb_zip_dict_cols_init(void* p) -{ - DBUG_ENTER("i_s_xtradb_zip_dict_cols_init"); - - ST_SCHEMA_TABLE* schema = static_cast(p); - - schema->fields_info = xtradb_sys_zip_dict_cols_fields_info; - schema->fill_table = xtradb_i_s_sys_zip_dict_cols_fill_table; - - DBUG_RETURN(0); -} - -UNIV_INTERN struct st_mysql_plugin i_s_xtradb_zip_dict_cols = -{ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - STRUCT_FLD(info, &i_s_info), - STRUCT_FLD(name, "XTRADB_ZIP_DICT_COLS"), - STRUCT_FLD(author, PLUGIN_AUTHOR), - STRUCT_FLD(descr, "InnoDB compressed columns information"), - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - STRUCT_FLD(init, i_s_xtradb_zip_dict_cols_init), - STRUCT_FLD(deinit, i_s_common_deinit), - STRUCT_FLD(version, INNODB_VERSION_SHORT), - STRUCT_FLD(status_vars, NULL), - STRUCT_FLD(system_vars, NULL), - STRUCT_FLD(__reserved1, NULL), - STRUCT_FLD(flags, 0UL), -}; -#endif diff --git a/storage/xtradb/handler/xtradb_i_s.h b/storage/xtradb/handler/xtradb_i_s.h index 905d84587affd..994bc11c1b832 100644 --- a/storage/xtradb/handler/xtradb_i_s.h +++ b/storage/xtradb/handler/xtradb_i_s.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2010-2012, Percona Inc. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -22,7 +23,5 @@ this program; if not, write to the Free Software Foundation, Inc., extern struct st_mysql_plugin i_s_xtradb_read_view; extern struct st_mysql_plugin i_s_xtradb_internal_hash_tables; extern struct st_mysql_plugin i_s_xtradb_rseg; -extern struct st_mysql_plugin i_s_xtradb_zip_dict; -extern struct st_mysql_plugin i_s_xtradb_zip_dict_cols; #endif /* XTRADB_I_S_H */ diff --git a/storage/xtradb/include/data0type.h b/storage/xtradb/include/data0type.h index f269c266efb9b..df6b6a41c1184 100644 --- a/storage/xtradb/include/data0type.h +++ b/storage/xtradb/include/data0type.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -170,9 +171,6 @@ be less than 256 */ type when the column is true VARCHAR where MySQL uses 2 bytes to store the data len; for shorter VARCHARs MySQL uses only 1 byte */ -#define DATA_COMPRESSED 16384 /* this is ORed to the precise data - type when the column has COLUMN_FORMAT = - COMPRESSED attribute*/ /*-------------------------------------------*/ /* This many bytes we need to store the type information affecting the @@ -503,17 +501,6 @@ dtype_print( /*========*/ const dtype_t* type); /*!< in: type */ -/** -Calculates the number of extra bytes needed for compression header -depending on precise column type. -@reval 0 if prtype does not include DATA_COMPRESSED flag -@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag -*/ -UNIV_INLINE -ulint -prtype_get_compression_extra( - ulint prtype); /*!< in: precise type */ - /* Structure for an SQL data type. If you add fields to this structure, be sure to initialize them everywhere. This structure is initialized in the following functions: diff --git a/storage/xtradb/include/data0type.ic b/storage/xtradb/include/data0type.ic index 29dc480a19c3e..555852474aa2e 100644 --- a/storage/xtradb/include/data0type.ic +++ b/storage/xtradb/include/data0type.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -710,18 +711,3 @@ dtype_get_sql_null_size( 0, 0)); #endif /* !UNIV_HOTBACKUP */ } - -/** -Calculates the number of extra bytes needed for compression header -depending on precise column type. -@reval 0 if prtype does not include DATA_COMPRESSED flag -@reval ZIP_COLUMN_HEADER_LENGTH if prtype includes DATA_COMPRESSED flag -*/ -UNIV_INLINE -ulint -prtype_get_compression_extra( - ulint prtype) /*!< in: precise type */ -{ - return (prtype & DATA_COMPRESSED) != 0 ? - ZIP_COLUMN_HEADER_LENGTH : 0; -} diff --git a/storage/xtradb/include/dict0boot.h b/storage/xtradb/include/dict0boot.h index d5bee886cbf43..4fd9b0b7f98a6 100644 --- a/storage/xtradb/include/dict0boot.h +++ b/storage/xtradb/include/dict0boot.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -324,38 +325,6 @@ enum dict_fld_sys_datafiles_enum { DICT_FLD__SYS_DATAFILES__PATH = 3, DICT_NUM_FIELDS__SYS_DATAFILES = 4 }; -/* The columns in SYS_DICT */ -enum dict_col_sys_zip_dict_enum { - DICT_COL__SYS_ZIP_DICT__ID = 0, - DICT_COL__SYS_ZIP_DICT__NAME = 1, - DICT_COL__SYS_ZIP_DICT__DATA = 2, - DICT_NUM_COLS__SYS_ZIP_DICT = 3 -}; -/* The field numbers in the SYS_DICT clustered index */ -enum dict_fld_sys_zip_dict_enum { - DICT_FLD__SYS_ZIP_DICT__ID = 0, - DICT_FLD__SYS_ZIP_DICT__DB_TRX_ID = 1, - DICT_FLD__SYS_ZIP_DICT__DB_ROLL_PTR = 2, - DICT_FLD__SYS_ZIP_DICT__NAME = 3, - DICT_FLD__SYS_ZIP_DICT__DATA = 4, - DICT_NUM_FIELDS__SYS_ZIP_DICT = 5 -}; -/* The columns in SYS_DICT_COLS */ -enum dict_col_sys_zip_dict_cols_enum { - DICT_COL__SYS_ZIP_DICT_COLS__TABLE_ID = 0, - DICT_COL__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, - DICT_COL__SYS_ZIP_DICT_COLS__DICT_ID = 2, - DICT_NUM_COLS__SYS_ZIP_DICT_COLS = 3 -}; -/* The field numbers in the SYS_DICT_COLS clustered index */ -enum dict_fld_sys_zip_dict_cols_enum { - DICT_FLD__SYS_ZIP_DICT_COLS__TABLE_ID = 0, - DICT_FLD__SYS_ZIP_DICT_COLS__COLUMN_POS = 1, - DICT_FLD__SYS_ZIP_DICT_COLS__DB_TRX_ID = 2, - DICT_FLD__SYS_ZIP_DICT_COLS__DB_ROLL_PTR = 3, - DICT_FLD__SYS_ZIP_DICT_COLS__DICT_ID = 4, - DICT_NUM_FIELDS__SYS_ZIP_DICT_COLS = 5 -}; /* A number of the columns above occur in multiple tables. These are the length of thos fields. */ diff --git a/storage/xtradb/include/dict0boot.ic b/storage/xtradb/include/dict0boot.ic index 2b156a4f67263..42e91ee930ea1 100644 --- a/storage/xtradb/include/dict0boot.ic +++ b/storage/xtradb/include/dict0boot.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2012, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -92,5 +93,3 @@ dict_is_sys_table( { return(id < DICT_HDR_FIRST_ID); } - - diff --git a/storage/xtradb/include/dict0crea.h b/storage/xtradb/include/dict0crea.h index 686f56ad58c61..e5977cbabf680 100644 --- a/storage/xtradb/include/dict0crea.h +++ b/storage/xtradb/include/dict0crea.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -164,18 +165,6 @@ dberr_t dict_create_or_check_sys_tablespace(void); /*=====================================*/ -#define ZIP_DICT_MAX_NAME_LENGTH 64 -/* Max window size (2^15) minus 262 */ -#define ZIP_DICT_MAX_DATA_LENGTH 32506 - -/** Creates the zip_dict system table inside InnoDB -at server bootstrap or server start if it is not found or is -not of the right form. -@return DB_SUCCESS or error code */ -UNIV_INTERN -dberr_t -dict_create_or_check_sys_zip_dict(void); - /********************************************************************//** Add a single tablespace definition to the data dictionary tables in the database. @@ -192,83 +181,6 @@ dict_create_add_tablespace_to_dictionary( bool commit); /*!< in: if true then commit the transaction */ -/** Add a single compression dictionary definition to the SYS_ZIP_DICT -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - const char* data, /*!< in: dict data */ - ulint data_len, /*!< in: dict data length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Add a single compression dictionary reference to the SYS_ZIP_DICT_COLS -InnoDB system table. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_add_zip_dict_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint dict_id, /*!< in: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get a single compression dictionary id for the given -(table id, column pos) pair. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_reference( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get compression dictionary id for the given name. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_id_by_name( - const char* dict_name, /*!< in: dict name */ - ulint dict_name_len, /*!< in: dict name length */ - ulint* dict_id, /*!< out: dict id */ - trx_t* trx); /*!< in/out: transaction */ - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory for name and data on success. -Must be freed with mem_free(). -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_get_zip_dict_info_by_id( - ulint dict_id, /*!< in: dict id */ - char** name, /*!< out: dict name */ - ulint* name_len, /*!< out: dict name length */ - char** data, /*!< out: dict data */ - ulint* data_len, /*!< out: dict data length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Remove a single compression dictionary from the data dictionary -tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict( - const char* name, /*!< in: dict name */ - ulint name_len, /*!< in: dict name length */ - trx_t* trx); /*!< in/out: transaction */ - -/** Remove all compression dictionary references for the given table ID from -the data dictionary tables in the database. -@return error code or DB_SUCCESS */ -UNIV_INTERN -dberr_t -dict_create_remove_zip_dict_references_for_table( - ulint table_id, /*!< in: table id */ - trx_t* trx); /*!< in/out: transaction */ - /********************************************************************//** Add a foreign key definition to the data dictionary tables. @return error code or DB_SUCCESS */ diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index 1b6110dd0102c..a85f6f1eef96a 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2014, 2015, MariaDB Corporation. +Copyright (c) 2014, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1871,52 +1871,6 @@ dict_table_set_corrupt_by_space( ulint space_id, ibool need_mutex); -/** Insert a records into SYS_ZIP_DICT. -@retval DB_SUCCESS if OK -@retval dberr_t if the insert failed */ -UNIV_INTERN -dberr_t -dict_create_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len, /*!< in: zip_dict name length*/ - const char* data, /*!< in: zip_dict data */ - ulint data_len); /*!< in: zip_dict data length */ - -/** Get single compression dictionary id for the given -(table id, column pos) pair. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_id_by_key( - ulint table_id, /*!< in: table id */ - ulint column_pos, /*!< in: column position */ - ulint* dict_id); /*!< out: zip_dict id */ - -/** Get compression dictionary info (name and data) for the given id. -Allocates memory in name->str and data->str on success. -Must be freed with mem_free(). -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found */ -UNIV_INTERN -dberr_t -dict_get_dictionary_info_by_id( - ulint dict_id, /*!< in: table name */ - char** name, /*!< out: dictionary name */ - ulint* name_len, /*!< out: dictionary name length*/ - char** data, /*!< out: dictionary data */ - ulint* data_len); /*!< out: dictionary data length*/ - -/** Delete a record in SYS_ZIP_DICT with the given name. -@retval DB_SUCCESS if OK -@retval DB_RECORD_NOT_FOUND if not found -@retval DB_ROW_IS_REFERENCED if in use */ -UNIV_INTERN -dberr_t -dict_drop_zip_dict( - const char* name, /*!< in: zip_dict name */ - ulint name_len); /*!< in: zip_dict name length*/ - #ifndef UNIV_NONINL #include "dict0dict.ic" #endif diff --git a/storage/xtradb/include/dict0load.h b/storage/xtradb/include/dict0load.h index 85e3e5656371b..1a720de5bb661 100644 --- a/storage/xtradb/include/dict0load.h +++ b/storage/xtradb/include/dict0load.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -44,8 +45,6 @@ enum dict_system_id_t { SYS_FOREIGN_COLS, SYS_TABLESPACES, SYS_DATAFILES, - SYS_ZIP_DICT, - SYS_ZIP_DICT_COLS, /* This must be last item. Defines the number of system tables. */ SYS_NUM_SYSTEM_TABLES @@ -389,32 +388,6 @@ dict_process_sys_datafiles( ulint* space, /*!< out: pace id */ const char** path); /*!< out: datafile path */ -/** This function parses a SYS_ZIP_DICT record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict( - mem_heap_t* heap, /*!< in/out: heap memory */ - ulint zip_size, /*!< in: nonzero=compressed BLOB page size */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* id, /*!< out: dict id */ - const char** name, /*!< out: dict name */ - const char** data, /*!< out: dict data */ - ulint* data_len); /*!< out: dict data length */ - -/** This function parses a SYS_ZIP_DICT_COLS record, extracts necessary -information from the record and returns to caller. -@return error message, or NULL on success */ -UNIV_INTERN -const char* -dict_process_sys_zip_dict_cols( - mem_heap_t* heap, /*!< in/out: heap memory */ - const rec_t* rec, /*!< in: current SYS_ZIP_DICT rec */ - ulint* table_id, /*!< out: table id */ - ulint* column_pos, /*!< out: column position */ - ulint* dict_id); /*!< out: dict id */ - /********************************************************************//** Get the filepath for a spaceid from SYS_DATAFILES. This function provides a temporary heap which is used for the table lookup, but not for the path. diff --git a/storage/xtradb/include/row0mysql.h b/storage/xtradb/include/row0mysql.h index 70da84640e533..e1148517d99fb 100644 --- a/storage/xtradb/include/row0mysql.h +++ b/storage/xtradb/include/row0mysql.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -41,9 +42,6 @@ struct SysIndexCallback; extern ibool row_rollback_on_timeout; -extern uint srv_compressed_columns_zip_level; -extern ulong srv_compressed_columns_threshold; - struct row_prebuilt_t; /*******************************************************************//** @@ -55,48 +53,6 @@ row_mysql_prebuilt_free_blob_heap( row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a ha_innobase:: table handle */ -/** Frees the compress heap in prebuilt when no longer needed. */ -UNIV_INTERN -void -row_mysql_prebuilt_free_compress_heap( - row_prebuilt_t* prebuilt); /*!< in: prebuilt struct of a - ha_innobase:: table handle */ - -/** Uncompress blob/text/varchar column using zlib -@return pointer to the uncompressed data */ -const byte* -row_decompress_column( - const byte* data, /*!< in: data in innodb(compressed) format */ - ulint *len, /*!< in: data length; out: length of - decompressed data*/ - const byte* dict_data, - /*!< in: optional dictionary data used for - decompression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt); - /*!< in: use prebuilt->compress_heap only - here*/ - -/** Compress blob/text/varchar column using zlib -@return pointer to the compressed data */ -byte* -row_compress_column( - const byte* data, /*!< in: data in mysql(uncompressed) - format */ - ulint *len, /*!< in: data length; out: length of - compressed data*/ - ulint lenlen, /*!< in: bytes used to store the length of - data */ - const byte* dict_data, - /*!< in: optional dictionary data used for - compression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt); - /*!< in: use prebuilt->compress_heap only - here*/ - /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -135,21 +91,10 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len, /*!< in: BLOB length; if the value to store + ulint len); /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ - bool need_decompression, - /*!< in: if the data need to be compressed*/ - const byte* dict_data, - /*!< in: optional compression dictionary - data */ - ulint dict_data_len, - /*!< in: optional compression dictionary data - length */ - row_prebuilt_t* prebuilt); - /*compress_heap only - here */ /*******************************************************************//** Reads a reference to a BLOB in the MySQL format. @return pointer to BLOB data */ @@ -160,17 +105,8 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len, /*!< in: BLOB reference length + ulint col_len); /*!< in: BLOB reference length (not BLOB length) */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap - only here */ /**************************************************************//** Pad a column with spaces. */ UNIV_INTERN @@ -218,16 +154,7 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp, /*!< in: nonzero=compact format */ - bool need_compression, - /*!< in: if the data need to be - compressed */ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt); /*!< in: use prebuilt->compress_heap - only here */ + ulint comp); /*!< in: nonzero=compact format */ /****************************************************************//** Handles user errors and lock waits detected by the database engine. @return true if it was a lock wait and we should continue running the @@ -721,8 +648,6 @@ struct mysql_row_templ_t { ulint is_unsigned; /*!< if a column type is an integer type and this field is != 0, then it is an unsigned integer type */ - bool compressed; /*!< if column format is compressed */ - LEX_CSTRING zip_dict_data; /*!< associated compression dictionary */ }; #define MYSQL_FETCH_CACHE_SIZE 8 @@ -920,8 +845,6 @@ struct row_prebuilt_t { in fetch_cache */ mem_heap_t* blob_heap; /*!< in SELECTS BLOB fields are copied to this heap */ - mem_heap_t* compress_heap; /*!< memory heap used to compress - /decompress blob column*/ mem_heap_t* old_vers_heap; /*!< memory heap where a previous version is built in consistent read */ bool in_fts_query; /*!< Whether we are in a FTS query */ diff --git a/storage/xtradb/rem/rem0rec.cc b/storage/xtradb/rem/rem0rec.cc index 09cd810cd7b08..6db11be18e94e 100644 --- a/storage/xtradb/rem/rem0rec.cc +++ b/storage/xtradb/rem/rem0rec.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -320,8 +321,7 @@ rec_init_offsets_comp_ordinary( stored in one byte for 0..127. The length will be encoded in two bytes when it is 128 or more, or when the field is stored externally. */ - if (UNIV_UNLIKELY(col->len > 255 - - prtype_get_compression_extra(col->prtype)) + if (UNIV_UNLIKELY(col->len > 255) || UNIV_UNLIKELY(col->mtype == DATA_BLOB)) { if (len & 0x80) { @@ -846,8 +846,7 @@ rec_get_converted_size_comp_prefix_low( ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY || col->mtype == DATA_VARMYSQL) && (col->len == 0 - || len <= col->len + - prtype_get_compression_extra(col->prtype)))); + || len <= col->len))); fixed_len = field->fixed_len; if (temp && fixed_len @@ -879,8 +878,7 @@ rec_get_converted_size_comp_prefix_low( ut_ad(col->len >= 256 || col->mtype == DATA_BLOB); extra_size += 2; } else if (len < 128 - || (col->len < 256 - - prtype_get_compression_extra(col->prtype) + || (col->len < 256 && col->mtype != DATA_BLOB)) { extra_size++; } else { @@ -1276,16 +1274,12 @@ rec_convert_dtuple_to_rec_comp( *lens-- = (byte) (len >> 8) | 0xc0; *lens-- = (byte) len; } else { - ut_ad(len <= dtype_get_len(type) + - prtype_get_compression_extra( - dtype_get_prtype(type)) + ut_ad(len <= dtype_get_len(type) || dtype_get_mtype(type) == DATA_BLOB || !strcmp(index->name, FTS_INDEX_TABLE_IND_NAME)); if (len < 128 - || (dtype_get_len(type) < 256 - - prtype_get_compression_extra( - dtype_get_prtype(type)) + || (dtype_get_len(type) < 256 && dtype_get_mtype(type) != DATA_BLOB)) { *lens-- = (byte) len; diff --git a/storage/xtradb/row/row0merge.cc b/storage/xtradb/row/row0merge.cc index e397053949e1a..c1436f9e1b733 100644 --- a/storage/xtradb/row/row0merge.cc +++ b/storage/xtradb/row/row0merge.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2005, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -529,8 +530,7 @@ row_merge_buf_add( ((col->mtype == DATA_VARCHAR || col->mtype == DATA_BINARY || col->mtype == DATA_VARMYSQL) && (col->len == 0 - || len <= col->len + - prtype_get_compression_extra(col->prtype)))); + || len <= col->len))); fixed_len = ifield->fixed_len; if (fixed_len && !dict_table_is_comp(index->table) @@ -559,8 +559,7 @@ row_merge_buf_add( } else if (dfield_is_ext(field)) { extra_size += 2; } else if (len < 128 - || (col->len < 256 - - prtype_get_compression_extra(col->prtype) + || (col->len < 256 && col->mtype != DATA_BLOB)) { extra_size++; } else { diff --git a/storage/xtradb/row/row0mysql.cc b/storage/xtradb/row/row0mysql.cc index 46bf523750c6d..b366a55b2d9f0 100644 --- a/storage/xtradb/row/row0mysql.cc +++ b/storage/xtradb/row/row0mysql.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -69,48 +70,6 @@ Created 9/17/2000 Heikki Tuuri /** Provide optional 4.x backwards compatibility for 5.0 and above */ UNIV_INTERN ibool row_rollback_on_timeout = FALSE; -/** -Z_NO_COMPRESSION = 0 -Z_BEST_SPEED = 1 -Z_BEST_COMPRESSION = 9 -Z_DEFAULT_COMPRESSION = -1 -Compression level to be used by zlib for compressed-blob columns. -Settable by user. -*/ -UNIV_INTERN uint srv_compressed_columns_zip_level = DEFAULT_COMPRESSION_LEVEL; -/** -(Z_FILTERED | Z_HUFFMAN_ONLY | Z_RLE | Z_FIXED | Z_DEFAULT_STRATEGY) - -The strategy parameter is used to tune the compression algorithm. Use the -value Z_DEFAULT_STRATEGY for normal data, Z_FILTERED for data produced by a -filter (or predictor), Z_HUFFMAN_ONLY to force Huffman encoding only -(no string match), or Z_RLE to limit match distances to one -(run-length encoding). Filtered data consists mostly of small values with a -somewhat random distribution. In this case, the compression algorithm is -tuned to compress them better. -The effect of Z_FILTERED is to force more Huffman coding and less string -matching; it is somewhat intermediate between Z_DEFAULT_STRATEGY and -Z_HUFFMAN_ONLY. Z_RLE is designed to be almost as fast as Z_HUFFMAN_ONLY, -but give better compression for PNG image data. The strategy parameter only -affects the compression ratio but not the correctness of the compressed -output even if it is not set appropriately. Z_FIXED prevents the use of -dynamic Huffman codes, allowing for a simpler decoder for special -applications. -*/ -const uint srv_compressed_columns_zlib_strategy = Z_DEFAULT_STRATEGY; -/** Compress the column if the data length exceeds this value. */ -UNIV_INTERN ulong srv_compressed_columns_threshold = 96; -/** -Determine if zlib needs to compute adler32 value for the compressed data. -This variables is similar to page_zip_zlib_wrap, but only used by -compressed blob columns. -*/ -const bool srv_compressed_columns_zlib_wrap = true; -/** -Determine if zlib will use custom memory allocation functions based on -InnoDB memory heap routines (mem_heap_t*). -*/ -const bool srv_compressed_columns_zlib_use_heap = false; /** Chain node of the list of tables to drop in the background. */ struct row_mysql_drop_t{ char* table_name; /*!< table name */ @@ -214,17 +173,6 @@ row_mysql_prebuilt_free_blob_heap( prebuilt->blob_heap = NULL; } -/** Frees the compress heap in prebuilt when no longer needed. */ -UNIV_INTERN -void -row_mysql_prebuilt_free_compress_heap( - row_prebuilt_t* prebuilt) /*!< in: prebuilt struct of a - ha_innobase:: table handle */ -{ - mem_heap_free(prebuilt->compress_heap); - prebuilt->compress_heap = NULL; -} - /*******************************************************************//** Stores a >= 5.0.3 format true VARCHAR length to dest, in the MySQL row format. @@ -281,425 +229,6 @@ row_mysql_read_true_varchar( return(field + 1); } -/** - Compressed BLOB header format: - --------------------------------------------------------------- - | reserved | wrap | algorithm | len-len | compressed | unused | - | [1] | [1] | [5] | [3] | [1] | [5] | - --------------------------------------------------------------- - | 0 0 | 1 1 | 2 6 | 7 9 | 10 10 | 11 15 | - --------------------------------------------------------------- - * 'reserved' bit is planned to be used in future versions of the BLOB - header. In this version it must always be - 'default_zip_column_reserved_value' (0). - * 'wrap' identifies if compression algorithm calculated a checksum - (adler32 in case of zlib) and appended it to the compressed data. - * 'algorithm' identifies which algoritm was used to compress this BLOB. - Currently, the only value 'default_zip_column_algorithm_value' (0) is - supported. - * 'len-len' field identifies the length of the column length data portion - followed by this header (see below). - * If 'compressed' bit is set to 1, then this header is immediately followed - by 1..8 bytes (depending on the value of 'len-len' bitfield) which - determine original (uncompressed) block size. These 'len-len' bytes are - followed by compressed representation of the original data. - * If 'compressed' bit is set to 0, every other bitfield ('wrap', - 'algorithm' and 'le-len') must be ignored. In this case the header is - immediately followed by uncompressed (original) data. -*/ - -/** - Currently the only supported value for the 'reserved' field is - false (0). -*/ -static const bool default_zip_column_reserved_value = false; - -/** - Currently the only supported value for the 'algorithm' field is 0, which - means 'zlib'. -*/ -static const uint default_zip_column_algorithm_value = 0; - -static const size_t zip_column_prefix_max_length = - ZIP_COLUMN_HEADER_LENGTH + 8; -static const size_t zip_column_header_length = ZIP_COLUMN_HEADER_LENGTH; - -/* 'reserved', bit 0 */ -static const uint zip_column_reserved = 0; -/* 0000 0000 0000 0001 */ -static const uint zip_column_reserved_mask = 0x0001; - -/* 'wrap', bit 1 */ -static const uint zip_column_wrap = 1; -/* 0000 0000 0000 0010 */ -static const uint zip_column_wrap_mask = 0x0002; - -/* 'algorithm', bit 2,3,4,5,6 */ -static const uint zip_column_algorithm = 2; -/* 0000 0000 0111 1100 */ -static const uint zip_column_algorithm_mask = 0x007C; - -/* 'len-len', bit 7,8,9 */ -static const uint zip_column_data_length = 7; -/* 0000 0011 1000 0000 */ -static const uint zip_column_data_length_mask = 0x0380; - -/* 'compressed', bit 10 */ -static const uint zip_column_compressed = 10; -/* 0000 0100 0000 0000 */ -static const uint zip_column_compressed_mask = 0x0400; - -/** Updates compressed block header with the given components */ -static void -column_set_compress_header( - byte* data, - bool compressed, - ulint lenlen, - uint alg, - bool wrap, - bool reserved) -{ - ulint header = 0; - header |= (compressed << zip_column_compressed); - header |= (lenlen << zip_column_data_length); - header |= (alg << zip_column_algorithm); - header |= (wrap << zip_column_wrap); - header |= (reserved << zip_column_reserved); - mach_write_to_2(data, header); -} - -/** Parse compressed block header into components */ -static void -column_get_compress_header( - const byte* data, - bool* compressed, - ulint* lenlen, - uint* alg, - bool* wrap, - bool* reserved -) -{ - ulint header = mach_read_from_2(data); - *compressed = ((header & zip_column_compressed_mask) >> - zip_column_compressed); - *lenlen = ((header & zip_column_data_length_mask) >> - zip_column_data_length); - *alg = ((header & zip_column_algorithm_mask) >> - zip_column_algorithm); - *wrap = ((header & zip_column_wrap_mask) >> - zip_column_wrap); - *reserved = ((header & zip_column_reserved_mask) >> - zip_column_reserved); -} - -/** Allocate memory for zlib. */ -static -void* -column_zip_zalloc( - void* opaque, /*!< in/out: memory heap */ - uInt items, /*!< in: number of items to allocate */ - uInt size) /*!< in: size of an item in bytes */ -{ - return(mem_heap_zalloc(static_cast(opaque), - items * size)); -} - -/** Deallocate memory for zlib. */ -static -void -column_zip_free( - void* opaque MY_ATTRIBUTE((unused)), /*!< in: memory heap */ - void* address MY_ATTRIBUTE((unused))) /*!< in: object to free */ -{ -} - -/** Configure the zlib allocator to use the given memory heap. */ -UNIV_INTERN -void -column_zip_set_alloc( - void* stream, /*!< in/out: zlib stream */ - mem_heap_t* heap) /*!< in: memory heap to use */ -{ - z_stream* strm = static_cast(stream); - - if (srv_compressed_columns_zlib_use_heap) { - strm->zalloc = column_zip_zalloc; - strm->zfree = column_zip_free; - strm->opaque = heap; - } else { - strm->zalloc = (alloc_func)0; - strm->zfree = (free_func)0; - strm->opaque = (voidpf)0; - } -} - -/** Compress blob/text/varchar column using zlib -@return pointer to the compressed data */ -byte* -row_compress_column( - const byte* data, /*!< in: data in mysql(uncompressed) - format */ - ulint *len, /*!< in: data length; out: length of - compressed data*/ - ulint lenlen, /*!< in: bytes used to store the length of - data */ - const byte* dict_data, - /*!< in: optional dictionary data used for - compression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap only - here*/ -{ - int err = 0; - ulint comp_len = *len; - ulint buf_len = *len + zip_column_prefix_max_length; - byte* buf; - byte* ptr; - z_stream c_stream; - bool wrap = srv_compressed_columns_zlib_wrap; - - int window_bits = wrap ? MAX_WBITS : -MAX_WBITS; - - if (!prebuilt->compress_heap) { - prebuilt->compress_heap = - mem_heap_create(max(UNIV_PAGE_SIZE, buf_len)); - } - - buf = static_cast(mem_heap_zalloc( - prebuilt->compress_heap,buf_len)); - - if (*len < srv_compressed_columns_threshold || - srv_compressed_columns_zip_level == Z_NO_COMPRESSION) - goto do_not_compress; - - ptr = buf + zip_column_header_length + lenlen; - - /*init deflate object*/ - c_stream.next_in = const_cast(data); - c_stream.avail_in = *len; - c_stream.next_out = ptr; - c_stream.avail_out = comp_len; - - column_zip_set_alloc(&c_stream, prebuilt->compress_heap); - - err = deflateInit2(&c_stream, srv_compressed_columns_zip_level, - Z_DEFLATED, window_bits, MAX_MEM_LEVEL, - srv_compressed_columns_zlib_strategy); - ut_a(err == Z_OK); - - if (dict_data != 0 && dict_data_len != 0) { - err = deflateSetDictionary(&c_stream, dict_data, - dict_data_len); - ut_a(err == Z_OK); - } - - err = deflate(&c_stream, Z_FINISH); - if (err != Z_STREAM_END) { - deflateEnd(&c_stream); - if (err == Z_OK) - err = Z_BUF_ERROR; - } else { - comp_len = c_stream.total_out; - err = deflateEnd(&c_stream); - } - - switch (err) { - case Z_OK: - break; - case Z_BUF_ERROR: - /* data after compress is larger than uncompressed data*/ - break; - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "failed to compress the column, error: %d\n", err); - } - - /* make sure the compressed data size is smaller than - uncompressed data */ - if (err == Z_OK && - *len > (comp_len + zip_column_header_length + lenlen)) { - column_set_compress_header(buf, true, lenlen - 1, - default_zip_column_algorithm_value, wrap, - default_zip_column_reserved_value); - ptr = buf + zip_column_header_length; - /*store the uncompressed data length*/ - switch (lenlen) { - case 1: - mach_write_to_1(ptr, *len); - break; - case 2: - mach_write_to_2(ptr, *len); - break; - case 3: - mach_write_to_3(ptr, *len); - break; - case 4: - mach_write_to_4(ptr, *len); - break; - default: - ut_error; - } - - *len = comp_len + zip_column_header_length + lenlen; - return buf; - } - -do_not_compress: - ptr = buf; - column_set_compress_header(ptr, false, 0, - default_zip_column_algorithm_value, false, - default_zip_column_reserved_value); - ptr += zip_column_header_length; - memcpy(ptr, data, *len); - *len += zip_column_header_length; - return buf; -} - -/** Uncompress blob/text/varchar column using zlib -@return pointer to the uncompressed data */ -const byte* -row_decompress_column( - const byte* data, /*!< in: data in innodb(compressed) format */ - ulint *len, /*!< in: data length; out: length of - decompressed data*/ - const byte* dict_data, - /*!< in: optional dictionary data used for - decompression */ - ulint dict_data_len, - /*!< in: optional dictionary data length */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap only - here*/ -{ - ulint buf_len = 0; - byte* buf; - int err = 0; - int window_bits = 0; - z_stream d_stream; - bool is_compressed = false; - bool wrap = false; - bool reserved = false; - ulint lenlen = 0; - uint alg = 0; - - ut_ad(*len != ULINT_UNDEFINED); - ut_ad(*len >= zip_column_header_length); - - column_get_compress_header(data, &is_compressed, &lenlen, &alg, - &wrap, &reserved); - - if (reserved != default_zip_column_reserved_value) { - ib_logf(IB_LOG_LEVEL_FATAL, - "unsupported compressed BLOB header format\n"); - } - - if (alg != default_zip_column_algorithm_value) { - ib_logf(IB_LOG_LEVEL_FATAL, - "unsupported 'algorithm' value in the" - " compressed BLOB header\n"); - } - - ut_a(lenlen < 4); - - data += zip_column_header_length; - if (!is_compressed) { /* column not compressed */ - *len -= zip_column_header_length; - return data; - } - - lenlen++; - - ulint comp_len = *len - zip_column_header_length - lenlen; - - ulint uncomp_len = 0; - switch (lenlen) { - case 1: - uncomp_len = mach_read_from_1(data); - break; - case 2: - uncomp_len = mach_read_from_2(data); - break; - case 3: - uncomp_len = mach_read_from_3(data); - break; - case 4: - uncomp_len = mach_read_from_4(data); - break; - default: - ut_error; - } - - data += lenlen; - - /* data is compressed, decompress it*/ - if (!prebuilt->compress_heap) { - prebuilt->compress_heap = - mem_heap_create(max(UNIV_PAGE_SIZE, uncomp_len)); - } - - buf_len = uncomp_len; - buf = static_cast(mem_heap_zalloc( - prebuilt->compress_heap, buf_len)); - - /* init d_stream */ - d_stream.next_in = const_cast(data); - d_stream.avail_in = comp_len; - d_stream.next_out = buf; - d_stream.avail_out = buf_len; - - column_zip_set_alloc(&d_stream, prebuilt->compress_heap); - - window_bits = wrap ? MAX_WBITS : -MAX_WBITS; - err = inflateInit2(&d_stream, window_bits); - ut_a(err == Z_OK); - - err = inflate(&d_stream, Z_FINISH); - if (err == Z_NEED_DICT) { - ut_a(dict_data != 0 && dict_data_len != 0); - err = inflateSetDictionary(&d_stream, dict_data, - dict_data_len); - ut_a(err == Z_OK); - err = inflate(&d_stream, Z_FINISH); - } - - if (err != Z_STREAM_END) { - inflateEnd(&d_stream); - if (err == Z_BUF_ERROR && d_stream.avail_in == 0) - err = Z_DATA_ERROR; - } else { - buf_len = d_stream.total_out; - err = inflateEnd(&d_stream); - } - - switch (err) { - case Z_OK: - break; - case Z_BUF_ERROR: - ib_logf(IB_LOG_LEVEL_FATAL, - "zlib buf error, this shouldn't happen\n"); - break; - default: - ib_logf(IB_LOG_LEVEL_FATAL, - "failed to decompress column, error: %d\n", err); - } - - if (err == Z_OK) { - if (buf_len != uncomp_len) { - ib_logf(IB_LOG_LEVEL_FATAL, - "failed to decompress blob column, may" - " be corrupted\n"); - } - *len = buf_len; - return buf; - } - - *len -= (zip_column_header_length + lenlen); - return data; -} - - /*******************************************************************//** Stores a reference to a BLOB in the MySQL format. */ UNIV_INTERN @@ -713,21 +242,10 @@ row_mysql_store_blob_ref( to 4 bytes */ const void* data, /*!< in: BLOB data; if the value to store is SQL NULL this should be NULL pointer */ - ulint len, /*!< in: BLOB length; if the value to store + ulint len) /*!< in: BLOB length; if the value to store is SQL NULL this should be 0; remember also to set the NULL bit in the MySQL record header! */ - bool need_decompression, - /*!< in: if the data need to be compressed*/ - const byte* dict_data, - /*!< in: optional compression dictionary - data */ - ulint dict_data_len, - /*!< in: optional compression dictionary data - length */ - row_prebuilt_t* prebuilt) - /*compress_heap only - here */ { /* MySQL might assume the field is set to zero except the length and the pointer fields */ @@ -739,27 +257,11 @@ row_mysql_store_blob_ref( In 32-bit architectures we only use the first 4 bytes of the pointer slot. */ - ut_a(col_len - 8 > 1 || - len < 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - ut_a(col_len - 8 > 2 || - len < 256 * 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - ut_a(col_len - 8 > 3 || - len < 256 * 256 * 256 + - (need_decompression ? ZIP_COLUMN_HEADER_LENGTH : 0)); - - const byte *ptr = NULL; - - if (need_decompression) - ptr = row_decompress_column((const byte*)data, &len, - dict_data, dict_data_len, prebuilt); - - if (ptr) - memcpy(dest + col_len - 8, &ptr, sizeof ptr); - else - memcpy(dest + col_len - 8, &data, sizeof data); + ut_a(col_len - 8 > 1 || len < 256); + ut_a(col_len - 8 > 2 || len < 256 * 256); + ut_a(col_len - 8 > 3 || len < 256 * 256 * 256); + memcpy(dest + col_len - 8, &data, sizeof data); mach_write_to_n_little_endian(dest, col_len - 8, len); } @@ -773,32 +275,15 @@ row_mysql_read_blob_ref( ulint* len, /*!< out: BLOB length */ const byte* ref, /*!< in: BLOB reference in the MySQL format */ - ulint col_len, /*!< in: BLOB reference length + ulint col_len) /*!< in: BLOB reference length (not BLOB length) */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap - only here */ { byte* data = NULL; - byte* ptr = NULL; *len = mach_read_from_n_little_endian(ref, col_len - 8); memcpy(&data, ref + col_len - 8, sizeof data); - if (need_compression) { - ptr = row_compress_column(data, len, col_len - 8, dict_data, - dict_data_len, prebuilt); - if (ptr) - data = ptr; - } - return(data); } @@ -881,16 +366,7 @@ row_mysql_store_col_in_innobase_format( necessarily the length of the actual payload data; if the column is a true VARCHAR then this is irrelevant */ - ulint comp, /*!< in: nonzero=compact format */ - bool need_compression, - /*!< in: if the data need to be - compressed*/ - const byte* dict_data, /*!< in: optional compression - dictionary data */ - ulint dict_data_len, /*!< in: optional compression - dictionary data length */ - row_prebuilt_t* prebuilt) /*!< in: use prebuilt->compress_heap - only here */ + ulint comp) /*!< in: nonzero=compact format */ { const byte* ptr = mysql_data; const dtype_t* dtype; @@ -943,14 +419,8 @@ row_mysql_store_col_in_innobase_format( lenlen = 2; } - const byte* tmp_ptr = row_mysql_read_true_varchar( + ptr = row_mysql_read_true_varchar( &col_len, mysql_data, lenlen); - if (need_compression) - ptr = row_compress_column(tmp_ptr, &col_len, - lenlen, dict_data, dict_data_len, - prebuilt); - else - ptr = tmp_ptr; } else { /* Remove trailing spaces from old style VARCHAR columns. */ @@ -1032,9 +502,7 @@ row_mysql_store_col_in_innobase_format( } } else if (type == DATA_BLOB && row_format_col) { - ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len, - need_compression, dict_data, dict_data_len, - prebuilt); + ptr = row_mysql_read_blob_ref(&col_len, mysql_data, col_len); } dfield_set_data(dfield, ptr, col_len); @@ -1092,11 +560,7 @@ row_mysql_convert_row_to_innobase( TRUE, /* MySQL row format data */ mysql_rec + templ->mysql_col_offset, templ->mysql_col_len, - dict_table_is_comp(prebuilt->table), - templ->compressed, - reinterpret_cast( - templ->zip_dict_data.str), - templ->zip_dict_data.length, prebuilt); + dict_table_is_comp(prebuilt->table)); next_column: ; } @@ -1442,10 +906,6 @@ row_prebuilt_free( mem_heap_free(prebuilt->blob_heap); } - if (prebuilt->compress_heap) { - mem_heap_free(prebuilt->compress_heap); - } - if (prebuilt->old_vers_heap) { mem_heap_free(prebuilt->old_vers_heap); } @@ -1875,9 +1335,6 @@ row_insert_for_mysql( return(DB_READ_ONLY); } - if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) - mem_heap_empty(prebuilt->compress_heap); - trx->op_info = "inserting"; row_mysql_delay_if_needed(); @@ -3443,8 +2900,6 @@ row_mysql_table_id_reassign( " WHERE TABLE_ID = :old_id;\n" "UPDATE SYS_INDEXES SET TABLE_ID = :new_id\n" " WHERE TABLE_ID = :old_id;\n" - "UPDATE SYS_ZIP_DICT_COLS SET TABLE_ID = :new_id_narrow\n" - " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n", FALSE, trx); return(err); @@ -4228,9 +3683,6 @@ row_truncate_table_for_mysql( "UPDATE SYS_INDEXES" " SET TABLE_ID = :new_id, SPACE = :new_space\n" " WHERE TABLE_ID = :old_id;\n" - "UPDATE SYS_ZIP_DICT_COLS\n" - " SET TABLE_ID = :new_id_narrow\n" - " WHERE TABLE_ID = :old_id_narrow;\n" "END;\n" , FALSE, trx); @@ -4858,19 +4310,6 @@ row_drop_table_for_mysql( filepath = fil_make_ibd_name(tablename, false); } - /* Remove all compression dictionary references for the - table */ - err = dict_create_remove_zip_dict_references_for_table( - table->id, trx); - if (err != DB_SUCCESS) { - ib_logf(IB_LOG_LEVEL_ERROR, "Error: (%s) not " - "able to remove compression dictionary " - "references for table %s", ut_strerr(err), - tablename); - - goto funct_exit; - } - if (dict_table_has_fts_index(table) || DICT_TF2_FLAG_IS_SET(table, DICT_TF2_FTS_HAS_DOC_ID)) { ut_ad(table->n_ref_count == 0); diff --git a/storage/xtradb/row/row0sel.cc b/storage/xtradb/row/row0sel.cc index ad1e9e2bf9d83..fd23d83e0e51e 100644 --- a/storage/xtradb/row/row0sel.cc +++ b/storage/xtradb/row/row0sel.cc @@ -2,6 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2459,8 +2460,7 @@ row_sel_convert_mysql_key_to_innobase( /* MySQL key value format col */ FALSE, key_ptr + data_offset, data_len, - dict_table_is_comp(index->table), - false, 0, 0 ,0); + dict_table_is_comp(index->table)); ut_a(buf <= original_buf + buf_len); } @@ -2554,15 +2554,15 @@ row_sel_store_row_id_to_prebuilt( #ifdef UNIV_DEBUG /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ # define row_sel_field_store_in_mysql_format( \ - dest,templ,idx,field,src,len,prebuilt) \ + dest,templ,idx,field,src,len) \ row_sel_field_store_in_mysql_format_func \ - (dest,templ,idx,field,src,len, prebuilt) + (dest,templ,idx,field,src,len) #else /* UNIV_DEBUG */ /** Convert a non-SQL-NULL field from Innobase format to MySQL format. */ # define row_sel_field_store_in_mysql_format( \ - dest,templ,idx,field,src,len,prebuilt) \ + dest,templ,idx,field,src,len) \ row_sel_field_store_in_mysql_format_func \ - (dest,templ,src,len, prebuilt) + (dest,templ,src,len) #endif /* UNIV_DEBUG */ /**************************************************************//** @@ -2592,10 +2592,7 @@ row_sel_field_store_in_mysql_format_func( templ->icp_rec_field_no */ #endif /* UNIV_DEBUG */ const byte* data, /*!< in: data to store */ - ulint len, /*!< in: length of the data */ - row_prebuilt_t* prebuilt) - /*!< in: use prebuilt->compress_heap - only here */ + ulint len) /*!< in: length of the data */ { byte* ptr; #ifdef UNIV_DEBUG @@ -2639,15 +2636,6 @@ row_sel_field_store_in_mysql_format_func( field_end = dest + templ->mysql_col_len; if (templ->mysql_type == DATA_MYSQL_TRUE_VARCHAR) { - /* If this is a compressed column, - decompress it first */ - if (templ->compressed) - data = row_decompress_column(data, &len, - reinterpret_cast( - templ->zip_dict_data.str), - templ->zip_dict_data.length, - prebuilt); - /* This is a >= 5.0.3 type true VARCHAR. Store the length of the data to the first byte or the first two bytes of dest. */ @@ -2698,11 +2686,7 @@ row_sel_field_store_in_mysql_format_func( already copied to the buffer in row_sel_store_mysql_rec */ row_mysql_store_blob_ref(dest, templ->mysql_col_len, data, - len, templ->compressed, - reinterpret_cast( - templ->zip_dict_data.str), - templ->zip_dict_data.length, - prebuilt); + len); break; case DATA_MYSQL: @@ -2855,7 +2839,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len, prebuilt); + templ, index, field_no, data, len); if (heap != prebuilt->blob_heap) { mem_heap_free(heap); @@ -2905,7 +2889,7 @@ row_sel_store_mysql_field_func( row_sel_field_store_in_mysql_format( mysql_rec + templ->mysql_col_offset, - templ, index, field_no, data, len, prebuilt); + templ, index, field_no, data, len); } ut_ad(len != UNIV_SQL_NULL); @@ -2953,9 +2937,6 @@ row_sel_store_mysql_rec( prebuilt->blob_heap = NULL; } - if (UNIV_LIKELY_NULL(prebuilt->compress_heap)) - mem_heap_empty(prebuilt->compress_heap); - for (i = 0; i < prebuilt->n_template; i++) { const mysql_row_templ_t*templ = &prebuilt->mysql_template[i]; const ulint field_no diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index cae3b71c2bb66..9f426a8b51331 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -3,6 +3,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. +Copyright (c) 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -2776,12 +2777,6 @@ innobase_start_or_create_for_mysql(void) } } - /* Create the SYS_ZIP_DICT system table */ - err = dict_create_or_check_sys_zip_dict(); - if (err != DB_SUCCESS) { - return(err); - } - srv_is_being_started = FALSE; ut_a(trx_purge_state() == PURGE_STATE_INIT); From bc4cac358ed0d08daaa3cb869b3ecc444afa09fa Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Wed, 4 Jan 2017 13:26:09 +0100 Subject: [PATCH 21/74] MDEV-10035: DBUG_ASSERT on CREATE VIEW v1 AS SELECT * FROM t1 FOR UPDATE Ability to print lock type added. Restoring correct lock type for CREATE VIEW added. --- mysql-test/r/view.result | 25 +++++++++++++++++++++++++ mysql-test/t/view.test | 22 ++++++++++++++++++++++ sql/sql_lex.cc | 1 + sql/sql_lex.h | 3 +++ sql/sql_select.cc | 6 ++++++ sql/sql_view.cc | 9 +++++++++ sql/sql_yacc.yy | 2 ++ 7 files changed, 68 insertions(+) diff --git a/mysql-test/r/view.result b/mysql-test/r/view.result index 924b3a11fef1c..6530fd10a6a7e 100644 --- a/mysql-test/r/view.result +++ b/mysql-test/r/view.result @@ -5799,6 +5799,31 @@ f1 f2 drop table t1, t2; SELECT 1 FROM (SELECT 1 as a) AS b HAVING (SELECT `SOME_GARBAGE`.b.a)=1; ERROR 42S22: Unknown column 'SOME_GARBAGE.b.a' in 'field list' +# +# MDEV-10035: DBUG_ASSERT on CREATE VIEW v1 AS SELECT * FROM t1 +# FOR UPDATE +# +CREATE TABLE t1 (a INT); +insert into t1 values (1),(2); +CREATE VIEW v1 AS SELECT * FROM t1 FOR UPDATE; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `t1`.`a` AS `a` from `t1` for update latin1 latin1_swedish_ci +select * from v1; +a +1 +2 +DROP VIEW v1; +CREATE VIEW v1 AS SELECT * FROM t1 LOCK IN SHARE MODE; +SHOW CREATE VIEW v1; +View Create View character_set_client collation_connection +v1 CREATE ALGORITHM=UNDEFINED DEFINER=`root`@`localhost` SQL SECURITY DEFINER VIEW `v1` AS select `t1`.`a` AS `a` from `t1` lock in share mode latin1 latin1_swedish_ci +select * from v1; +a +1 +2 +DROP VIEW v1; +DROP TABLE t1; # ----------------------------------------------------------------- # -- End of 10.0 tests. # ----------------------------------------------------------------- diff --git a/mysql-test/t/view.test b/mysql-test/t/view.test index 241b3b414f6b8..5ea97f6465ec2 100644 --- a/mysql-test/t/view.test +++ b/mysql-test/t/view.test @@ -5694,6 +5694,28 @@ drop table t1, t2; --error ER_BAD_FIELD_ERROR SELECT 1 FROM (SELECT 1 as a) AS b HAVING (SELECT `SOME_GARBAGE`.b.a)=1; + +--echo # +--echo # MDEV-10035: DBUG_ASSERT on CREATE VIEW v1 AS SELECT * FROM t1 +--echo # FOR UPDATE +--echo # + +CREATE TABLE t1 (a INT); +insert into t1 values (1),(2); + +CREATE VIEW v1 AS SELECT * FROM t1 FOR UPDATE; +SHOW CREATE VIEW v1; +select * from v1; +DROP VIEW v1; + +CREATE VIEW v1 AS SELECT * FROM t1 LOCK IN SHARE MODE; +SHOW CREATE VIEW v1; +select * from v1; +DROP VIEW v1; + +DROP TABLE t1; + + --echo # ----------------------------------------------------------------- --echo # -- End of 10.0 tests. --echo # ----------------------------------------------------------------- diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 9d5f4cfcb5b65..d20c5ae78aff5 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -1943,6 +1943,7 @@ void st_select_lex::init_select() m_agg_func_used= false; name_visibility_map= 0; join= 0; + lock_type= TL_READ_DEFAULT; } /* diff --git a/sql/sql_lex.h b/sql/sql_lex.h index 019231e20046e..c4fd109009fd6 100644 --- a/sql/sql_lex.h +++ b/sql/sql_lex.h @@ -846,6 +846,9 @@ class st_select_lex: public st_select_lex_node /* namp of nesting SELECT visibility (for aggregate functions check) */ nesting_map name_visibility_map; + /* it is for correct printing SELECT options */ + thr_lock_type lock_type; + void init_query(); void init_select(); st_select_lex_unit* master_unit(); diff --git a/sql/sql_select.cc b/sql/sql_select.cc index f345d3c96876c..09ceb16fcba75 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -24592,6 +24592,12 @@ void st_select_lex::print(THD *thd, String *str, enum_query_type query_type) // limit print_limit(thd, str, query_type); + // lock type + if (lock_type == TL_READ_WITH_SHARED_LOCKS) + str->append(" lock in share mode"); + else if (lock_type == TL_WRITE) + str->append(" for update"); + // PROCEDURE unsupported here } diff --git a/sql/sql_view.cc b/sql/sql_view.cc index f3717e3ded269..9fe4dd4849dda 100644 --- a/sql/sql_view.cc +++ b/sql/sql_view.cc @@ -430,6 +430,15 @@ bool mysql_create_view(THD *thd, TABLE_LIST *views, lex->link_first_table_back(view, link_to_local); view->open_type= OT_BASE_ONLY; + /* + ignore lock specs for CREATE statement + */ + if (lex->current_select->lock_type != TL_READ_DEFAULT) + { + lex->current_select->set_lock_for_tables(TL_READ_DEFAULT); + view->mdl_request.set_type(MDL_EXCLUSIVE); + } + if (open_temporary_tables(thd, lex->query_tables) || open_and_lock_tables(thd, lex->query_tables, TRUE, 0)) { diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index c9fd000141af7..bf35449643354 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -8491,12 +8491,14 @@ select_lock_type: | FOR_SYM UPDATE_SYM { LEX *lex=Lex; + lex->current_select->lock_type= TL_WRITE; lex->current_select->set_lock_for_tables(TL_WRITE); lex->safe_to_cache_query=0; } | LOCK_SYM IN_SYM SHARE_SYM MODE_SYM { LEX *lex=Lex; + lex->current_select->lock_type= TL_READ_WITH_SHARED_LOCKS; lex->current_select-> set_lock_for_tables(TL_READ_WITH_SHARED_LOCKS); lex->safe_to_cache_query=0; From 9bf92706d19761722b46d66a671734466cb6e98e Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 1 Jan 2017 19:35:44 +0200 Subject: [PATCH 22/74] MDEV-8518 rpl.sec_behind_master-5114 fails sporadically in buildbot - fix the test to avoid false-negatives before MDEV-5114 patch; - fix the race condition which made the test fail on slow builders --- .../suite/rpl/r/sec_behind_master-5114.result | 3 +- .../suite/rpl/t/sec_behind_master-5114.test | 42 +++++++++++++++++-- 2 files changed, 41 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/rpl/r/sec_behind_master-5114.result b/mysql-test/suite/rpl/r/sec_behind_master-5114.result index 5554b17347de1..17f50f0b61225 100644 --- a/mysql-test/suite/rpl/r/sec_behind_master-5114.result +++ b/mysql-test/suite/rpl/r/sec_behind_master-5114.result @@ -3,7 +3,8 @@ include/master-slave.inc call mtr.add_suppression("Unsafe statement written to the binary log"); CREATE TABLE t1 (a int); INSERT INTO t1 VALUES(SLEEP(2)); -Seconds_Behind_Master: 1 +Seconds_Behind_Master_is_less_than_100 +1 Warnings: Note 1592 Unsafe statement written to the binary log using statement format since BINLOG_FORMAT = STATEMENT. Statement is unsafe because it uses a system function that may return a different value on the slave. drop table t1; diff --git a/mysql-test/suite/rpl/t/sec_behind_master-5114.test b/mysql-test/suite/rpl/t/sec_behind_master-5114.test index 8c70da4f0a878..ff8cab54c4f81 100644 --- a/mysql-test/suite/rpl/t/sec_behind_master-5114.test +++ b/mysql-test/suite/rpl/t/sec_behind_master-5114.test @@ -3,24 +3,60 @@ source include/have_binlog_format_statement.inc; call mtr.add_suppression("Unsafe statement written to the binary log"); + +# Make sure that the start time of the first event is certainly different +# from the next event +--let $timestamp= `SELECT @@timestamp` +--disable_query_log +eval SET TIMESTAMP= $timestamp-100; +--enable_query_log CREATE TABLE t1 (a int); + +# Make sure that the slave is done with the first event, and all checks +# that we'll perform later will be really against the second event +sync_slave_with_master; + +connection master; + +# Restore the timestamp now. It doesn't matter that it's not precise, +# it just needs to be very different from the earlier event +--disable_query_log +eval SET TIMESTAMP= $timestamp; +--enable_query_log + send INSERT INTO t1 VALUES(SLEEP(2)); connection slave; -let $run = 10; + +# When the slave starts executing the event, Seconds_Behind_Master +# should start growing steadilly. The bugfix ensures that they are +# calculated based on the start time of the current event, rather +# than the start time of the previous event. To check it, we only need +# the first non-zero value + +let $run = 20; while ($run) { dec $run; let $sbm=query_get_value(SHOW SLAVE STATUS, Seconds_Behind_Master, 1); # for debugging uncomment echo and remove the if() - #echo Seconds_Behind_Master: $sbm; +# echo Seconds_Behind_Master: $sbm; if ($sbm) { let $run = 0; } sleep 0.5; } -echo Seconds_Behind_Master: $sbm; + +# Normally the first non-zero value should be 1. However, due to race +# conditions on slow servers, sometimes the check might miss the value 1, +# and only catch a higher one. It does not matter, we just need to make +# sure it didn't start with 100+, as it would have with bug MDEV-5114 + +--disable_query_log +eval SELECT $sbm > 0 and $sbm < 99 AS Seconds_Behind_Master_is_less_than_100; +--enable_query_log + connection master; reap; drop table t1; From 0f8e17af92cd0126aadf283e25edcd64872b2ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 4 Jan 2017 18:16:37 +0200 Subject: [PATCH 23/74] Part 1 of MDEV-8139 Fix scrubbing tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Port a bug fix from MySQL 5.7, so that all undo log pages will be freed during a slow shutdown. We cannot scrub pages that are left allocated. commit 173e171c6fb55f064eea278c76fbb28e2b1c757b Author: Thirunarayanan Balathandayuthapani Date: Fri Sep 9 18:01:27 2016 +0530 Bug #24450908 UNDO LOG EXISTS AFTER SLOW SHUTDOWN Problem: ======== 1) cached undo segment is not removed from rollback segment history (RSEG_HISTORY) during slow shutdown. In other words, If the segment is not completely free, we are failing to remove an entry from the history list. While starting the server, we traverse all rollback segment slots history list and make it as list of undo logs to be purged in purge queue. In that case, purge queue will never be empty after slow shutdown. 2) Freeing of undo log segment is linked with removing undo log header from history. Fix: ==== 1) Have separate logic of removing the undo log header from history list from rollback segment slots and remove it from rollback segment history even though it is not completely free. Reviewed-by: Debarun Banerjee Reviewed-by: Marko Mäkelä RB:13672 --- storage/innobase/fut/fut0lst.cc | 98 ------------------------------ storage/innobase/include/fut0lst.h | 25 -------- storage/innobase/trx/trx0purge.cc | 67 ++++++++++---------- storage/xtradb/fut/fut0lst.cc | 98 ------------------------------ storage/xtradb/include/fut0lst.h | 25 -------- storage/xtradb/trx/trx0purge.cc | 67 ++++++++++---------- 6 files changed, 62 insertions(+), 318 deletions(-) diff --git a/storage/innobase/fut/fut0lst.cc b/storage/innobase/fut/fut0lst.cc index 8f96a6426d2e9..dd3fa1238d926 100644 --- a/storage/innobase/fut/fut0lst.cc +++ b/storage/innobase/fut/fut0lst.cc @@ -338,104 +338,6 @@ flst_remove( mlog_write_ulint(base + FLST_LEN, len - 1, MLOG_4BYTES, mtr); } -/********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - ulint space; - flst_node_t* node1; - fil_addr_t node1_addr; - fil_addr_t node2_addr; - ulint len; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX)); - ut_ad(n_nodes > 0); - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - node1_addr = flst_get_prev_addr(node2, mtr); - - if (!fil_addr_is_null(node1_addr)) { - - /* Update next field of node1 */ - - if (node1_addr.page == node2_addr.page) { - - node1 = page_align(node2) + node1_addr.boffset; - } else { - node1 = fut_get_ptr(space, - fil_space_get_zip_size(space), - node1_addr, RW_X_LATCH, mtr); - } - - flst_write_addr(node1 + FLST_NEXT, fil_addr_null, mtr); - } else { - /* node2 was first in list: update the field in base */ - flst_write_addr(base + FLST_FIRST, fil_addr_null, mtr); - } - - flst_write_addr(base + FLST_LAST, node1_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base, mtr); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - fil_addr_t node2_addr; - ulint len; - ulint space; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX)); - if (n_nodes == 0) { - - ut_ad(fil_addr_is_null(flst_get_next_addr(node2, mtr))); - - return; - } - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - /* Update next field of node2 */ - flst_write_addr(node2 + FLST_NEXT, fil_addr_null, mtr); - - flst_write_addr(base + FLST_LAST, node2_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base, mtr); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - /********************************************************************//** Validates a file-based list. @return TRUE if ok */ diff --git a/storage/innobase/include/fut0lst.h b/storage/innobase/include/fut0lst.h index 90f9a65d4fac3..8554cc60cdde4 100644 --- a/storage/innobase/include/fut0lst.h +++ b/storage/innobase/include/fut0lst.h @@ -102,31 +102,6 @@ flst_remove( flst_node_t* node2, /*!< in: node to remove */ mtr_t* mtr); /*!< in: mini-transaction handle */ /********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** Gets list length. @return length */ UNIV_INLINE diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index efc600d16b1cc..42b94d9b6a3b6 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -281,18 +281,33 @@ trx_purge_add_update_undo_to_history( } } -/**********************************************************************//** -Frees an undo log segment which is in the history list. Cuts the end of the -history list at the youngest undo log in this segment. */ +/** Remove undo log header from the history list. +@param[in,out] rseg_hdr rollback segment header +@param[in] log_hdr undo log segment header +@param[in,out] mtr mini transaction. */ +static +void +trx_purge_remove_log_hdr( + trx_rsegf_t* rseg_hdr, + trx_ulogf_t* log_hdr, + mtr_t* mtr) +{ + flst_remove(rseg_hdr + TRX_RSEG_HISTORY, + log_hdr + TRX_UNDO_HISTORY_NODE, mtr); + + os_atomic_decrement_ulint(&trx_sys->rseg_history_len, 1); +} + +/** Frees an undo log segment which is in the history list. Removes the +undo log hdr from the history list. +@param[in,out] rseg rollback segment +@param[in] hdr_addr file address of log_hdr +@param[in] noredo skip redo logging. */ static void trx_purge_free_segment( -/*===================*/ - trx_rseg_t* rseg, /*!< in: rollback segment */ - fil_addr_t hdr_addr, /*!< in: the file address of log_hdr */ - ulint n_removed_logs) /*!< in: count of how many undo logs we - will cut off from the end of the - history list */ + trx_rseg_t* rseg, + fil_addr_t hdr_addr) { mtr_t mtr; trx_rsegf_t* rseg_hdr; @@ -356,16 +371,7 @@ trx_purge_free_segment( history list: otherwise, in case of a database crash, the segment could become inaccessible garbage in the file space. */ - flst_cut_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, n_removed_logs, &mtr); - -#ifdef HAVE_ATOMIC_BUILTINS - os_atomic_decrement_ulint(&trx_sys->rseg_history_len, n_removed_logs); -#else - mutex_enter(&trx_sys->mutex); - trx_sys->rseg_history_len -= n_removed_logs; - mutex_exit(&trx_sys->mutex); -#endif /* HAVE_ATOMIC_BUILTINS */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); do { @@ -407,7 +413,6 @@ trx_purge_truncate_rseg_history( page_t* undo_page; trx_ulogf_t* log_hdr; trx_usegf_t* seg_hdr; - ulint n_removed_logs = 0; mtr_t mtr; trx_id_t undo_trx_no; @@ -445,19 +450,6 @@ trx_purge_truncate_rseg_history( hdr_addr.boffset, limit->undo_no); } -#ifdef HAVE_ATOMIC_BUILTINS - os_atomic_decrement_ulint( - &trx_sys->rseg_history_len, n_removed_logs); -#else - mutex_enter(&trx_sys->mutex); - trx_sys->rseg_history_len -= n_removed_logs; - mutex_exit(&trx_sys->mutex); -#endif /* HAVE_ATOMIC_BUILTINS */ - - flst_truncate_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, - n_removed_logs, &mtr); - mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); @@ -466,7 +458,6 @@ trx_purge_truncate_rseg_history( prev_hdr_addr = trx_purge_get_log_from_hist( flst_get_prev_addr(log_hdr + TRX_UNDO_HISTORY_NODE, &mtr)); - n_removed_logs++; seg_hdr = undo_page + TRX_UNDO_SEG_HDR; @@ -478,10 +469,14 @@ trx_purge_truncate_rseg_history( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - trx_purge_free_segment(rseg, hdr_addr, n_removed_logs); + /* calls the trx_purge_remove_log_hdr() + inside trx_purge_free_segment(). */ + trx_purge_free_segment(rseg, hdr_addr); - n_removed_logs = 0; } else { + /* Remove the log hdr from the rseg history. */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); + mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); } diff --git a/storage/xtradb/fut/fut0lst.cc b/storage/xtradb/fut/fut0lst.cc index 8f96a6426d2e9..dd3fa1238d926 100644 --- a/storage/xtradb/fut/fut0lst.cc +++ b/storage/xtradb/fut/fut0lst.cc @@ -338,104 +338,6 @@ flst_remove( mlog_write_ulint(base + FLST_LEN, len - 1, MLOG_4BYTES, mtr); } -/********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - ulint space; - flst_node_t* node1; - fil_addr_t node1_addr; - fil_addr_t node2_addr; - ulint len; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX)); - ut_ad(n_nodes > 0); - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - node1_addr = flst_get_prev_addr(node2, mtr); - - if (!fil_addr_is_null(node1_addr)) { - - /* Update next field of node1 */ - - if (node1_addr.page == node2_addr.page) { - - node1 = page_align(node2) + node1_addr.boffset; - } else { - node1 = fut_get_ptr(space, - fil_space_get_zip_size(space), - node1_addr, RW_X_LATCH, mtr); - } - - flst_write_addr(node1 + FLST_NEXT, fil_addr_null, mtr); - } else { - /* node2 was first in list: update the field in base */ - flst_write_addr(base + FLST_FIRST, fil_addr_null, mtr); - } - - flst_write_addr(base + FLST_LAST, node1_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base, mtr); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr) /*!< in: mini-transaction handle */ -{ - fil_addr_t node2_addr; - ulint len; - ulint space; - - ut_ad(mtr && node2 && base); - ut_ad(mtr_memo_contains_page(mtr, base, MTR_MEMO_PAGE_X_FIX)); - ut_ad(mtr_memo_contains_page(mtr, node2, MTR_MEMO_PAGE_X_FIX)); - if (n_nodes == 0) { - - ut_ad(fil_addr_is_null(flst_get_next_addr(node2, mtr))); - - return; - } - - buf_ptr_get_fsp_addr(node2, &space, &node2_addr); - - /* Update next field of node2 */ - flst_write_addr(node2 + FLST_NEXT, fil_addr_null, mtr); - - flst_write_addr(base + FLST_LAST, node2_addr, mtr); - - /* Update len of base node */ - len = flst_get_len(base, mtr); - ut_ad(len >= n_nodes); - - mlog_write_ulint(base + FLST_LEN, len - n_nodes, MLOG_4BYTES, mtr); -} - /********************************************************************//** Validates a file-based list. @return TRUE if ok */ diff --git a/storage/xtradb/include/fut0lst.h b/storage/xtradb/include/fut0lst.h index 90f9a65d4fac3..8554cc60cdde4 100644 --- a/storage/xtradb/include/fut0lst.h +++ b/storage/xtradb/include/fut0lst.h @@ -102,31 +102,6 @@ flst_remove( flst_node_t* node2, /*!< in: node to remove */ mtr_t* mtr); /*!< in: mini-transaction handle */ /********************************************************************//** -Cuts off the tail of the list, including the node given. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_cut_end( -/*=========*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node to remove */ - ulint n_nodes,/*!< in: number of nodes to remove, - must be >= 1 */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** -Cuts off the tail of the list, not including the given node. The number of -nodes which will be removed must be provided by the caller, as this function -does not measure the length of the tail. */ -UNIV_INTERN -void -flst_truncate_end( -/*==============*/ - flst_base_node_t* base, /*!< in: pointer to base node of list */ - flst_node_t* node2, /*!< in: first node not to remove */ - ulint n_nodes,/*!< in: number of nodes to remove */ - mtr_t* mtr); /*!< in: mini-transaction handle */ -/********************************************************************//** Gets list length. @return length */ UNIV_INLINE diff --git a/storage/xtradb/trx/trx0purge.cc b/storage/xtradb/trx/trx0purge.cc index d9e40c5d6f513..57338a7345068 100644 --- a/storage/xtradb/trx/trx0purge.cc +++ b/storage/xtradb/trx/trx0purge.cc @@ -285,18 +285,33 @@ trx_purge_add_update_undo_to_history( } } -/**********************************************************************//** -Frees an undo log segment which is in the history list. Cuts the end of the -history list at the youngest undo log in this segment. */ +/** Remove undo log header from the history list. +@param[in,out] rseg_hdr rollback segment header +@param[in] log_hdr undo log segment header +@param[in,out] mtr mini transaction. */ +static +void +trx_purge_remove_log_hdr( + trx_rsegf_t* rseg_hdr, + trx_ulogf_t* log_hdr, + mtr_t* mtr) +{ + flst_remove(rseg_hdr + TRX_RSEG_HISTORY, + log_hdr + TRX_UNDO_HISTORY_NODE, mtr); + + os_atomic_decrement_ulint(&trx_sys->rseg_history_len, 1); +} + +/** Frees an undo log segment which is in the history list. Removes the +undo log hdr from the history list. +@param[in,out] rseg rollback segment +@param[in] hdr_addr file address of log_hdr +@param[in] noredo skip redo logging. */ static void trx_purge_free_segment( -/*===================*/ - trx_rseg_t* rseg, /*!< in: rollback segment */ - fil_addr_t hdr_addr, /*!< in: the file address of log_hdr */ - ulint n_removed_logs) /*!< in: count of how many undo logs we - will cut off from the end of the - history list */ + trx_rseg_t* rseg, + fil_addr_t hdr_addr) { mtr_t mtr; trx_rsegf_t* rseg_hdr; @@ -360,16 +375,7 @@ trx_purge_free_segment( history list: otherwise, in case of a database crash, the segment could become inaccessible garbage in the file space. */ - flst_cut_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, n_removed_logs, &mtr); - -#ifdef HAVE_ATOMIC_BUILTINS - os_atomic_decrement_ulint(&trx_sys->rseg_history_len, n_removed_logs); -#else - mutex_enter(&trx_sys->mutex); - trx_sys->rseg_history_len -= n_removed_logs; - mutex_exit(&trx_sys->mutex); -#endif /* HAVE_ATOMIC_BUILTINS */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); do { @@ -411,7 +417,6 @@ trx_purge_truncate_rseg_history( page_t* undo_page; trx_ulogf_t* log_hdr; trx_usegf_t* seg_hdr; - ulint n_removed_logs = 0; mtr_t mtr; trx_id_t undo_trx_no; @@ -449,19 +454,6 @@ trx_purge_truncate_rseg_history( hdr_addr.boffset, limit->undo_no); } -#ifdef HAVE_ATOMIC_BUILTINS - os_atomic_decrement_ulint( - &trx_sys->rseg_history_len, n_removed_logs); -#else - mutex_enter(&trx_sys->mutex); - trx_sys->rseg_history_len -= n_removed_logs; - mutex_exit(&trx_sys->mutex); -#endif /* HAVE_ATOMIC_BUILTINS */ - - flst_truncate_end(rseg_hdr + TRX_RSEG_HISTORY, - log_hdr + TRX_UNDO_HISTORY_NODE, - n_removed_logs, &mtr); - mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); @@ -470,7 +462,6 @@ trx_purge_truncate_rseg_history( prev_hdr_addr = trx_purge_get_log_from_hist( flst_get_prev_addr(log_hdr + TRX_UNDO_HISTORY_NODE, &mtr)); - n_removed_logs++; seg_hdr = undo_page + TRX_UNDO_SEG_HDR; @@ -482,10 +473,14 @@ trx_purge_truncate_rseg_history( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - trx_purge_free_segment(rseg, hdr_addr, n_removed_logs); + /* calls the trx_purge_remove_log_hdr() + inside trx_purge_free_segment(). */ + trx_purge_free_segment(rseg, hdr_addr); - n_removed_logs = 0; } else { + /* Remove the log hdr from the rseg history. */ + trx_purge_remove_log_hdr(rseg_hdr, log_hdr, &mtr); + mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); } From 719321e78e69249e796a907c17400dac14ef0921 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 4 Jan 2017 18:43:32 +0200 Subject: [PATCH 24/74] MDEV-11638 Encryption causes race conditions in InnoDB shutdown InnoDB shutdown failed to properly take fil_crypt_thread() into account. The encryption threads were signalled to shut down together with other non-critical tasks. This could be much too early in case of slow shutdown, which could need minutes to complete the purge. Furthermore, InnoDB failed to wait for the fil_crypt_thread() to actually exit before proceeding to the final steps of shutdown, causing the race conditions. Furthermore, the log_scrub_thread() was shut down way too early. Also it should remain until the SRV_SHUTDOWN_FLUSH_PHASE. fil_crypt_threads_end(): Remove. This would cause the threads to be terminated way too early. srv_buf_dump_thread_active, srv_dict_stats_thread_active, lock_sys->timeout_thread_active, log_scrub_thread_active, srv_monitor_active, srv_error_monitor_active: Remove a race condition between startup and shutdown, by setting these in the startup thread that creates threads, not in each created thread. In this way, once the flag is cleared, it will remain cleared during shutdown. srv_n_fil_crypt_threads_started, fil_crypt_threads_event: Declare in global rather than static scope. log_scrub_event, srv_log_scrub_thread_active, log_scrub_thread(): Declare in static rather than global scope. Let these be created by log_init() and freed by log_shutdown(). rotate_thread_t::should_shutdown(): Do not shut down before the SRV_SHUTDOWN_FLUSH_PHASE. srv_any_background_threads_are_active(): Remove. These checks now exist in logs_empty_and_mark_files_at_shutdown(). logs_empty_and_mark_files_at_shutdown(): Shut down the threads in the proper order. Keep fil_crypt_thread() and log_scrub_thread() alive until SRV_SHUTDOWN_FLUSH_PHASE, and check that they actually terminate. --- storage/innobase/buf/buf0dump.cc | 10 +- storage/innobase/dict/dict0stats_bg.cc | 10 +- storage/innobase/fil/fil0crypt.cc | 35 ++-- storage/innobase/handler/ha_innodb.cc | 4 +- storage/innobase/include/fil0crypt.h | 12 +- storage/innobase/include/log0log.h | 14 +- storage/innobase/include/srv0srv.h | 24 +-- storage/innobase/lock/lock0wait.cc | 9 +- storage/innobase/log/log0log.cc | 243 ++++++++++++------------- storage/innobase/srv/srv0srv.cc | 71 +------- storage/innobase/srv/srv0start.cc | 29 +-- storage/xtradb/buf/buf0dump.cc | 10 +- storage/xtradb/dict/dict0stats_bg.cc | 10 +- storage/xtradb/fil/fil0crypt.cc | 35 ++-- storage/xtradb/handler/ha_innodb.cc | 2 - storage/xtradb/include/fil0crypt.h | 12 +- storage/xtradb/include/log0log.h | 14 +- storage/xtradb/include/srv0srv.h | 25 +-- storage/xtradb/lock/lock0wait.cc | 9 +- storage/xtradb/log/log0log.cc | 239 +++++++++++------------- storage/xtradb/srv/srv0srv.cc | 71 +------- storage/xtradb/srv/srv0start.cc | 29 +-- 22 files changed, 339 insertions(+), 578 deletions(-) diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index 0abf7118b4f7d..934e47cea864b 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -682,15 +683,10 @@ again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(buf_dump_thread)( -/*============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(buf_dump_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_buf_dump_thread_active = TRUE; - buf_dump_status(STATUS_INFO, "Dumping buffer pool(s) not yet started"); buf_load_status(STATUS_INFO, "Loading buffer pool(s) not yet started"); @@ -720,7 +716,7 @@ DECLARE_THREAD(buf_dump_thread)( keep going even if we are in a shutdown state */); } - srv_buf_dump_thread_active = FALSE; + srv_buf_dump_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/innobase/dict/dict0stats_bg.cc b/storage/innobase/dict/dict0stats_bg.cc index 7aefa6a1d4d4b..45fa0fc03f083 100644 --- a/storage/innobase/dict/dict0stats_bg.cc +++ b/storage/innobase/dict/dict0stats_bg.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -525,15 +526,10 @@ statistics. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(dict_stats_thread)( -/*==============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(dict_stats_thread)(void*) { ut_a(!srv_read_only_mode); - srv_dict_stats_thread_active = TRUE; - while (!SHUTTING_DOWN()) { /* Wake up periodically even if not signaled. This is @@ -556,7 +552,7 @@ DECLARE_THREAD(dict_stats_thread)( os_event_reset(dict_stats_event); } - srv_dict_stats_thread_active = FALSE; + srv_dict_stats_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit instead of return(). */ diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 1eaabf6867888..2999bea276505 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,7 +56,7 @@ UNIV_INTERN ulong srv_encrypt_tables = 0; UNIV_INTERN uint srv_n_fil_crypt_threads = 0; /** No of key rotation threads started */ -static uint srv_n_fil_crypt_threads_started = 0; +UNIV_INTERN uint srv_n_fil_crypt_threads_started = 0; /** At this age or older a space/page will be rotated */ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; @@ -65,7 +65,7 @@ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; static os_event_t fil_crypt_event; /** Event to signal TO the key rotation threads. */ -static os_event_t fil_crypt_threads_event; +UNIV_INTERN os_event_t fil_crypt_threads_event; /** Event for waking up threads throttle */ static os_event_t fil_crypt_throttle_sleep_event; @@ -1303,10 +1303,20 @@ struct rotate_thread_t { btr_scrub_t scrub_data; /* thread local data used by btr_scrub-functions * when iterating pages of tablespace */ - /* check if this thread should shutdown */ + /** @return whether this thread should terminate */ bool should_shutdown() const { - return ! (srv_shutdown_state == SRV_SHUTDOWN_NONE && - thread_no < srv_n_fil_crypt_threads); + switch (srv_shutdown_state) { + case SRV_SHUTDOWN_NONE: + case SRV_SHUTDOWN_CLEANUP: + return thread_no >= srv_n_fil_crypt_threads; + case SRV_SHUTDOWN_FLUSH_PHASE: + return true; + case SRV_SHUTDOWN_LAST_PHASE: + case SRV_SHUTDOWN_EXIT_THREADS: + break; + } + ut_ad(0); + return true; } }; @@ -2458,18 +2468,6 @@ fil_crypt_threads_init() } } -/********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end() -/*===================*/ -{ - if (fil_crypt_threads_inited) { - fil_crypt_set_thread_cnt(0); - } -} - /********************************************************************* Clean up key rotation threads resources */ UNIV_INTERN @@ -2480,6 +2478,7 @@ fil_crypt_threads_cleanup() if (!fil_crypt_threads_inited) { return; } + ut_a(!srv_n_fil_crypt_threads_started); os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index b283bbda7f49a..625ae5f5d2760 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1,10 +1,10 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -73,7 +73,6 @@ MYSQL_PLUGIN_IMPORT extern char mysql_unpacked_real_data_home[]; #include "srv0srv.h" #include "trx0roll.h" #include "trx0trx.h" - #include "trx0sys.h" #include "rem0types.h" #include "row0ins.h" @@ -248,7 +247,6 @@ static char* internal_innobase_data_file_path = NULL; static char* innodb_version_str = (char*) INNODB_VERSION_STR; -extern uint srv_n_fil_crypt_threads; extern uint srv_fil_crypt_rotate_key_age; extern uint srv_n_fil_crypt_iops; diff --git a/storage/innobase/include/fil0crypt.h b/storage/innobase/include/fil0crypt.h index 9a35f6591e751..42cdafde4d0c4 100644 --- a/storage/innobase/include/fil0crypt.h +++ b/storage/innobase/include/fil0crypt.h @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,6 +26,8 @@ Created 04/01/2015 Jan Lindström #ifndef fil0crypt_h #define fil0crypt_h +#include "os0sync.h" + /** * Magic pattern in start of crypt data on page 0 */ @@ -45,6 +47,8 @@ typedef enum { FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */ } fil_encryption_t; +extern os_event_t fil_crypt_threads_event; + /** * CRYPT_SCHEME_UNENCRYPTED * @@ -391,12 +395,6 @@ fil_crypt_set_thread_cnt( /*=====================*/ uint new_cnt); /*!< in: requested #threads */ -/********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end(); - /********************************************************************* Cleanup resources for threads for key rotation */ UNIV_INTERN diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 0015ea15c350e..0f4cb611586e1 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1013,22 +1014,9 @@ struct log_t{ /* @} */ #endif /* UNIV_LOG_ARCHIVE */ -extern os_event_t log_scrub_event; /* log scrubbing speed, in bytes/sec */ extern ulonglong innodb_scrub_log_speed; -/*****************************************************************//** -This is the main thread for log scrub. It waits for an event and -when waked up fills current log block with dummy records and -sleeps again. -@return this function does not return, it calls os_thread_exit() */ -extern "C" UNIV_INTERN -os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg); /*!< in: a dummy parameter - required by os_thread_create */ - #ifndef UNIV_NONINL #include "log0log.ic" #endif diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 905cc80f0dfa3..1ba1e3ae9c65d 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -328,6 +328,9 @@ extern char** srv_data_file_names; extern ulint* srv_data_file_sizes; extern ulint* srv_data_file_is_raw_partition; +extern uint srv_n_fil_crypt_threads; +extern uint srv_n_fil_crypt_threads_started; + extern ibool srv_auto_extend_last_data_file; extern ulint srv_last_file_size_max; extern char* srv_log_group_home_dir; @@ -478,19 +481,17 @@ extern ibool srv_print_verbose_log; "tables instead, see " REFMAN "innodb-i_s-tables.html" extern ibool srv_print_innodb_table_monitor; -extern ibool srv_monitor_active; -extern ibool srv_error_monitor_active; +extern bool srv_monitor_active; +extern bool srv_error_monitor_active; /* TRUE during the lifetime of the buffer pool dump/load thread */ -extern ibool srv_buf_dump_thread_active; +extern bool srv_buf_dump_thread_active; /* TRUE during the lifetime of the stats thread */ -extern ibool srv_dict_stats_thread_active; +extern bool srv_dict_stats_thread_active; /* TRUE if enable log scrubbing */ extern my_bool srv_scrub_log; -/* TRUE during the lifetime of the log scrub thread */ -extern ibool srv_log_scrub_thread_active; extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; @@ -899,15 +900,6 @@ srv_release_threads( enum srv_thread_type type, /*!< in: thread type */ ulint n); /*!< in: number of threads to release */ -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -UNIV_INTERN -const char* -srv_any_background_threads_are_active(void); -/*=======================================*/ - /**********************************************************************//** Wakeup the purge threads. */ UNIV_INTERN diff --git a/storage/innobase/lock/lock0wait.cc b/storage/innobase/lock/lock0wait.cc index c7bd223c491d0..8f9ea7e10aa8d 100644 --- a/storage/innobase/lock/lock0wait.cc +++ b/storage/innobase/lock/lock0wait.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -510,11 +511,7 @@ A thread which wakes up threads whose lock wait may have lasted too long. @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(lock_wait_timeout_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /* in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(lock_wait_timeout_thread)(void*) { ib_int64_t sig_count = 0; os_event_t event = lock_sys->timeout_event; @@ -525,8 +522,6 @@ DECLARE_THREAD(lock_wait_timeout_thread)( pfs_register_thread(srv_lock_timeout_thread_key); #endif /* UNIV_PFS_THREAD */ - lock_sys->timeout_thread_active = true; - do { srv_slot_t* slot; diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index a832657136684..3c44baaf1de3d 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -42,10 +42,11 @@ Created 12/9/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0flu.h" #include "srv0srv.h" +#include "lock0lock.h" #include "log0recv.h" #include "fil0fil.h" #include "dict0boot.h" -#include "srv0srv.h" +#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0start.h" #include "trx0sys.h" #include "trx0trx.h" @@ -154,6 +155,15 @@ the previous */ #define LOG_ARCHIVE_READ 1 #define LOG_ARCHIVE_WRITE 2 +/** Event to wake up the log scrub thread */ +static os_event_t log_scrub_event; + +static bool log_scrub_thread_active; + +extern "C" UNIV_INTERN +os_thread_ret_t +DECLARE_THREAD(log_scrub_thread)(void*); + /******************************************************//** Completes a checkpoint write i/o to a log file. */ static @@ -943,6 +953,12 @@ log_init(void) mutex_exit(&(log_sys->mutex)); + log_scrub_thread_active = !srv_read_only_mode && srv_scrub_log; + if (log_scrub_thread_active) { + log_scrub_event = os_event_create(); + os_thread_create(log_scrub_thread, NULL, NULL); + } + #ifdef UNIV_LOG_DEBUG recv_sys_create(); recv_sys_init(buf_pool_get_curr_size()); @@ -3232,8 +3248,6 @@ logs_empty_and_mark_files_at_shutdown(void) ulint count = 0; ulint total_trx; ulint pending_io; - enum srv_thread_type active_thd; - const char* thread_name; ibool server_busy; ib_logf(IB_LOG_LEVEL_INFO, "Starting shutdown..."); @@ -3249,29 +3263,17 @@ logs_empty_and_mark_files_at_shutdown(void) srv_shutdown_state = SRV_SHUTDOWN_CLEANUP; loop: + if (!srv_read_only_mode) { + os_event_set(srv_error_event); + os_event_set(srv_monitor_event); + os_event_set(srv_buf_dump_event); + os_event_set(lock_sys->timeout_event); + os_event_set(dict_stats_event); + } os_thread_sleep(100000); count++; - /* We need the monitor threads to stop before we proceed with - a shutdown. */ - - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - /* Print a message every 60 seconds if we are waiting - for the monitor thread to exit. Master and worker - threads check will be done later. */ - - if (srv_print_verbose_log && count > 600) { - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for %s to exit", thread_name); - count = 0; - } - - goto loop; - } - /* Check that there are no longer transactions, except for PREPARED ones. We need this wait even for the 'very fast' shutdown, because the InnoDB layer may have committed or @@ -3292,55 +3294,61 @@ logs_empty_and_mark_files_at_shutdown(void) goto loop; } - /* Check that the background threads are suspended */ - - active_thd = srv_get_active_thread_type(); - - if (active_thd != SRV_NONE) { + /* We need these threads to stop early in shutdown. */ + const char* thread_name; - if (active_thd == SRV_PURGE) { - srv_purge_wakeup(); - } + if (srv_error_monitor_active) { + thread_name = "srv_error_monitor_thread"; + } else if (srv_monitor_active) { + thread_name = "srv_monitor_thread"; + } else if (srv_dict_stats_thread_active) { + thread_name = "dict_stats_thread"; + } else if (lock_sys->timeout_thread_active) { + thread_name = "lock_wait_timeout_thread"; + } else if (srv_buf_dump_thread_active) { + thread_name = "buf_dump_thread"; + } else { + thread_name = NULL; + } - /* The srv_lock_timeout_thread, srv_error_monitor_thread - and srv_monitor_thread should already exit by now. The - only threads to be suspended are the master threads - and worker threads (purge threads). Print the thread - type if any of such threads not in suspended mode */ + if (thread_name) { + ut_ad(!srv_read_only_mode); +wait_suspend_loop: if (srv_print_verbose_log && count > 600) { - const char* thread_type = ""; - - switch (active_thd) { - case SRV_NONE: - /* This shouldn't happen because we've - already checked for this case before - entering the if(). We handle it here - to avoid a compiler warning. */ - ut_error; - case SRV_WORKER: - thread_type = "worker threads"; - break; - case SRV_MASTER: - thread_type = "master thread"; - break; - case SRV_PURGE: - thread_type = "purge thread"; - break; - } - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for %s to be suspended", - thread_type); + "Waiting for %s to exit", thread_name); count = 0; } - goto loop; } + /* Check that the background threads are suspended */ + + switch (srv_get_active_thread_type()) { + case SRV_NONE: + srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + if (!srv_n_fil_crypt_threads_started) { + break; + } + os_event_set(fil_crypt_threads_event); + thread_name = "fil_crypt_thread"; + goto wait_suspend_loop; + case SRV_PURGE: + srv_purge_wakeup(); + thread_name = "purge thread"; + goto wait_suspend_loop; + case SRV_MASTER: + thread_name = "master thread"; + goto wait_suspend_loop; + case SRV_WORKER: + thread_name = "worker threads"; + goto wait_suspend_loop; + } + /* At this point only page_cleaner should be active. We wait here to let it complete the flushing of the buffer pools before proceeding further. */ - srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + count = 0; while (buf_page_cleaner_is_active) { ++count; @@ -3353,8 +3361,14 @@ logs_empty_and_mark_files_at_shutdown(void) } } + if (log_scrub_thread_active) { + ut_ad(!srv_read_only_mode); + os_event_set(log_scrub_event); + } + mutex_enter(&log_sys->mutex); - server_busy = log_sys->n_pending_checkpoint_writes + server_busy = log_scrub_thread_active + || log_sys->n_pending_checkpoint_writes #ifdef UNIV_LOG_ARCHIVE || log_sys->n_pending_archive_ios #endif /* UNIV_LOG_ARCHIVE */ @@ -3373,6 +3387,8 @@ logs_empty_and_mark_files_at_shutdown(void) goto loop; } + ut_ad(!log_scrub_thread_active); + pending_io = buf_pool_check_no_pending_io(); if (pending_io) { @@ -3408,93 +3424,67 @@ logs_empty_and_mark_files_at_shutdown(void) from the stamps if the previous shutdown was clean. */ log_buffer_flush_to_disk(); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - ib_logf(IB_LOG_LEVEL_WARN, - "Background thread %s woke up " - "during shutdown", thread_name); - goto loop; - } } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; fil_close_all_files(); - - thread_name = srv_any_background_threads_are_active(); - - ut_a(!thread_name); - return; } if (!srv_read_only_mode) { log_make_checkpoint_at(LSN_MAX, TRUE); - } - mutex_enter(&log_sys->mutex); + mutex_enter(&log_sys->mutex); - lsn = log_sys->lsn; + lsn = log_sys->lsn; - if (lsn != log_sys->last_checkpoint_lsn + if (lsn != log_sys->last_checkpoint_lsn #ifdef UNIV_LOG_ARCHIVE - || (srv_log_archive_on - && lsn != log_sys->archived_lsn + LOG_BLOCK_HDR_SIZE) + || (srv_log_archive_on + && lsn != log_sys->archived_lsn + LOG_BLOCK_HDR_SIZE) #endif /* UNIV_LOG_ARCHIVE */ - ) { + ) { - mutex_exit(&log_sys->mutex); + mutex_exit(&log_sys->mutex); - goto loop; - } + goto loop; + } - arch_log_no = 0; + arch_log_no = 0; #ifdef UNIV_LOG_ARCHIVE - UT_LIST_GET_FIRST(log_sys->log_groups)->archived_file_no; + UT_LIST_GET_FIRST(log_sys->log_groups)->archived_file_no; - if (0 == UT_LIST_GET_FIRST(log_sys->log_groups)->archived_offset) { + if (!UT_LIST_GET_FIRST(log_sys->log_groups)->archived_offset) { - arch_log_no--; - } + arch_log_no--; + } - log_archive_close_groups(TRUE); + log_archive_close_groups(TRUE); #endif /* UNIV_LOG_ARCHIVE */ - mutex_exit(&log_sys->mutex); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - if (thread_name != NULL) { - ib_logf(IB_LOG_LEVEL_WARN, - "Background thread %s woke up during shutdown", - thread_name); - - goto loop; - } + mutex_exit(&log_sys->mutex); - if (!srv_read_only_mode) { fil_flush_file_spaces(FIL_TABLESPACE); fil_flush_file_spaces(FIL_LOG); - } - /* The call fil_write_flushed_lsn_to_data_files() will pass the buffer - pool: therefore it is essential that the buffer pool has been - completely flushed to disk! (We do not call fil_write... if the - 'very fast' shutdown is enabled.) */ + /* The call fil_write_flushed_lsn_to_data_files() will + bypass the buffer pool: therefore it is essential that + the buffer pool has been completely flushed to disk! */ - if (!buf_all_freed()) { + if (!buf_all_freed()) { + if (srv_print_verbose_log && count > 600) { + ib_logf(IB_LOG_LEVEL_INFO, + "Waiting for dirty buffer pages" + " to be flushed"); + count = 0; + } - if (srv_print_verbose_log && count > 600) { - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for dirty buffer pages to be flushed"); - count = 0; + goto loop; } - - goto loop; + } else { + lsn = srv_start_lsn; } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; @@ -3739,6 +3729,10 @@ log_shutdown(void) mutex_free(&log_sys->mutex); + if (!srv_read_only_mode && srv_scrub_log) { + os_event_free(log_scrub_event); + } + #ifdef UNIV_LOG_ARCHIVE rw_lock_free(&log_sys->archive_lock); os_event_create(); @@ -3766,11 +3760,6 @@ log_mem_free(void) } } -/** Event to wake up the log scrub thread */ -UNIV_INTERN os_event_t log_scrub_event = NULL; - -UNIV_INTERN ibool srv_log_scrub_thread_active = FALSE; - /*****************************************************************//* If no log record has been written for a while, fill current log block with dummy records. */ @@ -3797,17 +3786,11 @@ sleeps again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg __attribute__((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(log_scrub_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_log_scrub_thread_active = TRUE; - - while(srv_shutdown_state == SRV_SHUTDOWN_NONE) - { + while (srv_shutdown_state < SRV_SHUTDOWN_FLUSH_PHASE) { /* log scrubbing interval in µs. */ ulonglong interval = 1000*1000*512/innodb_scrub_log_speed; @@ -3818,7 +3801,7 @@ DECLARE_THREAD(log_scrub_thread)( os_event_reset(log_scrub_event); } - srv_log_scrub_thread_active = FALSE; + log_scrub_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index 1e323625b133d..bf2a0a7fa3692 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -60,7 +60,6 @@ Created 10/8/1995 Heikki Tuuri #include "btr0sea.h" #include "dict0load.h" #include "dict0boot.h" -#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0start.h" #include "row0mysql.h" #include "row0log.h" @@ -89,15 +88,14 @@ UNIV_INTERN ulong srv_fatal_semaphore_wait_threshold = DEFAULT_SRV_FATAL_SEMAPH in microseconds, in order to reduce the lagging of the purge thread. */ UNIV_INTERN ulint srv_dml_needed_delay = 0; -UNIV_INTERN ibool srv_monitor_active = FALSE; -UNIV_INTERN ibool srv_error_monitor_active = FALSE; +UNIV_INTERN bool srv_monitor_active; +UNIV_INTERN bool srv_error_monitor_active; -UNIV_INTERN ibool srv_buf_dump_thread_active = FALSE; +UNIV_INTERN bool srv_buf_dump_thread_active; -UNIV_INTERN ibool srv_dict_stats_thread_active = FALSE; +UNIV_INTERN bool srv_dict_stats_thread_active; -UNIV_INTERN ibool srv_log_scrub_active = FALSE; -UNIV_INTERN my_bool srv_scrub_log = FALSE; +UNIV_INTERN my_bool srv_scrub_log; UNIV_INTERN const char* srv_main_thread_op_info = ""; @@ -1687,11 +1685,7 @@ A thread which prints the info output by various InnoDB monitors. @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(srv_monitor_thread)( -/*===============================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_monitor_thread)(void*) { ib_int64_t sig_count; double time_elapsed; @@ -1712,9 +1706,7 @@ DECLARE_THREAD(srv_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_monitor_active = TRUE; - UT_NOT_USED(arg); srv_last_monitor_time = ut_time(); last_table_monitor_time = ut_time(); last_tablespace_monitor_time = ut_time(); @@ -1846,7 +1838,7 @@ DECLARE_THREAD(srv_monitor_thread)( goto loop; exit_func: - srv_monitor_active = FALSE; + srv_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -1864,11 +1856,7 @@ we should avoid waiting any mutexes in this function! @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(srv_error_monitor_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_error_monitor_thread)(void*) { /* number of successive fatal timeouts observed */ ulint fatal_cnt = 0; @@ -1894,7 +1882,6 @@ DECLARE_THREAD(srv_error_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_error_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_error_monitor_active = TRUE; loop: /* Try to track a strange bug reported by Harald Fuchs and others, @@ -1980,7 +1967,7 @@ DECLARE_THREAD(srv_error_monitor_thread)( goto loop; } - srv_error_monitor_active = FALSE; + srv_error_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -2039,44 +2026,6 @@ srv_get_active_thread_type(void) return(ret); } -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -UNIV_INTERN -const char* -srv_any_background_threads_are_active(void) -/*=======================================*/ -{ - const char* thread_active = NULL; - - if (srv_read_only_mode) { - return(NULL); - } else if (srv_error_monitor_active) { - thread_active = "srv_error_monitor_thread"; - } else if (lock_sys->timeout_thread_active) { - thread_active = "srv_lock_timeout thread"; - } else if (srv_monitor_active) { - thread_active = "srv_monitor_thread"; - } else if (srv_buf_dump_thread_active) { - thread_active = "buf_dump_thread"; - } else if (srv_dict_stats_thread_active) { - thread_active = "dict_stats_thread"; - } else if (srv_scrub_log && srv_log_scrub_thread_active) { - thread_active = "log_scrub_thread"; - } - - os_event_set(srv_error_event); - os_event_set(srv_monitor_event); - os_event_set(srv_buf_dump_event); - os_event_set(lock_sys->timeout_event); - os_event_set(dict_stats_event); - if (srv_scrub_log) - os_event_set(log_scrub_event); - - return(thread_active); -} - /*******************************************************************//** Tells the InnoDB server that there has been activity in the database and wakes up the master thread if it is suspended (not sleeping). Used diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index acfce27499267..ffd97b6109158 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2356,11 +2356,6 @@ innobase_start_or_create_for_mysql(void) dict_stats_thread_init(); } - if (!srv_read_only_mode && srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_init() instead? */ - log_scrub_event = os_event_create(); - } - trx_sys_file_format_init(); trx_sys_create(); @@ -2725,14 +2720,17 @@ innobase_start_or_create_for_mysql(void) lock_wait_timeout_thread, NULL, thread_ids + 2 + SRV_MAX_N_IO_THREADS); thread_started[2 + SRV_MAX_N_IO_THREADS] = true; + lock_sys->timeout_thread_active = true; /* Create the thread which warns of long semaphore waits */ + srv_error_monitor_active = true; thread_handles[3 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_error_monitor_thread, NULL, thread_ids + 3 + SRV_MAX_N_IO_THREADS); thread_started[3 + SRV_MAX_N_IO_THREADS] = true; /* Create the thread which prints InnoDB monitor info */ + srv_monitor_active = true; thread_handles[4 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_monitor_thread, NULL, thread_ids + 4 + SRV_MAX_N_IO_THREADS); @@ -2965,6 +2963,8 @@ innobase_start_or_create_for_mysql(void) /* Create the buffer pool dump/load thread */ buf_dump_thread_handle= os_thread_create(buf_dump_thread, NULL, NULL); + + srv_buf_dump_thread_active = true; buf_dump_thread_started = true; #ifdef WITH_WSREP } else { @@ -2975,7 +2975,9 @@ innobase_start_or_create_for_mysql(void) #endif /* WITH_WSREP */ /* Create the dict stats gathering thread */ - dict_stats_thread_handle = os_thread_create(dict_stats_thread, NULL, NULL); + dict_stats_thread_handle = os_thread_create( + dict_stats_thread, NULL, NULL); + srv_dict_stats_thread_active = true; dict_stats_thread_started = true; /* Create the thread that will optimize the FTS sub-system. */ @@ -2985,10 +2987,6 @@ innobase_start_or_create_for_mysql(void) fil_system_enter(); fil_crypt_threads_init(); fil_system_exit(); - - /* Create the log scrub thread */ - if (srv_scrub_log) - os_thread_create(log_scrub_thread, NULL, NULL); } /* Init data for datafile scrub threads */ @@ -3057,9 +3055,6 @@ innobase_shutdown_for_mysql(void) fts_optimize_start_shutdown(); fts_optimize_end(); - - /* Shutdown key rotation threads */ - fil_crypt_threads_end(); } /* 1. Flush the buffer pool to disk, write the current lsn to @@ -3168,14 +3163,6 @@ innobase_shutdown_for_mysql(void) if (!srv_read_only_mode) { dict_stats_thread_deinit(); - if (srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_deinit() instead? */ - os_event_free(log_scrub_event); - log_scrub_event = NULL; - } - } - - if (!srv_read_only_mode) { fil_crypt_threads_cleanup(); } diff --git a/storage/xtradb/buf/buf0dump.cc b/storage/xtradb/buf/buf0dump.cc index 114c96cec9809..6abf7375775a9 100644 --- a/storage/xtradb/buf/buf0dump.cc +++ b/storage/xtradb/buf/buf0dump.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -682,15 +683,10 @@ again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(buf_dump_thread)( -/*============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(buf_dump_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_buf_dump_thread_active = TRUE; - buf_dump_status(STATUS_INFO, "Dumping buffer pool(s) not yet started"); buf_load_status(STATUS_INFO, "Loading buffer pool(s) not yet started"); @@ -720,7 +716,7 @@ DECLARE_THREAD(buf_dump_thread)( keep going even if we are in a shutdown state */); } - srv_buf_dump_thread_active = FALSE; + srv_buf_dump_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/xtradb/dict/dict0stats_bg.cc b/storage/xtradb/dict/dict0stats_bg.cc index 5f24bb2b7426f..c2265d6abd641 100644 --- a/storage/xtradb/dict/dict0stats_bg.cc +++ b/storage/xtradb/dict/dict0stats_bg.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 2012, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -526,15 +527,10 @@ statistics. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(dict_stats_thread)( -/*==============================*/ - void* arg MY_ATTRIBUTE((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(dict_stats_thread)(void*) { ut_a(!srv_read_only_mode); - srv_dict_stats_thread_active = TRUE; - while (!SHUTTING_DOWN()) { /* Wake up periodically even if not signaled. This is @@ -557,7 +553,7 @@ DECLARE_THREAD(dict_stats_thread)( os_event_reset(dict_stats_event); } - srv_dict_stats_thread_active = FALSE; + srv_dict_stats_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit instead of return(). */ diff --git a/storage/xtradb/fil/fil0crypt.cc b/storage/xtradb/fil/fil0crypt.cc index 1eaabf6867888..2999bea276505 100644 --- a/storage/xtradb/fil/fil0crypt.cc +++ b/storage/xtradb/fil/fil0crypt.cc @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -56,7 +56,7 @@ UNIV_INTERN ulong srv_encrypt_tables = 0; UNIV_INTERN uint srv_n_fil_crypt_threads = 0; /** No of key rotation threads started */ -static uint srv_n_fil_crypt_threads_started = 0; +UNIV_INTERN uint srv_n_fil_crypt_threads_started = 0; /** At this age or older a space/page will be rotated */ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; @@ -65,7 +65,7 @@ UNIV_INTERN uint srv_fil_crypt_rotate_key_age = 1; static os_event_t fil_crypt_event; /** Event to signal TO the key rotation threads. */ -static os_event_t fil_crypt_threads_event; +UNIV_INTERN os_event_t fil_crypt_threads_event; /** Event for waking up threads throttle */ static os_event_t fil_crypt_throttle_sleep_event; @@ -1303,10 +1303,20 @@ struct rotate_thread_t { btr_scrub_t scrub_data; /* thread local data used by btr_scrub-functions * when iterating pages of tablespace */ - /* check if this thread should shutdown */ + /** @return whether this thread should terminate */ bool should_shutdown() const { - return ! (srv_shutdown_state == SRV_SHUTDOWN_NONE && - thread_no < srv_n_fil_crypt_threads); + switch (srv_shutdown_state) { + case SRV_SHUTDOWN_NONE: + case SRV_SHUTDOWN_CLEANUP: + return thread_no >= srv_n_fil_crypt_threads; + case SRV_SHUTDOWN_FLUSH_PHASE: + return true; + case SRV_SHUTDOWN_LAST_PHASE: + case SRV_SHUTDOWN_EXIT_THREADS: + break; + } + ut_ad(0); + return true; } }; @@ -2458,18 +2468,6 @@ fil_crypt_threads_init() } } -/********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end() -/*===================*/ -{ - if (fil_crypt_threads_inited) { - fil_crypt_set_thread_cnt(0); - } -} - /********************************************************************* Clean up key rotation threads resources */ UNIV_INTERN @@ -2480,6 +2478,7 @@ fil_crypt_threads_cleanup() if (!fil_crypt_threads_inited) { return; } + ut_a(!srv_n_fil_crypt_threads_started); os_event_free(fil_crypt_event); os_event_free(fil_crypt_threads_event); mutex_free(&fil_crypt_threads_mutex); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 6db23a3af2a79..a361ab17a7caf 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -68,7 +68,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "srv0srv.h" #include "trx0roll.h" #include "trx0trx.h" - #include "trx0sys.h" #include "rem0types.h" #include "row0ins.h" @@ -252,7 +251,6 @@ static char* internal_innobase_data_file_path = NULL; static char* innodb_version_str = (char*) INNODB_VERSION_STR; -extern uint srv_n_fil_crypt_threads; extern uint srv_fil_crypt_rotate_key_age; extern uint srv_n_fil_crypt_iops; diff --git a/storage/xtradb/include/fil0crypt.h b/storage/xtradb/include/fil0crypt.h index 9a35f6591e751..42cdafde4d0c4 100644 --- a/storage/xtradb/include/fil0crypt.h +++ b/storage/xtradb/include/fil0crypt.h @@ -1,6 +1,6 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (c) 2015, 2016, MariaDB Corporation. +Copyright (c) 2015, 2017, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -26,6 +26,8 @@ Created 04/01/2015 Jan Lindström #ifndef fil0crypt_h #define fil0crypt_h +#include "os0sync.h" + /** * Magic pattern in start of crypt data on page 0 */ @@ -45,6 +47,8 @@ typedef enum { FIL_SPACE_ENCRYPTION_OFF = 2 /* Tablespace is not encrypted */ } fil_encryption_t; +extern os_event_t fil_crypt_threads_event; + /** * CRYPT_SCHEME_UNENCRYPTED * @@ -391,12 +395,6 @@ fil_crypt_set_thread_cnt( /*=====================*/ uint new_cnt); /*!< in: requested #threads */ -/********************************************************************* -End threads for key rotation */ -UNIV_INTERN -void -fil_crypt_threads_end(); - /********************************************************************* Cleanup resources for threads for key rotation */ UNIV_INTERN diff --git a/storage/xtradb/include/log0log.h b/storage/xtradb/include/log0log.h index 470274761fc4f..ec3ecbd1d66df 100644 --- a/storage/xtradb/include/log0log.h +++ b/storage/xtradb/include/log0log.h @@ -2,6 +2,7 @@ Copyright (c) 1995, 2013, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. +Copyright (c) 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1070,22 +1071,9 @@ struct log_t{ /* @} */ #endif /* UNIV_LOG_ARCHIVE */ -extern os_event_t log_scrub_event; /* log scrubbing speed, in bytes/sec */ extern ulonglong innodb_scrub_log_speed; -/*****************************************************************//** -This is the main thread for log scrub. It waits for an event and -when waked up fills current log block with dummy records and -sleeps again. -@return this function does not return, it calls os_thread_exit() */ -extern "C" UNIV_INTERN -os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg); /*!< in: a dummy parameter - required by os_thread_create */ - #ifndef UNIV_NONINL #include "log0log.ic" #endif diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index e2d141b414092..f3b6098a875a1 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, 2009, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation +Copyright (c) 2013, 2017, MariaDB Corporation Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -349,7 +349,6 @@ extern char** srv_data_file_names; extern ulint* srv_data_file_sizes; extern ulint* srv_data_file_is_raw_partition; - /** Whether the redo log tracking is currently enabled. Note that it is possible for the log tracker thread to be running and the tracking to be disabled */ @@ -359,6 +358,9 @@ extern ulonglong srv_max_bitmap_file_size; extern ulonglong srv_max_changed_pages; +extern uint srv_n_fil_crypt_threads; +extern uint srv_n_fil_crypt_threads_started; + extern ibool srv_auto_extend_last_data_file; extern ulint srv_last_file_size_max; extern char* srv_log_group_home_dir; @@ -577,19 +579,17 @@ extern ibool srv_print_verbose_log; "tables instead, see " REFMAN "innodb-i_s-tables.html" extern ibool srv_print_innodb_table_monitor; -extern ibool srv_monitor_active; -extern ibool srv_error_monitor_active; +extern bool srv_monitor_active; +extern bool srv_error_monitor_active; /* TRUE during the lifetime of the buffer pool dump/load thread */ -extern ibool srv_buf_dump_thread_active; +extern bool srv_buf_dump_thread_active; /* TRUE during the lifetime of the stats thread */ -extern ibool srv_dict_stats_thread_active; +extern bool srv_dict_stats_thread_active; /* TRUE if enable log scrubbing */ extern my_bool srv_scrub_log; -/* TRUE during the lifetime of the log scrub thread */ -extern ibool srv_log_scrub_thread_active; extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; @@ -1085,15 +1085,6 @@ srv_release_threads( enum srv_thread_type type, /*!< in: thread type */ ulint n); /*!< in: number of threads to release */ -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -UNIV_INTERN -const char* -srv_any_background_threads_are_active(void); -/*=======================================*/ - /**********************************************************************//** Wakeup the purge threads. */ UNIV_INTERN diff --git a/storage/xtradb/lock/lock0wait.cc b/storage/xtradb/lock/lock0wait.cc index c7bd223c491d0..8f9ea7e10aa8d 100644 --- a/storage/xtradb/lock/lock0wait.cc +++ b/storage/xtradb/lock/lock0wait.cc @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -510,11 +511,7 @@ A thread which wakes up threads whose lock wait may have lasted too long. @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(lock_wait_timeout_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /* in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(lock_wait_timeout_thread)(void*) { ib_int64_t sig_count = 0; os_event_t event = lock_sys->timeout_event; @@ -525,8 +522,6 @@ DECLARE_THREAD(lock_wait_timeout_thread)( pfs_register_thread(srv_lock_timeout_thread_key); #endif /* UNIV_PFS_THREAD */ - lock_sys->timeout_thread_active = true; - do { srv_slot_t* slot; diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index f3a3486017cc4..9245ae160e6d6 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -2,7 +2,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2009, Google Inc. -Copyright (C) 2014, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2014, 2017, MariaDB Corporation. All Rights Reserved. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -52,9 +52,11 @@ Created 12/9/1995 Heikki Tuuri #include "buf0buf.h" #include "buf0flu.h" #include "srv0srv.h" +#include "lock0lock.h" #include "log0recv.h" #include "fil0fil.h" #include "dict0boot.h" +#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0srv.h" #include "srv0start.h" #include "trx0sys.h" @@ -95,6 +97,10 @@ UNIV_INTERN log_t* log_sys = NULL; UNIV_INTERN log_checksum_func_t log_checksum_algorithm_ptr = log_block_calc_checksum_innodb; +extern "C" UNIV_INTERN +os_thread_ret_t +DECLARE_THREAD(log_scrub_thread)(void*); + /* Next log block number to do dummy record filling if no log records written for a while */ static ulint next_lbn_to_pad = 0; @@ -167,6 +173,11 @@ the previous */ #define LOG_ARCHIVE_READ 1 #define LOG_ARCHIVE_WRITE 2 +/** Event to wake up the log scrub thread */ +static os_event_t log_scrub_event; + +static bool log_scrub_thread_active; + /******************************************************//** Completes a checkpoint write i/o to a log file. */ static @@ -1049,6 +1060,12 @@ log_init(void) mutex_exit(&(log_sys->mutex)); + log_scrub_thread_active = !srv_read_only_mode && srv_scrub_log; + if (log_scrub_thread_active) { + log_scrub_event = os_event_create(); + os_thread_create(log_scrub_thread, NULL, NULL); + } + #ifdef UNIV_LOG_DEBUG recv_sys_create(); recv_sys_init(buf_pool_get_curr_size()); @@ -3545,8 +3562,6 @@ logs_empty_and_mark_files_at_shutdown(void) ulint count = 0; ulint total_trx; ulint pending_io; - enum srv_thread_type active_thd; - const char* thread_name; ibool server_busy; ib_logf(IB_LOG_LEVEL_INFO, "Starting shutdown..."); @@ -3566,29 +3581,17 @@ logs_empty_and_mark_files_at_shutdown(void) srv_shutdown_state = SRV_SHUTDOWN_CLEANUP; loop: + if (!srv_read_only_mode) { + os_event_set(srv_error_event); + os_event_set(srv_monitor_event); + os_event_set(srv_buf_dump_event); + os_event_set(lock_sys->timeout_event); + os_event_set(dict_stats_event); + } os_thread_sleep(100000); count++; - /* We need the monitor threads to stop before we proceed with - a shutdown. */ - - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - /* Print a message every 60 seconds if we are waiting - for the monitor thread to exit. Master and worker - threads check will be done later. */ - - if (srv_print_verbose_log && count > 600) { - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for %s to exit", thread_name); - count = 0; - } - - goto loop; - } - /* Check that there are no longer transactions, except for PREPARED ones. We need this wait even for the 'very fast' shutdown, because the InnoDB layer may have committed or @@ -3609,55 +3612,61 @@ logs_empty_and_mark_files_at_shutdown(void) goto loop; } - /* Check that the background threads are suspended */ - - active_thd = srv_get_active_thread_type(); - - if (active_thd != SRV_NONE) { + /* We need these threads to stop early in shutdown. */ + const char* thread_name; - if (active_thd == SRV_PURGE) { - srv_purge_wakeup(); - } + if (srv_error_monitor_active) { + thread_name = "srv_error_monitor_thread"; + } else if (srv_monitor_active) { + thread_name = "srv_monitor_thread"; + } else if (srv_dict_stats_thread_active) { + thread_name = "dict_stats_thread"; + } else if (lock_sys->timeout_thread_active) { + thread_name = "lock_wait_timeout_thread"; + } else if (srv_buf_dump_thread_active) { + thread_name = "buf_dump_thread"; + } else { + thread_name = NULL; + } - /* The srv_lock_timeout_thread, srv_error_monitor_thread - and srv_monitor_thread should already exit by now. The - only threads to be suspended are the master threads - and worker threads (purge threads). Print the thread - type if any of such threads not in suspended mode */ + if (thread_name) { + ut_ad(!srv_read_only_mode); +wait_suspend_loop: if (srv_print_verbose_log && count > 600) { - const char* thread_type = ""; - - switch (active_thd) { - case SRV_NONE: - /* This shouldn't happen because we've - already checked for this case before - entering the if(). We handle it here - to avoid a compiler warning. */ - ut_error; - case SRV_WORKER: - thread_type = "worker threads"; - break; - case SRV_MASTER: - thread_type = "master thread"; - break; - case SRV_PURGE: - thread_type = "purge thread"; - break; - } - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for %s to be suspended", - thread_type); + "Waiting for %s to exit", thread_name); count = 0; } - goto loop; } + /* Check that the background threads are suspended */ + + switch (srv_get_active_thread_type()) { + case SRV_NONE: + srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + if (!srv_n_fil_crypt_threads_started) { + break; + } + os_event_set(fil_crypt_threads_event); + thread_name = "fil_crypt_thread"; + goto wait_suspend_loop; + case SRV_PURGE: + srv_purge_wakeup(); + thread_name = "purge thread"; + goto wait_suspend_loop; + case SRV_MASTER: + thread_name = "master thread"; + goto wait_suspend_loop; + case SRV_WORKER: + thread_name = "worker threads"; + goto wait_suspend_loop; + } + /* At this point only page_cleaner should be active. We wait here to let it complete the flushing of the buffer pools before proceeding further. */ - srv_shutdown_state = SRV_SHUTDOWN_FLUSH_PHASE; + count = 0; while (buf_page_cleaner_is_active || buf_lru_manager_is_active) { if (srv_print_verbose_log && count == 0) { @@ -3672,8 +3681,14 @@ logs_empty_and_mark_files_at_shutdown(void) } } + if (log_scrub_thread_active) { + ut_ad(!srv_read_only_mode); + os_event_set(log_scrub_event); + } + mutex_enter(&log_sys->mutex); - server_busy = log_sys->n_pending_checkpoint_writes + server_busy = log_scrub_thread_active + || log_sys->n_pending_checkpoint_writes #ifdef UNIV_LOG_ARCHIVE || log_sys->n_pending_archive_ios #endif /* UNIV_LOG_ARCHIVE */ @@ -3692,6 +3707,8 @@ logs_empty_and_mark_files_at_shutdown(void) goto loop; } + ut_ad(!log_scrub_thread_active); + pending_io = buf_pool_check_no_pending_io(); if (pending_io) { @@ -3727,16 +3744,6 @@ logs_empty_and_mark_files_at_shutdown(void) from the stamps if the previous shutdown was clean. */ log_buffer_flush_to_disk(); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - - if (thread_name != NULL) { - ib_logf(IB_LOG_LEVEL_WARN, - "Background thread %s woke up " - "during shutdown", thread_name); - goto loop; - } } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; @@ -3749,74 +3756,57 @@ logs_empty_and_mark_files_at_shutdown(void) } fil_close_all_files(); - - thread_name = srv_any_background_threads_are_active(); - - ut_a(!thread_name); - return; } if (!srv_read_only_mode) { log_make_checkpoint_at(LSN_MAX, TRUE); - } - mutex_enter(&log_sys->mutex); + mutex_enter(&log_sys->mutex); - tracked_lsn = log_get_tracked_lsn(); + tracked_lsn = log_get_tracked_lsn(); - lsn = log_sys->lsn; + lsn = log_sys->lsn; - if (lsn != log_sys->last_checkpoint_lsn - || (srv_track_changed_pages - && (tracked_lsn != log_sys->last_checkpoint_lsn)) + if (lsn != log_sys->last_checkpoint_lsn + || (srv_track_changed_pages + && (tracked_lsn != log_sys->last_checkpoint_lsn)) #ifdef UNIV_LOG_ARCHIVE - || (srv_log_archive_on - && lsn != log_sys->archived_lsn + LOG_BLOCK_HDR_SIZE) + || (srv_log_archive_on + && lsn != log_sys->archived_lsn + LOG_BLOCK_HDR_SIZE) #endif /* UNIV_LOG_ARCHIVE */ - ) { + ) { - mutex_exit(&log_sys->mutex); + mutex_exit(&log_sys->mutex); - goto loop; - } + goto loop; + } #ifdef UNIV_LOG_ARCHIVE - - log_archive_close_groups(TRUE); + log_archive_close_groups(TRUE); #endif /* UNIV_LOG_ARCHIVE */ - mutex_exit(&log_sys->mutex); - - /* Check that the background threads stay suspended */ - thread_name = srv_any_background_threads_are_active(); - if (thread_name != NULL) { - ib_logf(IB_LOG_LEVEL_WARN, - "Background thread %s woke up during shutdown", - thread_name); - - goto loop; - } + mutex_exit(&log_sys->mutex); - if (!srv_read_only_mode) { fil_flush_file_spaces(FIL_TABLESPACE); fil_flush_file_spaces(FIL_LOG); - } - /* The call fil_write_flushed_lsn_to_data_files() will pass the buffer - pool: therefore it is essential that the buffer pool has been - completely flushed to disk! (We do not call fil_write... if the - 'very fast' shutdown is enabled.) */ + /* The call fil_write_flushed_lsn_to_data_files() will + bypass the buffer pool: therefore it is essential that + the buffer pool has been completely flushed to disk! */ - if (!buf_all_freed()) { + if (!buf_all_freed()) { + if (srv_print_verbose_log && count > 600) { + ib_logf(IB_LOG_LEVEL_INFO, + "Waiting for dirty buffer pages" + " to be flushed"); + count = 0; + } - if (srv_print_verbose_log && count > 600) { - ib_logf(IB_LOG_LEVEL_INFO, - "Waiting for dirty buffer pages to be flushed"); - count = 0; + goto loop; } - - goto loop; + } else { + lsn = srv_start_lsn; } srv_shutdown_state = SRV_SHUTDOWN_LAST_PHASE; @@ -4094,6 +4084,10 @@ log_shutdown(void) mutex_free(&log_sys->mutex); mutex_free(&log_sys->log_flush_order_mutex); + if (!srv_read_only_mode && srv_scrub_log) { + os_event_free(log_scrub_event); + } + #ifdef UNIV_LOG_ARCHIVE rw_lock_free(&log_sys->archive_lock); os_event_free(log_sys->archiving_on); @@ -4121,11 +4115,6 @@ log_mem_free(void) } } -/** Event to wake up the log scrub thread */ -UNIV_INTERN os_event_t log_scrub_event = NULL; - -UNIV_INTERN ibool srv_log_scrub_thread_active = FALSE; - /*****************************************************************//* If no log record has been written for a while, fill current log block with dummy records. */ @@ -4152,17 +4141,11 @@ sleeps again. @return this function does not return, it calls os_thread_exit() */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(log_scrub_thread)( -/*===============================*/ - void* arg __attribute__((unused))) /*!< in: a dummy parameter - required by os_thread_create */ +DECLARE_THREAD(log_scrub_thread)(void*) { ut_ad(!srv_read_only_mode); - srv_log_scrub_thread_active = TRUE; - - while(srv_shutdown_state == SRV_SHUTDOWN_NONE) - { + while (srv_shutdown_state < SRV_SHUTDOWN_FLUSH_PHASE) { /* log scrubbing interval in µs. */ ulonglong interval = 1000*1000*512/innodb_scrub_log_speed; @@ -4173,7 +4156,7 @@ DECLARE_THREAD(log_scrub_thread)( os_event_reset(log_scrub_event); } - srv_log_scrub_thread_active = FALSE; + log_scrub_thread_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index eff680a4ef0d6..bc222b864ef89 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -3,7 +3,7 @@ Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2016, MariaDB Corporation. +Copyright (c) 2013, 2017, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -61,7 +61,6 @@ Created 10/8/1995 Heikki Tuuri #include "btr0sea.h" #include "dict0load.h" #include "dict0boot.h" -#include "dict0stats_bg.h" /* dict_stats_event */ #include "srv0start.h" #include "row0mysql.h" #include "row0log.h" @@ -107,15 +106,14 @@ UNIV_INTERN long long srv_kill_idle_transaction = 0; in microseconds, in order to reduce the lagging of the purge thread. */ UNIV_INTERN ulint srv_dml_needed_delay = 0; -UNIV_INTERN ibool srv_monitor_active = FALSE; -UNIV_INTERN ibool srv_error_monitor_active = FALSE; +UNIV_INTERN bool srv_monitor_active; +UNIV_INTERN bool srv_error_monitor_active; -UNIV_INTERN ibool srv_buf_dump_thread_active = FALSE; +UNIV_INTERN bool srv_buf_dump_thread_active; -UNIV_INTERN ibool srv_dict_stats_thread_active = FALSE; +UNIV_INTERN bool srv_dict_stats_thread_active; -UNIV_INTERN ibool srv_log_scrub_active = FALSE; -UNIV_INTERN my_bool srv_scrub_log = FALSE; +UNIV_INTERN my_bool srv_scrub_log; UNIV_INTERN const char* srv_main_thread_op_info = ""; @@ -2100,11 +2098,7 @@ A thread which prints the info output by various InnoDB monitors. @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(srv_monitor_thread)( -/*===============================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_monitor_thread)(void*) { ib_int64_t sig_count; double time_elapsed; @@ -2125,9 +2119,7 @@ DECLARE_THREAD(srv_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_monitor_active = TRUE; - UT_NOT_USED(arg); srv_last_monitor_time = ut_time(); last_table_monitor_time = ut_time(); last_tablespace_monitor_time = ut_time(); @@ -2259,7 +2251,7 @@ DECLARE_THREAD(srv_monitor_thread)( goto loop; exit_func: - srv_monitor_active = FALSE; + srv_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -2277,11 +2269,7 @@ we should avoid waiting any mutexes in this function! @return a dummy parameter */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(srv_error_monitor_thread)( -/*=====================================*/ - void* arg MY_ATTRIBUTE((unused))) - /*!< in: a dummy parameter required by - os_thread_create */ +DECLARE_THREAD(srv_error_monitor_thread)(void*) { /* number of successive fatal timeouts observed */ ulint fatal_cnt = 0; @@ -2307,7 +2295,6 @@ DECLARE_THREAD(srv_error_monitor_thread)( #ifdef UNIV_PFS_THREAD pfs_register_thread(srv_error_monitor_thread_key); #endif /* UNIV_PFS_THREAD */ - srv_error_monitor_active = TRUE; loop: /* Try to track a strange bug reported by Harald Fuchs and others, @@ -2423,7 +2410,7 @@ DECLARE_THREAD(srv_error_monitor_thread)( goto loop; } - srv_error_monitor_active = FALSE; + srv_error_monitor_active = false; /* We count the number of threads in os_thread_exit(). A created thread should always use that to exit and not use return() to exit. */ @@ -2487,44 +2474,6 @@ srv_get_active_thread_type(void) return(ret); } -/**********************************************************************//** -Check whether any background thread are active. If so print which thread -is active. Send the threads wakeup signal. -@return name of thread that is active or NULL */ -UNIV_INTERN -const char* -srv_any_background_threads_are_active(void) -/*=======================================*/ -{ - const char* thread_active = NULL; - - if (srv_read_only_mode) { - return(NULL); - } else if (srv_error_monitor_active) { - thread_active = "srv_error_monitor_thread"; - } else if (lock_sys->timeout_thread_active) { - thread_active = "srv_lock_timeout thread"; - } else if (srv_monitor_active) { - thread_active = "srv_monitor_thread"; - } else if (srv_buf_dump_thread_active) { - thread_active = "buf_dump_thread"; - } else if (srv_dict_stats_thread_active) { - thread_active = "dict_stats_thread"; - } else if (srv_scrub_log && srv_log_scrub_thread_active) { - thread_active = "log_scrub_thread"; - } - - os_event_set(srv_error_event); - os_event_set(srv_monitor_event); - os_event_set(srv_buf_dump_event); - os_event_set(lock_sys->timeout_event); - os_event_set(dict_stats_event); - if (srv_scrub_log) - os_event_set(log_scrub_event); - - return(thread_active); -} - /******************************************************************//** A thread which follows the redo log and outputs the changed page bitmap. @return a dummy value */ diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 8f8cf26504108..31421c7a605ec 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -2435,11 +2435,6 @@ innobase_start_or_create_for_mysql(void) dict_stats_thread_init(); } - if (!srv_read_only_mode && srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_init() instead? */ - log_scrub_event = os_event_create(); - } - trx_sys_file_format_init(); trx_sys_create(); @@ -2782,14 +2777,17 @@ innobase_start_or_create_for_mysql(void) lock_wait_timeout_thread, NULL, thread_ids + 2 + SRV_MAX_N_IO_THREADS); thread_started[2 + SRV_MAX_N_IO_THREADS] = true; + lock_sys->timeout_thread_active = true; /* Create the thread which warns of long semaphore waits */ + srv_error_monitor_active = true; thread_handles[3 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_error_monitor_thread, NULL, thread_ids + 3 + SRV_MAX_N_IO_THREADS); thread_started[3 + SRV_MAX_N_IO_THREADS] = true; /* Create the thread which prints InnoDB monitor info */ + srv_monitor_active = true; thread_handles[4 + SRV_MAX_N_IO_THREADS] = os_thread_create( srv_monitor_thread, NULL, thread_ids + 4 + SRV_MAX_N_IO_THREADS); @@ -3033,6 +3031,8 @@ innobase_start_or_create_for_mysql(void) /* Create the buffer pool dump/load thread */ buf_dump_thread_handle= os_thread_create(buf_dump_thread, NULL, NULL); + + srv_buf_dump_thread_active = true; buf_dump_thread_started = true; #ifdef WITH_WSREP } else { @@ -3043,7 +3043,9 @@ innobase_start_or_create_for_mysql(void) #endif /* WITH_WSREP */ /* Create the dict stats gathering thread */ - dict_stats_thread_handle = os_thread_create(dict_stats_thread, NULL, NULL); + dict_stats_thread_handle = os_thread_create( + dict_stats_thread, NULL, NULL); + srv_dict_stats_thread_active = true; dict_stats_thread_started = true; /* Create the thread that will optimize the FTS sub-system. */ @@ -3053,10 +3055,6 @@ innobase_start_or_create_for_mysql(void) fil_system_enter(); fil_crypt_threads_init(); fil_system_exit(); - - /* Create the log scrub thread */ - if (srv_scrub_log) - os_thread_create(log_scrub_thread, NULL, NULL); } /* Init data for datafile scrub threads */ @@ -3125,9 +3123,6 @@ innobase_shutdown_for_mysql(void) fts_optimize_start_shutdown(); fts_optimize_end(); - - /* Shutdown key rotation threads */ - fil_crypt_threads_end(); } /* 1. Flush the buffer pool to disk, write the current lsn to @@ -3230,14 +3225,6 @@ innobase_shutdown_for_mysql(void) if (!srv_read_only_mode) { dict_stats_thread_deinit(); - if (srv_scrub_log) { - /* TODO(minliz): have/use log_scrub_thread_deinit() instead? */ - os_event_free(log_scrub_event); - log_scrub_event = NULL; - } - } - - if (!srv_read_only_mode) { fil_crypt_threads_cleanup(); } From ffb38c9771bc7248482dc6a3d2f26ca10fe09a9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 4 Jan 2017 20:49:13 +0200 Subject: [PATCH 25/74] MDEV-8139 Fix scrubbing tests encryption.innodb_scrub: Clean up. Make it also cover ROW_FORMAT=COMPRESSED, removing the need for encryption.innodb_scrub_compressed. Add a FIXME comment saying that we should create a secondary index, to demonstrate that also undo log pages get scrubbed. Currently that is not working! Also clean up encryption.innodb_scrub_background, but keep it disabled, because the background scrubbing does not work reliably. Fix both tests so that if something is not scrubbed, the test will be aborted, so that the data files will be preserved. Allow the tests to run on Windows as well. --- mysql-test/include/search_pattern_in_file.inc | 6 + mysql-test/suite/encryption/disabled.def | 6 +- .../suite/encryption/r/innodb_scrub.result | 238 ++++++------------ .../r/innodb_scrub_background.result | 169 +++++++++---- .../r/innodb_scrub_compressed.result | 71 ------ .../suite/encryption/t/innodb_scrub.test | 181 +++++++------ .../encryption/t/innodb_scrub_background.test | 160 ++++++------ .../encryption/t/innodb_scrub_compressed.opt | 9 - .../encryption/t/innodb_scrub_compressed.test | 161 ------------ 9 files changed, 376 insertions(+), 625 deletions(-) delete mode 100644 mysql-test/suite/encryption/r/innodb_scrub_compressed.result delete mode 100644 mysql-test/suite/encryption/t/innodb_scrub_compressed.opt delete mode 100644 mysql-test/suite/encryption/t/innodb_scrub_compressed.test diff --git a/mysql-test/include/search_pattern_in_file.inc b/mysql-test/include/search_pattern_in_file.inc index 3280dbfd574db..f77a7c6091607 100644 --- a/mysql-test/include/search_pattern_in_file.inc +++ b/mysql-test/include/search_pattern_in_file.inc @@ -82,8 +82,14 @@ perl; } $ENV{'SEARCH_FILE'} =~ s{^.*?([^/\\]+)$}{$1}; if ($content =~ m{$search_pattern}) { + die "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + if $ENV{SEARCH_ABORT} eq 'FOUND'; print "FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + unless defined $ENV{SEARCH_ABORT}; } else { + die "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + if $ENV{SEARCH_ABORT} eq 'NOT FOUND'; print "NOT FOUND /$search_pattern/ in $ENV{'SEARCH_FILE'}\n" + unless defined $ENV{SEARCH_ABORT}; } EOF diff --git a/mysql-test/suite/encryption/disabled.def b/mysql-test/suite/encryption/disabled.def index 8c0d47983fdfc..348e4c979eac8 100644 --- a/mysql-test/suite/encryption/disabled.def +++ b/mysql-test/suite/encryption/disabled.def @@ -10,8 +10,4 @@ # ############################################################################## -innodb_scrub : MDEV-8139 -innodb_scrub_compressed : MDEV-8139 -innodb_scrub_background : MDEV-8139 -innodb_encryption-page-compression : Fails with lost connection at line 156 - +innodb_scrub_background : MDEV-8139 background scrubbing does not work reliably diff --git a/mysql-test/suite/encryption/r/innodb_scrub.result b/mysql-test/suite/encryption/r/innodb_scrub.result index 95f0aed3226e5..6e8febc762d91 100644 --- a/mysql-test/suite/encryption/r/innodb_scrub.result +++ b/mysql-test/suite/encryption/r/innodb_scrub.result @@ -1,207 +1,121 @@ create table snapshot_status engine = myisam select * from information_schema.global_status where variable_name like 'innodb_scrub%'; -# -# Test delete of records -# -create table t1 ( +# MDEV-8139 Fix scrubbing tests +# FIXME: Add index(b) to each table; ensure that undo logs are scrubbed. +create table delete_3 ( +a int auto_increment primary key, +b varchar(256), +c text) engine = innodb row_format=compressed; +delete from delete_3; +create table delete_rollback_delete_3 ( +a int auto_increment primary key, +b varchar(256), +c text) engine = innodb row_format=compressed; +begin; +delete from delete_rollback_delete_3; +rollback; +delete from delete_rollback_delete_3; +create table insert_rollback_3 ( +a int auto_increment primary key, +b varchar(256), +c text) engine = innodb row_format=compressed; +begin; +rollback; +create table delete_2 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=compact; -# Populate table with rows -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# compact: delete from: grep -c bicycle t1.ibd -0 -# compact: delete from: grep -c bicycle ibdata1 -0 -# compact: delete from: grep -c repairman t1.ibd -0 -# compact: delete from: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete+rollback+delete -# -create table t1 ( +delete from delete_2; +create table delete_rollback_delete_2 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=compact; -# Populate table with rows begin; -delete from t1; +delete from delete_rollback_delete_2; rollback; -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# compact: delete rollback: grep -c bicycle t1.ibd -0 -# compact: delete rollback: grep -c bicycle ibdata1 -0 -# compact: delete rollback: grep -c repairman t1.ibd -0 -# compact: delete rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test insert+rollback -# -create table t1 ( +delete from delete_rollback_delete_2; +create table insert_rollback_2 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=compact; -# Populate table with rows begin; rollback; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# compact: insert rollback: grep -c bicycle t1.ibd -0 -# compact: insert rollback: grep -c bicycle ibdata1 -0 -# compact: insert rollback: grep -c repairman t1.ibd -0 -# compact: insert rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete of records -# -create table t1 ( +create table delete_1 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=redundant; -# Populate table with rows -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# redundant: delete from: grep -c bicycle t1.ibd -0 -# redundant: delete from: grep -c bicycle ibdata1 -0 -# redundant: delete from: grep -c repairman t1.ibd -0 -# redundant: delete from: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete+rollback+delete -# -create table t1 ( +delete from delete_1; +create table delete_rollback_delete_1 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=redundant; -# Populate table with rows begin; -delete from t1; +delete from delete_rollback_delete_1; rollback; -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# redundant: delete rollback: grep -c bicycle t1.ibd -0 -# redundant: delete rollback: grep -c bicycle ibdata1 -0 -# redundant: delete rollback: grep -c repairman t1.ibd -0 -# redundant: delete rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test insert+rollback -# -create table t1 ( +delete from delete_rollback_delete_1; +create table insert_rollback_1 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=redundant; -# Populate table with rows begin; rollback; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# redundant: insert rollback: grep -c bicycle t1.ibd -0 -# redundant: insert rollback: grep -c bicycle ibdata1 -0 -# redundant: insert rollback: grep -c repairman t1.ibd -0 -# redundant: insert rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete of records -# -create table t1 ( +create table delete_0 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=dynamic; -# Populate table with rows -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# dynamic: delete from: grep -c bicycle t1.ibd -0 -# dynamic: delete from: grep -c bicycle ibdata1 -0 -# dynamic: delete from: grep -c repairman t1.ibd -0 -# dynamic: delete from: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test delete+rollback+delete -# -create table t1 ( +delete from delete_0; +create table delete_rollback_delete_0 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=dynamic; -# Populate table with rows begin; -delete from t1; +delete from delete_rollback_delete_0; rollback; -delete from t1; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# dynamic: delete rollback: grep -c bicycle t1.ibd -0 -# dynamic: delete rollback: grep -c bicycle ibdata1 -0 -# dynamic: delete rollback: grep -c repairman t1.ibd -0 -# dynamic: delete rollback: grep -c repairman ibdata1 -0 -drop table t1; -# -# Test insert+rollback -# -create table t1 ( +delete from delete_rollback_delete_0; +create table insert_rollback_0 ( a int auto_increment primary key, b varchar(256), c text) engine = innodb row_format=dynamic; -# Populate table with rows begin; rollback; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# dynamic: insert rollback: grep -c bicycle t1.ibd -0 -# dynamic: insert rollback: grep -c bicycle ibdata1 -0 -# dynamic: insert rollback: grep -c repairman t1.ibd -0 -# dynamic: insert rollback: grep -c repairman ibdata1 -0 -drop table t1; +SET GLOBAL innodb_fast_shutdown=0; +# delete_3.ibd +# delete_rollback_delete_3.ibd +# insert_rollback_3.ibd +# delete_2.ibd +# delete_rollback_delete_2.ibd +# insert_rollback_2.ibd +# delete_1.ibd +# delete_rollback_delete_1.ibd +# insert_rollback_1.ibd +# delete_0.ibd +# delete_rollback_delete_0.ibd +# insert_rollback_0.ibd +check table delete_3, delete_rollback_delete_3, insert_rollback_3; +Table Op Msg_type Msg_text +test.delete_3 check status OK +test.delete_rollback_delete_3 check status OK +test.insert_rollback_3 check status OK +drop table delete_3, delete_rollback_delete_3, insert_rollback_3; +check table delete_2, delete_rollback_delete_2, insert_rollback_2; +Table Op Msg_type Msg_text +test.delete_2 check status OK +test.delete_rollback_delete_2 check status OK +test.insert_rollback_2 check status OK +drop table delete_2, delete_rollback_delete_2, insert_rollback_2; +check table delete_1, delete_rollback_delete_1, insert_rollback_1; +Table Op Msg_type Msg_text +test.delete_1 check status OK +test.delete_rollback_delete_1 check status OK +test.insert_rollback_1 check status OK +drop table delete_1, delete_rollback_delete_1, insert_rollback_1; +check table delete_0, delete_rollback_delete_0, insert_rollback_0; +Table Op Msg_type Msg_text +test.delete_0 check status OK +test.delete_rollback_delete_0 check status OK +test.insert_rollback_0 check status OK +drop table delete_0, delete_rollback_delete_0, insert_rollback_0; show variables like 'innodb_%scrub_data%'; Variable_name Value innodb_background_scrub_data_check_interval 3600 diff --git a/mysql-test/suite/encryption/r/innodb_scrub_background.result b/mysql-test/suite/encryption/r/innodb_scrub_background.result index 1ff6c0bda982c..6a2f263d0a107 100644 --- a/mysql-test/suite/encryption/r/innodb_scrub_background.result +++ b/mysql-test/suite/encryption/r/innodb_scrub_background.result @@ -11,66 +11,146 @@ innodb_background_scrub_data_uncompressed ON innodb_immediate_scrub_data_uncompressed OFF # make sure spaces are checked quickly SET GLOBAL innodb_background_scrub_data_check_interval=1; -create table snapshot_status engine = myisam -select * from information_schema.global_status -where variable_name like 'innodb_scrub%'; -truncate table snapshot_status; -insert into snapshot_status -select * from information_schema.global_status -where variable_name like 'innodb_scrub%'; -# -# Test delete of records -# -create table t1 ( +create table delete_3 ( a int auto_increment primary key, b varchar(256), -c text, index(b)) engine = innodb row_format=dynamic; +c text, +index(b)) engine = innodb row_format=compressed; # Populate table with rows -delete from t1; -# -# Test delete+rollback+delete -# -create table t2 ( +delete from delete_3; +create table delete_rollback_delete_3 ( a int auto_increment primary key, b varchar(256), -c text, index(b)) engine = innodb row_format=dynamic; +c text, +index(b)) engine = innodb row_format=compressed; # Populate table with rows begin; -delete from t2; +delete from delete_rollback_delete_3; rollback; -delete from t2; -# -# Test insert+rollback -# -create table t3 ( +delete from delete_rollback_delete_3; +create table insert_rollback_3 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=compressed; +# Populate table with rows +begin; +rollback; +create table delete_2 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=compact; +# Populate table with rows +delete from delete_2; +create table delete_rollback_delete_2 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=compact; +# Populate table with rows +begin; +delete from delete_rollback_delete_2; +rollback; +delete from delete_rollback_delete_2; +create table insert_rollback_2 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=compact; +# Populate table with rows +begin; +rollback; +create table delete_1 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=redundant; +# Populate table with rows +delete from delete_1; +create table delete_rollback_delete_1 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=redundant; +# Populate table with rows +begin; +delete from delete_rollback_delete_1; +rollback; +delete from delete_rollback_delete_1; +create table insert_rollback_1 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=redundant; +# Populate table with rows +begin; +rollback; +create table delete_0 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=dynamic; +# Populate table with rows +delete from delete_0; +create table delete_rollback_delete_0 ( +a int auto_increment primary key, +b varchar(256), +c text, +index(b)) engine = innodb row_format=dynamic; +# Populate table with rows +begin; +delete from delete_rollback_delete_0; +rollback; +delete from delete_rollback_delete_0; +create table insert_rollback_0 ( a int auto_increment primary key, b varchar(256), -c text, index(b)) engine = innodb row_format=dynamic; +c text, +index(b)) engine = innodb row_format=dynamic; # Populate table with rows begin; rollback; # start scrubbing threads SET GLOBAL innodb_encryption_threads=5; # Wait max 10 min for scrubbing -# Success! -# stop scrubbing threads -SET GLOBAL innodb_encryption_threads=0; -# restart mysqld so that all pages are flushed -# read all rows from table -select * from t1; -# dynamic: delete: grep -c bicycle t1.ibd -0 -# dynamic: delete: grep -c repairman t1.ibd -0 -# dynamic: delete rollback: grep -c bicycle t2.ibd -0 -# dynamic: delete rollback: grep -c repairman t2.ibd -0 -# dynamic: insert rollback: grep -c bicycle t3.ibd -0 -# dynamic: insert rollback: grep -c repairman t3.ibd -0 -drop table t1, t2, t3; +SET GLOBAL innodb_fast_shutdown=0; +# delete_3.ibd +# delete_rollback_delete_3.ibd +# insert_rollback_3.ibd +# delete_2.ibd +# delete_rollback_delete_2.ibd +# insert_rollback_2.ibd +# delete_1.ibd +# delete_rollback_delete_1.ibd +# insert_rollback_1.ibd +# delete_0.ibd +# delete_rollback_delete_0.ibd +# insert_rollback_0.ibd +check table delete_3, delete_rollback_delete_3, insert_rollback_3; +Table Op Msg_type Msg_text +test.delete_3 check status OK +test.delete_rollback_delete_3 check status OK +test.insert_rollback_3 check status OK +drop table delete_3, delete_rollback_delete_3, insert_rollback_3; +check table delete_2, delete_rollback_delete_2, insert_rollback_2; +Table Op Msg_type Msg_text +test.delete_2 check status OK +test.delete_rollback_delete_2 check status OK +test.insert_rollback_2 check status OK +drop table delete_2, delete_rollback_delete_2, insert_rollback_2; +check table delete_1, delete_rollback_delete_1, insert_rollback_1; +Table Op Msg_type Msg_text +test.delete_1 check status OK +test.delete_rollback_delete_1 check status OK +test.insert_rollback_1 check status OK +drop table delete_1, delete_rollback_delete_1, insert_rollback_1; +check table delete_0, delete_rollback_delete_0, insert_rollback_0; +Table Op Msg_type Msg_text +test.delete_0 check status OK +test.delete_rollback_delete_0 check status OK +test.insert_rollback_0 check status OK +drop table delete_0, delete_rollback_delete_0, insert_rollback_0; show variables like 'innodb_%scrub_data%'; Variable_name Value innodb_background_scrub_data_check_interval 3600 @@ -78,4 +158,3 @@ innodb_background_scrub_data_compressed ON innodb_background_scrub_data_interval 604800 innodb_background_scrub_data_uncompressed ON innodb_immediate_scrub_data_uncompressed OFF -drop table snapshot_status; diff --git a/mysql-test/suite/encryption/r/innodb_scrub_compressed.result b/mysql-test/suite/encryption/r/innodb_scrub_compressed.result deleted file mode 100644 index 0b5e9f11a05cf..0000000000000 --- a/mysql-test/suite/encryption/r/innodb_scrub_compressed.result +++ /dev/null @@ -1,71 +0,0 @@ -# make sure spaces are checked quickly -SET GLOBAL innodb_background_scrub_data_check_interval=1; -# -# Test delete of records -# -create table t1 ( -a int auto_increment primary key, -b varchar(256), -c text) engine = innodb row_format=compressed; -# Populate table with rows -delete from t1; -# -# Test delete+rollback+delete -# -create table t2 ( -a int auto_increment primary key, -b varchar(256), -c text) engine = innodb row_format=compressed; -# Populate table with rows -begin; -delete from t2; -rollback; -delete from t2; -# -# Test insert+rollback -# -create table t3 ( -a int auto_increment primary key, -b varchar(256), -c text) engine = innodb row_format=compressed; -# Populate table with rows -begin; -rollback; -# start scrubbing threads -SET GLOBAL innodb_encryption_threads=5; -# Wait max 10 min for scrubbing of this table -# Success! -# stop scrubbing threads -SET GLOBAL innodb_encryption_threads=0; -# Now there should be background scrubs -# restart mysqld so that all pages are flushed (encryption off) -# so that grep will find stuff -# read all rows from table -select * from t1; -select * from t2; -select * from t3; -# grep -c bicycle t1.ibd -0 -# grep -c bicycle ibdata1 -0 -# grep -c repairman t1.ibd -0 -# grep -c repairman ibdata1 -0 -# grep -c boondoggle t2.ibd -0 -# grep -c boondoggle ibdata1 -0 -# grep -c waste t2.ibd -0 -# grep -c waste ibdata1 -0 -# grep -c keso t3.ibd -0 -# grep -c keso ibdata1 -0 -# grep -c kent t3.ibd -0 -# grep -c kent ibdata1 -0 -drop table t1, t2, t3; diff --git a/mysql-test/suite/encryption/t/innodb_scrub.test b/mysql-test/suite/encryption/t/innodb_scrub.test index 32170567b4cb9..6cb48530f86ea 100644 --- a/mysql-test/suite/encryption/t/innodb_scrub.test +++ b/mysql-test/suite/encryption/t/innodb_scrub.test @@ -1,147 +1,138 @@ -- source include/have_innodb.inc -- source include/not_embedded.inc -- source include/have_example_key_management_plugin.inc --- source include/not_windows.inc let $MYSQLD_DATADIR=`select @@datadir`; -let ib1_IBD = $MYSQLD_DATADIR/ibdata1; -let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd; +let INNODB_PAGE_SIZE= `select @@innodb_page_size`; create table snapshot_status engine = myisam select * from information_schema.global_status where variable_name like 'innodb_scrub%'; let $rowcount=500; -let $formatno = 3; +let $maxformatno= 4; +let $formatno= $maxformatno; + +--echo # MDEV-8139 Fix scrubbing tests +--echo # FIXME: Add index(b) to each table; ensure that undo logs are scrubbed. +let $tableformat= ( + a int auto_increment primary key, + b varchar(256), + c text) engine = innodb row_format; + while ($formatno) { +dec $formatno; let $format = `select case $formatno - when 1 then 'dynamic' - when 2 then 'redundant' - when 3 then 'compact' + when 0 then 'dynamic' + when 1 then 'redundant' + when 2 then 'compact' + when 3 then 'compressed' end`; -dec $formatno; --- echo # --- echo # Test delete of records --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=$format; +let $t= delete_$formatno; +eval create table $t $tableformat=$format; let $numinserts = $rowcount; --- echo # Populate table with rows --disable_query_log +begin; while ($numinserts) { dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('repairman', repeat('unicycle', 1000)); } +commit; --enable_query_log -delete from t1; +eval delete from $t; --- echo # restart mysqld so that all pages are flushed --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; --- enable_result_log - --- echo # $format: delete from: grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # $format: delete from: grep -c bicycle ibdata1 --- exec grep -c bicycle $ib1_IBD || true --- echo # $format: delete from: grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true --- echo # $format: delete from: grep -c repairman ibdata1 --- exec grep -c repairman $ib1_IBD || true - -drop table t1; - --- echo # --- echo # Test delete+rollback+delete --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=$format; +let $t= delete_rollback_delete_$formatno; +eval create table $t $tableformat=$format; let $numinserts = $rowcount; --- echo # Populate table with rows --disable_query_log +begin; while ($numinserts) { dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('breakhuman', repeat('bicycle', 1000)); } +commit; --enable_query_log begin; -delete from t1; +eval delete from $t; rollback; -delete from t1; - --- echo # restart mysqld so that all pages are flushed --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; --- enable_result_log - --- echo # $format: delete rollback: grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # $format: delete rollback: grep -c bicycle ibdata1 --- exec grep -c bicycle $ib1_IBD || true --- echo # $format: delete rollback: grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true --- echo # $format: delete rollback: grep -c repairman ibdata1 --- exec grep -c repairman $ib1_IBD || true - -drop table t1; - --- echo # --- echo # Test insert+rollback --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=$format; +eval delete from $t; + +let $t= insert_rollback_$formatno; + +eval create table $t $tableformat=$format; let $numinserts = $rowcount; --- echo # Populate table with rows begin; --disable_query_log while ($numinserts) { dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('wonderwoman', repeat('tricycle', 1000)); } --enable_query_log rollback; +} + +SET GLOBAL innodb_fast_shutdown=0; +-- source include/shutdown_mysqld.inc + +let SEARCH_ABORT= FOUND; +let SEARCH_PATTERN= (un|b|tr)icycle|(repair|breakhu|wonderwo)man; +let SEARCH_RANGE= 12582912; +let SEARCH_FILE= $MYSQLD_DATADIR/ibdata1; + +# We may randomly find copies of unscrubbed pages in the doublewrite buffer. +# Let us scrub the doublewrite buffer ourselves. +perl; +use Fcntl 'SEEK_SET'; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +open(FILE, "+<", "$ENV{SEARCH_FILE}") or die "cannot open: $!\n"; +seek(FILE, $page_size * 64, SEEK_SET) or die "cannot seek: $!\n"; +print(FILE chr(0) x ($page_size * 128)) or die "cannot write: $!\n"; +close FILE or die "cannot close: $!\n";; +EOF + +-- source include/search_pattern_in_file.inc + +let $formatno= $maxformatno; +while ($formatno) +{ +dec $formatno; + +let $t= delete_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +let $t= delete_rollback_delete_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +let $t= insert_rollback_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +} + +-- source include/start_mysqld.inc + +let $formatno= $maxformatno; +while ($formatno) +{ +dec $formatno; + +let $t= delete_$formatno, delete_rollback_delete_$formatno, insert_rollback_$formatno; --- echo # restart mysqld so that all pages are flushed --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; --- enable_result_log - --- echo # $format: insert rollback: grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # $format: insert rollback: grep -c bicycle ibdata1 --- exec grep -c bicycle $ib1_IBD || true --- echo # $format: insert rollback: grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true --- echo # $format: insert rollback: grep -c repairman ibdata1 --- exec grep -c repairman $ib1_IBD || true - -drop table t1; +eval check table $t; +eval drop table $t; } show variables like 'innodb_%scrub_data%'; diff --git a/mysql-test/suite/encryption/t/innodb_scrub_background.test b/mysql-test/suite/encryption/t/innodb_scrub_background.test index 19e566831173d..c705ee510068a 100644 --- a/mysql-test/suite/encryption/t/innodb_scrub_background.test +++ b/mysql-test/suite/encryption/t/innodb_scrub_background.test @@ -1,13 +1,9 @@ -- source include/have_innodb.inc -- source include/not_embedded.inc -- source include/have_example_key_management_plugin.inc --- source include/not_windows.inc let $MYSQLD_DATADIR=`select @@datadir`; -let ib1_IBD = $MYSQLD_DATADIR/ibdata1; -let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd; -let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; -let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; +let INNODB_PAGE_SIZE= `select @@innodb_page_size`; --echo # --echo # immediate scrubbing is off @@ -18,80 +14,67 @@ show variables like 'innodb_%scrub_data%'; -- echo # make sure spaces are checked quickly SET GLOBAL innodb_background_scrub_data_check_interval=1; -create table snapshot_status engine = myisam -select * from information_schema.global_status -where variable_name like 'innodb_scrub%'; - let $rowcount=500; -let $formatno = 1; +let $maxformatno= 4; +let $formatno= $maxformatno; + +let $tableformat= ( + a int auto_increment primary key, + b varchar(256), + c text, + index(b)) engine = innodb row_format; + while ($formatno) { +dec $formatno; let $format = `select case $formatno - when 1 then 'dynamic' - when 2 then 'redundant' - when 3 then 'compact' - when 4 then 'compressed' + when 0 then 'dynamic' + when 1 then 'redundant' + when 2 then 'compact' + when 3 then 'compressed' end`; -dec $formatno; - -truncate table snapshot_status; -insert into snapshot_status -select * from information_schema.global_status -where variable_name like 'innodb_scrub%'; - --- echo # --- echo # Test delete of records --- echo # -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text, index(b)) engine = innodb row_format=$format; +let $t= delete_$formatno; +eval create table $t $tableformat=$format; let $numinserts = $rowcount; -- echo # Populate table with rows --disable_query_log +begin; while ($numinserts) { dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('unicycle', repeat('wonderwoman', 1000)); } +commit; --enable_query_log -delete from t1; +eval delete from $t; --- echo # --- echo # Test delete+rollback+delete --- echo # +let $t= delete_rollback_delete_$formatno; -eval create table t2 ( - a int auto_increment primary key, - b varchar(256), - c text, index(b)) engine = innodb row_format=$format; +eval create table $t $tableformat=$format; let $numinserts = $rowcount; -- echo # Populate table with rows --disable_query_log +begin; while ($numinserts) { dec $numinserts; - insert into t2(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('bicycle', repeat('repairman', 1000)); } +commit; --enable_query_log begin; -delete from t2; +eval delete from $t; rollback; -delete from t2; +eval delete from $t; --- echo # --- echo # Test insert+rollback --- echo # +let $t= insert_rollback_$formatno; -eval create table t3 ( - a int auto_increment primary key, - b varchar(256), - c text, index(b)) engine = innodb row_format=$format; +eval create table $t $tableformat=$format; let $numinserts = $rowcount; -- echo # Populate table with rows @@ -100,11 +83,12 @@ begin; while ($numinserts) { dec $numinserts; - insert into t3(b,c) values ('bicycle', repeat('repairman', 1000)); + eval insert into $t(b,c) values ('tricycle', repeat('superhuman', 1000)); } --enable_query_log rollback; +} -- echo # start scrubbing threads SET GLOBAL innodb_encryption_threads=5; @@ -130,35 +114,57 @@ if (!$success) -- die Timeout waiting for background threads } --- echo # Success! --- echo # stop scrubbing threads -SET GLOBAL innodb_encryption_threads=0; - --- echo # restart mysqld so that all pages are flushed --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; --- enable_result_log - --- echo # $format: delete: grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # $format: delete: grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true - --- echo # $format: delete rollback: grep -c bicycle t2.ibd --- exec grep -c bicycle $t2_IBD || true --- echo # $format: delete rollback: grep -c repairman t2.ibd --- exec grep -c repairman $t2_IBD || true - --- echo # $format: insert rollback: grep -c bicycle t3.ibd --- exec grep -c bicycle $t3_IBD || true --- echo # $format: insert rollback: grep -c repairman t3.ibd --- exec grep -c repairman $t3_IBD || true - -drop table t1, t2, t3; +SET GLOBAL innodb_fast_shutdown=0; +-- source include/shutdown_mysqld.inc + +let SEARCH_ABORT= FOUND; +let SEARCH_PATTERN= (un|b|tr)icycle|(repair|breakhu|wonderwo)man; +let SEARCH_RANGE= 12582912; +let SEARCH_FILE= $MYSQLD_DATADIR/ibdata1; + +# We may randomly find copies of unscrubbed pages in the doublewrite buffer. +# Let us scrub the doublewrite buffer ourselves. +perl; +use Fcntl 'SEEK_SET'; +my $page_size = $ENV{INNODB_PAGE_SIZE}; +open(FILE, "+<", "$ENV{SEARCH_FILE}") or die "cannot open: $!\n"; +seek(FILE, $page_size * 64, SEEK_SET) or die "cannot seek: $!\n"; +print(FILE chr(0) x ($page_size * 128)) or die "cannot write: $!\n"; +close FILE or die "cannot close: $!\n";; +EOF + +-- source include/search_pattern_in_file.inc + +let $formatno= $maxformatno; +while ($formatno) +{ +dec $formatno; + +let $t= delete_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +let $t= delete_rollback_delete_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc +let $t= insert_rollback_$formatno.ibd; +let SEARCH_FILE= $MYSQLD_DATADIR/test/$t; +-- echo # $t +-- source include/search_pattern_in_file.inc } -show variables like 'innodb_%scrub_data%'; +-- source include/start_mysqld.inc -drop table snapshot_status; +let $formatno= $maxformatno; +while ($formatno) +{ +dec $formatno; + +let $t= delete_$formatno, delete_rollback_delete_$formatno, insert_rollback_$formatno; + +eval check table $t; +eval drop table $t; +} + +show variables like 'innodb_%scrub_data%'; diff --git a/mysql-test/suite/encryption/t/innodb_scrub_compressed.opt b/mysql-test/suite/encryption/t/innodb_scrub_compressed.opt deleted file mode 100644 index 48b04fa7049fc..0000000000000 --- a/mysql-test/suite/encryption/t/innodb_scrub_compressed.opt +++ /dev/null @@ -1,9 +0,0 @@ ---innodb-file-per-table=1 ---innodb-file-format=Barracuda ---innodb-immediate-scrub-data-uncompressed=ON ---innodb-background-scrub-data-uncompressed=ON ---innodb-background-scrub-data-compressed=ON ---loose-innodb-debug-force-scrubbing=ON ---innodb-encrypt-tables=OFF ---innodb-encrypt-log=OFF ---innodb-tablespaces-scrubbing diff --git a/mysql-test/suite/encryption/t/innodb_scrub_compressed.test b/mysql-test/suite/encryption/t/innodb_scrub_compressed.test deleted file mode 100644 index d41edac749408..0000000000000 --- a/mysql-test/suite/encryption/t/innodb_scrub_compressed.test +++ /dev/null @@ -1,161 +0,0 @@ --- source include/have_innodb.inc --- source include/not_embedded.inc --- source include/have_example_key_management_plugin.inc --- source include/not_windows.inc - -let $MYSQLD_DATADIR=`select @@datadir`; -let ib1_IBD = $MYSQLD_DATADIR/ibdata1; -let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd; -let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd; -let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd; - -let $rowcount=500; - --- echo # make sure spaces are checked quickly -SET GLOBAL innodb_background_scrub_data_check_interval=1; - --- echo # --- echo # Test delete of records --- echo # - -eval create table t1 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=compressed; - -let $numinserts = $rowcount; --- echo # Populate table with rows ---disable_query_log -while ($numinserts) -{ - dec $numinserts; - insert into t1(b,c) values ('bicycle', repeat('repairman', 1000)); -} ---enable_query_log - -delete from t1; - --- echo # --- echo # Test delete+rollback+delete --- echo # - -eval create table t2 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=compressed; - -let $numinserts = $rowcount; --- echo # Populate table with rows ---disable_query_log -while ($numinserts) -{ - dec $numinserts; - insert into t2(b,c) values ('boondoggle', repeat('waste of time', 1000)); -} ---enable_query_log - -begin; -delete from t2; -rollback; -delete from t2; - --- echo # --- echo # Test insert+rollback --- echo # - -eval create table t3 ( - a int auto_increment primary key, - b varchar(256), - c text) engine = innodb row_format=compressed; - -let $numinserts = $rowcount; --- echo # Populate table with rows -begin; ---disable_query_log -while ($numinserts) -{ - dec $numinserts; - insert into t3(b,c) values ('keso', repeat('kent', 1000)); -} ---enable_query_log - -rollback; - --- echo # start scrubbing threads -SET GLOBAL innodb_encryption_threads=5; --- echo # Wait max 10 min for scrubbing of this table -let $cnt=600; -while ($cnt) -{ - let $success=`SELECT COUNT(*) = 0 -FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_SCRUBBING -WHERE LAST_SCRUB_COMPLETED IS NULL AND ( NAME like 'test/%' OR SPACE = 0 )`; - - if ($success) - { - let $cnt=0; - } - if (!$success) - { - real_sleep 1; - dec $cnt; - } -} -if (!$success) -{ - SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_SCRUBBING; - SHOW STATUS LIKE 'innodb_%scrub%'; - -- die Timeout waiting for background threads -} --- echo # Success! --- echo # stop scrubbing threads -SET GLOBAL innodb_encryption_threads=0; - ---echo # Now there should be background scrubs -let $success=`select sum(variable_value) > 0 -from information_schema.global_status -where variable_name in ('innodb_scrub_background_page_reorganizations', -'innodb_scrub_background_page_splits')`; - -if (!$success) { - show status like 'innodb_scrub%'; -} - --- echo # restart mysqld so that all pages are flushed (encryption off) --- echo # so that grep will find stuff --- source include/restart_mysqld.inc --- echo # read all rows from table --- disable_result_log -select * from t1; -select * from t2; -select * from t3; --- enable_result_log - --- echo # grep -c bicycle t1.ibd --- exec grep -c bicycle $t1_IBD || true --- echo # grep -c bicycle ibdata1 --- exec grep -c bicycle $ib1_IBD || true --- echo # grep -c repairman t1.ibd --- exec grep -c repairman $t1_IBD || true --- echo # grep -c repairman ibdata1 --- exec grep -c repairman $ib1_IBD || true - --- echo # grep -c boondoggle t2.ibd --- exec grep -c boondoggle $t2_IBD || true --- echo # grep -c boondoggle ibdata1 --- exec grep -c boondoggle $ib1_IBD || true --- echo # grep -c waste t2.ibd --- exec grep -c waste $t2_IBD || true --- echo # grep -c waste ibdata1 --- exec grep -c waste $ib1_IBD || true - --- echo # grep -c keso t3.ibd --- exec grep -c keso $t3_IBD || true --- echo # grep -c keso ibdata1 --- exec grep -c keso $ib1_IBD || true --- echo # grep -c kent t3.ibd --- exec grep -c kent $t3_IBD || true --- echo # grep -c kent ibdata1 --- exec grep -c kent $ib1_IBD || true - -drop table t1, t2, t3; From 348ccb6f038a6c108ab9b6a01bdc356cefecd3d4 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Wed, 4 Jan 2017 14:33:24 -0800 Subject: [PATCH 26/74] Fixed bug mdev-11674. 1. The rows of a recursive CTE at some point may overflow the HEAP temporary table containing them. At this point the table is converted to a MyISAM temporary table and the new added rows are placed into this MyISAM table. A bug in the of select_union_recursive::send_data prevented the server from writing the row that caused the overflow into the temporary table used for the result of the iteration steps. This could lead, in particular,to a premature end of the iterations. 2. The method TABLE::insert_all_rows_into() that was used to copy all rows of one temporary table into another did not take into account that the destination temporary table must be converted to a MyISAM table at some point. This patch fixed this problem. It also renamed the method into TABLE::insert_all_rows_into_tmp_table() and added an extra parameter needed for the conversion. --- mysql-test/r/cte_recursive.result | 16 +++++++++++++++ mysql-test/t/cte_recursive.test | 21 +++++++++++++++++++ sql/sql_derived.cc | 5 ++++- sql/sql_union.cc | 7 +++++-- sql/table.cc | 34 +++++++++++++++++++++---------- sql/table.h | 6 +++++- 6 files changed, 74 insertions(+), 15 deletions(-) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index 0cc88d27e237d..d5476aec1c45a 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2327,3 +2327,19 @@ a b dist 7 6 3 DROP VIEW edges2; DROP TABLE edges; +# +# MDEV-11674: recursive CTE table that cannot be stored +# in a heap table +# +create table t1 (id int, test_data varchar(36)); +insert into t1(id, test_data) +select id, test_data +from ( +with recursive data_generator(id, test_data) as ( +select 1 as id, uuid() as test_data +union all +select id + 1, uuid() from data_generator where id < 150000 +) +select * from data_generator +) as a; +drop table t1; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 63bcfbffeb94e..ea0f73be2595a 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1484,3 +1484,24 @@ ORDER BY a, dist, b; DROP VIEW edges2; DROP TABLE edges; + + +--echo # +--echo # MDEV-11674: recursive CTE table that cannot be stored +--echo # in a heap table +--echo # + +create table t1 (id int, test_data varchar(36)); + +insert into t1(id, test_data) +select id, test_data + from ( + with recursive data_generator(id, test_data) as ( + select 1 as id, uuid() as test_data + union all + select id + 1, uuid() from data_generator where id < 150000 + ) + select * from data_generator + ) as a; + +drop table t1; diff --git a/sql/sql_derived.cc b/sql/sql_derived.cc index daac90d56f641..898b6336ae58f 100644 --- a/sql/sql_derived.cc +++ b/sql/sql_derived.cc @@ -969,7 +969,10 @@ bool TABLE_LIST::fill_recursive(THD *thd) if (!rc) { TABLE *src= with->rec_result->table; - rc =src->insert_all_rows_into(thd, table, true); + rc =src->insert_all_rows_into_tmp_table(thd, + table, + &with->rec_result->tmp_table_param, + true); } return rc; } diff --git a/sql/sql_union.cc b/sql/sql_union.cc index aad3701cca288..b30b55287417a 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -105,7 +105,7 @@ int select_union_recursive::send_data(List &values) { int rc= select_union::send_data(values); - if (!write_err) + if (write_err != HA_ERR_FOUND_DUPP_KEY) { int err; if ((err= incr_table->file->ha_write_tmp_row(table->record[0]))) @@ -1192,6 +1192,7 @@ bool st_select_lex_unit::exec_recursive() st_select_lex *end= NULL; bool is_unrestricted= with_element->is_unrestricted(); List_iterator_fast li(with_element->rec_result->rec_tables); + TMP_TABLE_PARAM *tmp_table_param= &with_element->rec_result->tmp_table_param; ha_rows examined_rows= 0; bool was_executed= executed; TABLE *rec_table; @@ -1247,7 +1248,9 @@ bool st_select_lex_unit::exec_recursive() while ((rec_table= li++)) { saved_error= - incr_table->insert_all_rows_into(thd, rec_table, !is_unrestricted); + incr_table->insert_all_rows_into_tmp_table(thd, rec_table, + tmp_table_param, + !is_unrestricted); if (!with_element->rec_result->first_rec_table_to_update) with_element->rec_result->first_rec_table_to_update= rec_table; if (with_element->level == 1) diff --git a/sql/table.cc b/sql/table.cc index 4688b77ecd774..6d11ac3d3eb98 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -7573,46 +7573,58 @@ bool TABLE::validate_default_values_of_unset_fields(THD *thd) const } -bool TABLE::insert_all_rows_into(THD *thd, TABLE *dest, bool with_cleanup) +bool TABLE::insert_all_rows_into_tmp_table(THD *thd, + TABLE *tmp_table, + TMP_TABLE_PARAM *tmp_table_param, + bool with_cleanup) { int write_err= 0; - DBUG_ENTER("TABLE::insert_all_rows_into"); + DBUG_ENTER("TABLE::insert_all_rows_into_tmp_table"); if (with_cleanup) { - if ((write_err= dest->file->ha_delete_all_rows())) + if ((write_err= tmp_table->file->ha_delete_all_rows())) goto err; } if (file->indexes_are_disabled()) - dest->file->ha_disable_indexes(HA_KEY_SWITCH_ALL); + tmp_table->file->ha_disable_indexes(HA_KEY_SWITCH_ALL); file->ha_index_or_rnd_end(); if (file->ha_rnd_init_with_error(1)) DBUG_RETURN(1); - if (dest->no_rows) - dest->file->extra(HA_EXTRA_NO_ROWS); + if (tmp_table->no_rows) + tmp_table->file->extra(HA_EXTRA_NO_ROWS); else { /* update table->file->stats.records */ file->info(HA_STATUS_VARIABLE); - dest->file->ha_start_bulk_insert(file->stats.records); + tmp_table->file->ha_start_bulk_insert(file->stats.records); } - while (!file->ha_rnd_next(dest->record[1])) + while (!file->ha_rnd_next(tmp_table->record[0])) { - write_err= dest->file->ha_write_tmp_row(dest->record[1]); + write_err= tmp_table->file->ha_write_tmp_row(tmp_table->record[0]); if (write_err) - goto err; + { + bool is_duplicate; + if (tmp_table->file->is_fatal_error(write_err, HA_CHECK_DUP) && + create_internal_tmp_table_from_heap(thd, tmp_table, + tmp_table_param->start_recinfo, + &tmp_table_param->recinfo, + write_err, 1, &is_duplicate)) + DBUG_RETURN(1); + + } if (thd->check_killed()) { thd->send_kill_message(); goto err_killed; } } - if (!dest->no_rows && dest->file->ha_end_bulk_insert()) + if (!tmp_table->no_rows && tmp_table->file->ha_end_bulk_insert()) goto err; DBUG_RETURN(0); diff --git a/sql/table.h b/sql/table.h index 6552a8e13da76..b2d5599b74095 100644 --- a/sql/table.h +++ b/sql/table.h @@ -53,6 +53,7 @@ class With_element; struct TDC_element; class Virtual_column_info; class Table_triggers_list; +class TMP_TABLE_PARAM; /* Used to identify NESTED_JOIN structures within a join (applicable only to @@ -1447,7 +1448,10 @@ struct TABLE inline Field **field_to_fill(); bool validate_default_values_of_unset_fields(THD *thd) const; - bool insert_all_rows_into(THD *thd, TABLE *dest, bool with_cleanup); + bool insert_all_rows_into_tmp_table(THD *thd, + TABLE *tmp_table, + TMP_TABLE_PARAM *tmp_table_param, + bool with_cleanup); }; From 758af98ff7c47cc1fb5debdc138312fa389d528f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jan 2017 10:42:19 +0200 Subject: [PATCH 27/74] Post-push fix for Part 1 of MDEV-8139 Fix scrubbing tests In the backport of Bug#24450908 UNDO LOG EXISTS AFTER SLOW SHUTDOWN from MySQL 5.7 to the MySQL 5.6 based MariaDB Server 10.1, we must use a mutex when HAVE_ATOMIC_BUILTINS is not defined. Also, correct a function comment. In MySQL 5.6 and MariaDB Server 10.1, also temporary InnoDB tables are redo-logged. --- storage/innobase/trx/trx0purge.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 42b94d9b6a3b6..b3d16546a6391 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -294,15 +294,19 @@ trx_purge_remove_log_hdr( { flst_remove(rseg_hdr + TRX_RSEG_HISTORY, log_hdr + TRX_UNDO_HISTORY_NODE, mtr); - +#ifdef HAVE_ATOMIC_BUILTINS os_atomic_decrement_ulint(&trx_sys->rseg_history_len, 1); +#else + mutex_enter(&trx_sys->mutex); + --trx_sys->rseg_history_len; + mutex_exit(&trx_sys->mutex); +#endif } /** Frees an undo log segment which is in the history list. Removes the undo log hdr from the history list. @param[in,out] rseg rollback segment -@param[in] hdr_addr file address of log_hdr -@param[in] noredo skip redo logging. */ +@param[in] hdr_addr file address of log_hdr */ static void trx_purge_free_segment( From bf35deda09f18f5c7f3179ace731a2a186e05445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 3 Jan 2017 15:38:09 +0200 Subject: [PATCH 28/74] MDEV-11713 Optimize DBUG_PRINT and introduce DBUG_LOG MariaDB Server is unnecessarily evaluating the arguments of DBUG_PRINT() macros when the label is not defined. The macro DBUG_LOG() for C++ operator<< output which was added for InnoDB diagnostics in MySQL 5.7 is missing from MariaDB. Unlike the MySQL 5.7 implementation, MariaDB will avoid allocating and initializing the output string when the label is not defined. Introduce DBUG_OUT("crypt") and DBUG_OUT("checksum") for some InnoDB diagnostics, replacing some use of ib::info(). --- dbug/dbug.c | 35 +++++++++---------- include/my_dbug.h | 18 +++++++--- storage/innobase/buf/buf0buf.cc | 62 ++++++++++++++------------------- storage/innobase/fil/fil0fil.cc | 28 +++++++-------- storage/innobase/fsp/fsp0fsp.cc | 6 ++++ storage/innobase/row/row0upd.cc | 16 ++------- 6 files changed, 77 insertions(+), 88 deletions(-) diff --git a/dbug/dbug.c b/dbug/dbug.c index b2b298beb096d..048e4803b1af2 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -1213,7 +1213,7 @@ void _db_return_(struct _db_stack_frame_ *_stack_frame_) * * SYNOPSIS * - * VOID _db_pargs_(_line_, keyword) + * int _db_pargs_(_line_, keyword) * int _line_; * char *keyword; * @@ -1226,12 +1226,14 @@ void _db_return_(struct _db_stack_frame_ *_stack_frame_) * */ -void _db_pargs_(uint _line_, const char *keyword) +int _db_pargs_(uint _line_, const char *keyword) { CODE_STATE *cs; - get_code_state_or_return; + get_code_state_or_return 0; cs->u_line= _line_; cs->u_keyword= keyword; + + return DEBUGGING && _db_keyword_(cs, cs->u_keyword, 0); } @@ -1265,27 +1267,24 @@ void _db_doprnt_(const char *format,...) { va_list args; CODE_STATE *cs; + int save_errno; + get_code_state_or_return; va_start(args,format); if (!cs->locked) pthread_mutex_lock(&THR_LOCK_dbug); - if (_db_keyword_(cs, cs->u_keyword, 0)) - { - int save_errno=errno; - DoPrefix(cs, cs->u_line); - if (TRACING) - Indent(cs, cs->level + 1); - else - (void) fprintf(cs->stack->out_file->file, "%s: ", cs->func); - (void) fprintf(cs->stack->out_file->file, "%s: ", cs->u_keyword); - DbugVfprintf(cs->stack->out_file->file, format, args); - DbugFlush(cs); - errno=save_errno; - } - else if (!cs->locked) - pthread_mutex_unlock(&THR_LOCK_dbug); + save_errno=errno; + DoPrefix(cs, cs->u_line); + if (TRACING) + Indent(cs, cs->level + 1); + else + (void) fprintf(cs->stack->out_file->file, "%s: ", cs->func); + (void) fprintf(cs->stack->out_file->file, "%s: ", cs->u_keyword); + DbugVfprintf(cs->stack->out_file->file, format, args); + DbugFlush(cs); + errno=save_errno; va_end(args); } diff --git a/include/my_dbug.h b/include/my_dbug.h index d56033ab02525..ba9e8a025d77b 100644 --- a/include/my_dbug.h +++ b/include/my_dbug.h @@ -1,5 +1,5 @@ /* Copyright (c) 2000, 2010, Oracle and/or its affiliates. - Copyright (C) 2000-2011 Monty Program Ab + Copyright (C) 2000, 2017, MariaDB Corporation Ab This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -50,7 +50,7 @@ extern void _db_set_init_(const char *control); extern void _db_enter_(const char *_func_, const char *_file_, uint _line_, struct _db_stack_frame_ *_stack_frame_); extern void _db_return_(struct _db_stack_frame_ *_stack_frame_); -extern void _db_pargs_(uint _line_,const char *keyword); +extern int _db_pargs_(uint _line_,const char *keyword); extern void _db_doprnt_(const char *format,...) ATTRIBUTE_FORMAT(printf, 1, 2); extern void _db_dump_(uint _line_,const char *keyword, @@ -91,7 +91,7 @@ extern const char* _db_get_func_(void); #define DBUG_EVALUATE_IF(keyword,a1,a2) \ (_db_keyword_(0,(keyword), 1) ? (a1) : (a2)) #define DBUG_PRINT(keyword,arglist) \ - do {_db_pargs_(__LINE__,keyword); _db_doprnt_ arglist;} while(0) + do if (_db_pargs_(__LINE__,keyword)) _db_doprnt_ arglist; while(0) #define DBUG_PUSH(a1) _db_push_ (a1) #define DBUG_POP() _db_pop_ () #define DBUG_SET(a1) _db_set_ (a1) @@ -193,8 +193,18 @@ void debug_sync_point(const char* lock_name, uint lock_timeout); #define DBUG_SYNC_POINT(lock_name,lock_timeout) #endif /* EXTRA_DEBUG */ -#ifdef __cplusplus +#ifdef __cplusplus } +# ifdef DBUG_OFF +# define DBUG_LOG(keyword, v) do {} while (0) +# else +# include +# define DBUG_LOG(keyword, v) do { \ + if (_db_pargs_(__LINE__, keyword)) { \ + std::ostringstream _db_s; _db_s << v; \ + _db_doprnt_("%s", _db_s.str().c_str()); \ + }} while (0) +# endif #endif #endif /* _my_dbug_h */ diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index eeb851462b5b0..2f4edfb50145a 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -128,11 +128,6 @@ struct set_numa_interleave_t #define NUMA_MEMPOLICY_INTERLEAVE_IN_SCOPE #endif /* HAVE_LIBNUMA */ -/* Enable this for checksum error messages. */ -//#ifdef UNIV_DEBUG -//#define UNIV_DEBUG_LEVEL2 1 -//#endif - /* IMPLEMENTATION OF THE BUFFER POOL ================================= @@ -661,12 +656,10 @@ buf_page_is_checksum_valid_crc32( } invalid: -#ifdef UNIV_DEBUG_LEVEL2 - ib::info() << "Page checksum crc32 not valid" + DBUG_LOG("checksum", "Page checksum crc32 not valid" << " field1 " << checksum_field1 << " field2 " << checksum_field2 - << " crc32 " << crc32; -#endif + << " crc32 " << crc32); return(false); } @@ -738,13 +731,13 @@ buf_page_is_checksum_valid_innodb( if (checksum_field2 != mach_read_from_4(read_buf + FIL_PAGE_LSN) && checksum_field2 != old_checksum) { -#ifdef UNIV_DEBUG_LEVEL2 - ib::info() << "Page checksum crc32 not valid" - << " field1 " << checksum_field1 - << " field2 " << checksum_field2 - << " crc32 " << buf_calc_page_old_checksum(read_buf) - << " lsn " << mach_read_from_4(read_buf + FIL_PAGE_LSN); -#endif + DBUG_LOG("checksum", + "Page checksum crc32 not valid" + << " field1 " << checksum_field1 + << " field2 " << checksum_field2 + << " crc32 " << buf_calc_page_old_checksum(read_buf) + << " lsn " << mach_read_from_4( + read_buf + FIL_PAGE_LSN)); return(false); } @@ -754,13 +747,13 @@ buf_page_is_checksum_valid_innodb( (always equal to 0), to FIL_PAGE_SPACE_OR_CHKSUM */ if (checksum_field1 != 0 && checksum_field1 != new_checksum) { -#ifdef UNIV_DEBUG_LEVEL2 - ib::info() << "Page checksum crc32 not valid" - << " field1 " << checksum_field1 - << " field2 " << checksum_field2 - << " crc32 " << buf_calc_page_new_checksum(read_buf) - << " lsn " << mach_read_from_4(read_buf + FIL_PAGE_LSN); -#endif + DBUG_LOG("checksum", + "Page checksum crc32 not valid" + << " field1 " << checksum_field1 + << " field2 " << checksum_field2 + << " crc32 " << buf_calc_page_new_checksum(read_buf) + << " lsn " << mach_read_from_4( + read_buf + FIL_PAGE_LSN)); return(false); } @@ -790,13 +783,16 @@ buf_page_is_checksum_valid_none( #endif /* UNIV_INNOCHECKSUM */ ) { -#ifdef UNIV_DEBUG_LEVEL2 - if (!(checksum_field1 == checksum_field2 || checksum_field1 == BUF_NO_CHECKSUM_MAGIC)) { - ib::info() << "Page checksum crc32 not valid" - << " field1 " << checksum_field1 - << " field2 " << checksum_field2 - << " crc32 " << BUF_NO_CHECKSUM_MAGIC - << " lsn " << mach_read_from_4(read_buf + FIL_PAGE_LSN); +#ifndef DBUG_OFF + if (checksum_field1 != checksum_field2 + && checksum_field1 != BUF_NO_CHECKSUM_MAGIC) { + DBUG_LOG("checksum", + "Page checksum crc32 not valid" + << " field1 " << checksum_field1 + << " field2 " << checksum_field2 + << " crc32 " << BUF_NO_CHECKSUM_MAGIC + << " lsn " << mach_read_from_4(read_buf + + FIL_PAGE_LSN)); } #endif @@ -3467,12 +3463,6 @@ buf_pool_watch_set( buf_pool->watch[]. However, it is not in the critical code path as this function will be called only by the purge thread. */ -/* Enable this for checksum error messages. Currently on by -default on UNIV_DEBUG for encryption bugs. */ -#ifdef UNIV_DEBUG -#define UNIV_DEBUG_LEVEL2 1 -#endif - /* To obey latching order first release the hash_lock. */ rw_lock_x_unlock(*hash_lock); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 43b3100230698..19318358dc479 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1691,12 +1691,17 @@ fil_space_create( space->page_0_crypt_read = true; } -#ifdef UNIV_DEBUG - ib::info() << "Created tablespace for space " << space->id - << " name " << space->name - << " key_id " << (space->crypt_data ? space->crypt_data->key_id : 0) - << " encryption " << (space->crypt_data ? space->crypt_data->encryption : 0); -#endif + DBUG_LOG("tablespace", + "Tablespace for space " << id << " name " << name + << create_table ? " created" : " opened"); + if (crypt_data) { + DBUG_LOG("crypt", + "Tablespace " << id << " name " << name + << " encryption " << crypt_data->encryption + << " key id " << crypt_data->key_id + << ":" << fil_crypt_get_mode(crypt_data) + << " " << fil_crypt_get_type(crypt_data)); + } space->encryption_type = Encryption::NONE; @@ -1718,15 +1723,6 @@ fil_space_create( fil_system->max_assigned_id = id; } -#ifdef UNIV_DEBUG - if (crypt_data) { - /* If table could be encrypted print info */ - ib::info() << "Tablespace ID " << id << " name " << space->name - << ":" << fil_crypt_get_mode(crypt_data) - << " " << fil_crypt_get_type(crypt_data); - } -#endif - mutex_exit(&fil_system->mutex); return(space); @@ -7735,4 +7731,4 @@ fil_system_exit(void) { ut_ad(mutex_own(&fil_system->mutex)); mutex_exit(&fil_system->mutex); -} +} \ No newline at end of file diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index fe272accd5a7f..54070c124067e 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -3730,11 +3730,15 @@ fseg_free_page( the adaptive hash index */ mtr_t* mtr) /*!< in/out: mini-transaction */ { + DBUG_ENTER("fseg_free_page"); fseg_inode_t* seg_inode; buf_block_t* iblock; const fil_space_t* space = mtr_x_lock_space(space_id, mtr); const page_size_t page_size(space->flags); + DBUG_LOG("fseg_free_page", "space_id: " << space_id + << ", page_no: " << page); + seg_inode = fseg_inode_get(seg_header, space_id, page_size, mtr, &iblock); fil_block_check_type(iblock, FIL_PAGE_INODE, mtr); @@ -3744,6 +3748,8 @@ fseg_free_page( fseg_free_page_low(seg_inode, page_id, page_size, ahi, mtr); ut_d(buf_page_set_file_page_was_freed(page_id)); + + DBUG_VOID_RETURN; } /**********************************************************************//** diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index 3f7c795230a91..048f75e413f78 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -3473,26 +3473,14 @@ void upd_node_t::dbug_trace() for (upd_cascade_t::const_iterator i = cascade_upd_nodes->begin(); i != cascade_upd_nodes->end(); ++i) { - - const upd_node_t* update_node = *i; - ib::info() << "cascade_upd_nodes: Cascade to table: " << - update_node->table->name; - /* JAN: TODO: MySQL 5.7 DBUG_LOG("upd_node_t", "cascade_upd_nodes: Cascade to table: " - << update_node->table->name); - */ + << (*i)->table->name); } for (upd_cascade_t::const_iterator j = new_upd_nodes->begin(); j != new_upd_nodes->end(); ++j) { - - const upd_node_t* update_node = *j; - ib::info() << "cascade_upd_nodes: Cascade to table: " << - update_node->table->name; - /* JAN: TODO: MySQL 5.7 DBUG_LOG("upd_node_t", "new_upd_nodes: Cascade to table: " - << update_node->table->name); - */ + << (*j)->table->name); } DBUG_VOID_RETURN; From a8ac6dc5065a82b11b3f1a699be87aabc95d7a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jan 2017 11:49:00 +0200 Subject: [PATCH 29/74] Fix InnoDB compilation warnings. Most of them are trivial, except for the thread_sync_t refactoring. We must not invoke memset() on non-POD objects. mtflush_work_initialized: Remove. Refer to mtflush_ctx != NULL instead. thread_sync_t::thread_sync_t(): Refactored from buf_mtflu_handler_init(). thread_sync_t::~thread_sync_t(): Refactored from buf_mtflu_io_thread_exit(). --- storage/innobase/buf/buf0lru.cc | 4 - storage/innobase/buf/buf0mtflu.cc | 148 +++++++++++++---------------- storage/innobase/fil/fil0fil.cc | 4 +- storage/innobase/row/row0ftsort.cc | 1 - 4 files changed, 66 insertions(+), 91 deletions(-) diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 6a57e6746ffeb..0169a63e972ad 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -79,10 +79,6 @@ static const ulint BUF_LRU_DROP_SEARCH_SIZE = 1024; during LRU eviction. */ static const ulint BUF_LRU_SEARCH_SCAN_THRESHOLD = 100; -/** We scan these many blocks when looking for a clean page to evict -during LRU eviction. */ -#define BUF_LRU_SEARCH_SCAN_THRESHOLD 100 - /** If we switch on the InnoDB monitor because there are too few available frames in the buffer pool, we set this to TRUE */ static bool buf_lru_switched_on_innodb_mon = false; diff --git a/storage/innobase/buf/buf0mtflu.cc b/storage/innobase/buf/buf0mtflu.cc index 117de5cc948e8..7c15b12950e31 100644 --- a/storage/innobase/buf/buf0mtflu.cc +++ b/storage/innobase/buf/buf0mtflu.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2014, Fusion-io. All Rights Reserved. -Copyright (C) 2013, 2015, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -118,15 +118,65 @@ typedef struct wrk_itm mem_heap_t *rheap; } wrk_t; -typedef struct thread_data +struct thread_data_t { os_thread_id_t wthread_id; /*!< Identifier */ wthr_status_t wt_status; /*!< Worker thread status */ -} thread_data_t; +}; -/* Thread syncronization data */ -typedef struct thread_sync +/** Flush dirty pages when multi-threaded flush is used. */ +extern "C" UNIV_INTERN +os_thread_ret_t +DECLARE_THREAD(mtflush_io_thread)(void* arg); + +/** Thread syncronization data */ +struct thread_sync_t { + /** Constructor */ + thread_sync_t(ulint n_threads, mem_heap_t* wheap, mem_heap_t* rheap) : + thread_global_mtx(), n_threads(n_threads), + wq(ib_wqueue_create()), + wr_cq(ib_wqueue_create()), + rd_cq(ib_wqueue_create()), + wheap(wheap), rheap(rheap), gwt_status(), + thread_data(static_cast( + mem_heap_zalloc(wheap, n_threads + * sizeof *thread_data))) + { + ut_a(wq); + ut_a(wr_cq); + ut_a(rd_cq); + ut_a(thread_data); + + mutex_create(LATCH_ID_MTFLUSH_THREAD_MUTEX, + &thread_global_mtx); + + /* Create threads for page-compression-flush */ + for(ulint i = 0; i < n_threads; i++) { + thread_data[i].wt_status = WTHR_INITIALIZED; + os_thread_create(mtflush_io_thread, this, + &thread_data[i].wthread_id); + } + } + + /** Destructor */ + ~thread_sync_t() + { + ut_a(ib_wqueue_is_empty(wq)); + ut_a(ib_wqueue_is_empty(wr_cq)); + ut_a(ib_wqueue_is_empty(rd_cq)); + + /* Free all queues */ + ib_wqueue_free(wq); + ib_wqueue_free(wr_cq); + ib_wqueue_free(rd_cq); + + mutex_free(&thread_global_mtx); + + mem_heap_free(rheap); + mem_heap_free(wheap); + } + /* Global variables used by all threads */ ib_mutex_t thread_global_mtx; /*!< Mutex used protecting below variables */ @@ -142,23 +192,11 @@ typedef struct thread_sync /* Variables used by only one thread at a time */ thread_data_t* thread_data; /*!< Thread specific data */ +}; -} thread_sync_t; - -static int mtflush_work_initialized = -1; -static thread_sync_t* mtflush_ctx=NULL; +static thread_sync_t* mtflush_ctx; static ib_mutex_t mtflush_mtx; -/******************************************************************//** -Set multi-threaded flush work initialized. */ -static inline -void -buf_mtflu_work_init(void) -/*=====================*/ -{ - mtflush_work_initialized = 1; -} - /******************************************************************//** Return true if multi-threaded flush is initialized @return true if initialized */ @@ -166,7 +204,7 @@ bool buf_mtflu_init_done(void) /*=====================*/ { - return(mtflush_work_initialized == 1); + return(mtflush_ctx != NULL); } /******************************************************************//** @@ -307,15 +345,10 @@ mtflush_service_io( } } -/******************************************************************//** -Thead used to flush dirty pages when multi-threaded flush is -used. -@return a dummy parameter*/ +/** Flush dirty pages when multi-threaded flush is used. */ extern "C" UNIV_INTERN os_thread_ret_t -DECLARE_THREAD(mtflush_io_thread)( -/*==============================*/ - void * arg) +DECLARE_THREAD(mtflush_io_thread)(void* arg) { thread_sync_t *mtflush_io = ((thread_sync_t *)arg); thread_data_t *this_thread_data = NULL; @@ -438,29 +471,10 @@ buf_mtflu_io_thread_exit(void) ib_wqueue_nowait(mtflush_io->wq); } - mutex_enter(&mtflush_mtx); - - ut_a(ib_wqueue_is_empty(mtflush_io->wq)); - ut_a(ib_wqueue_is_empty(mtflush_io->wr_cq)); - ut_a(ib_wqueue_is_empty(mtflush_io->rd_cq)); - - /* Free all queues */ - ib_wqueue_free(mtflush_io->wq); - ib_wqueue_free(mtflush_io->wr_cq); - ib_wqueue_free(mtflush_io->rd_cq); + mtflush_ctx->~thread_sync_t(); + mtflush_ctx = NULL; - mtflush_io->wq = NULL; - mtflush_io->wr_cq = NULL; - mtflush_io->rd_cq = NULL; - mtflush_work_initialized = 0; - - /* Free heap */ - mem_heap_free(mtflush_io->wheap); - mem_heap_free(mtflush_io->rheap); - - mutex_exit(&mtflush_mtx); mutex_free(&mtflush_mtx); - mutex_free(&mtflush_io->thread_global_mtx); } /******************************************************************//** @@ -472,7 +486,6 @@ buf_mtflu_handler_init( ulint n_threads, /*!< in: Number of threads to create */ ulint wrk_cnt) /*!< in: Number of work items */ { - ulint i; mem_heap_t* mtflush_heap; mem_heap_t* mtflush_heap2; @@ -484,43 +497,10 @@ buf_mtflu_handler_init( mtflush_heap2 = mem_heap_create(0); ut_a(mtflush_heap2 != NULL); - mtflush_ctx = (thread_sync_t *)mem_heap_alloc(mtflush_heap, - sizeof(thread_sync_t)); - memset(mtflush_ctx, 0, sizeof(thread_sync_t)); - ut_a(mtflush_ctx != NULL); - mtflush_ctx->thread_data = (thread_data_t*)mem_heap_alloc( - mtflush_heap, sizeof(thread_data_t) * n_threads); - ut_a(mtflush_ctx->thread_data); - memset(mtflush_ctx->thread_data, 0, sizeof(thread_data_t) * n_threads); - - mtflush_ctx->n_threads = n_threads; - mtflush_ctx->wq = ib_wqueue_create(); - ut_a(mtflush_ctx->wq); - mtflush_ctx->wr_cq = ib_wqueue_create(); - ut_a(mtflush_ctx->wr_cq); - mtflush_ctx->rd_cq = ib_wqueue_create(); - ut_a(mtflush_ctx->rd_cq); - mtflush_ctx->wheap = mtflush_heap; - mtflush_ctx->rheap = mtflush_heap2; - - mutex_create(LATCH_ID_MTFLUSH_THREAD_MUTEX, &mtflush_ctx->thread_global_mtx); mutex_create(LATCH_ID_MTFLUSH_MUTEX, &mtflush_mtx); - /* Create threads for page-compression-flush */ - for(i=0; i < n_threads; i++) { - os_thread_id_t new_thread_id; - - mtflush_ctx->thread_data[i].wt_status = WTHR_INITIALIZED; - - os_thread_create( - mtflush_io_thread, - ((void *) mtflush_ctx), - &new_thread_id); - - mtflush_ctx->thread_data[i].wthread_id = new_thread_id; - } - - buf_mtflu_work_init(); + mtflush_ctx = new (mem_heap_zalloc(mtflush_heap, sizeof *mtflush_ctx)) + thread_sync_t(n_threads, mtflush_heap, mtflush_heap2); return((void *)mtflush_ctx); } diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 19318358dc479..a2865141a4d6f 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1693,7 +1693,7 @@ fil_space_create( DBUG_LOG("tablespace", "Tablespace for space " << id << " name " << name - << create_table ? " created" : " opened"); + << (create_table ? " created" : " opened")); if (crypt_data) { DBUG_LOG("crypt", "Tablespace " << id << " name " << name @@ -7731,4 +7731,4 @@ fil_system_exit(void) { ut_ad(mutex_own(&fil_system->mutex)); mutex_exit(&fil_system->mutex); -} \ No newline at end of file +} diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 3f2255d464441..0d216d584d74d 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -524,7 +524,6 @@ row_merge_fts_doc_tokenize( while (t_ctx->processed_len < doc->text.f_len) { ulint idx = 0; ib_uint32_t position; - ulint offset = 0; ulint cur_len; doc_id_t write_doc_id; row_fts_token_t* fts_token = NULL; From 30f27b0de040b0199d0c3b46a539b638970fa0f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jan 2017 11:54:10 +0200 Subject: [PATCH 30/74] Post-merge fix for MDEV-11638. logs_empty_and_mark_files_at_shutdown(): Wait for the log_scrub_thread to exit. --- storage/innobase/log/log0log.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 4a10379716b77..cf7825bd542aa 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -2230,7 +2230,7 @@ logs_empty_and_mark_files_at_shutdown(void) const ulint n_flush = log_sys->n_pending_flushes; log_mutex_exit(); - if (n_write != 0 || n_flush != 0) { + if (log_scrub_thread_active || n_write || n_flush) { if (srv_print_verbose_log && count > 600) { ib::info() << "Pending checkpoint_writes: " << n_write << ". Pending log flush writes: " << n_flush; From fb5ee7d6d043b01fabc59f09d70d532e843add60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jan 2017 19:01:14 +0200 Subject: [PATCH 31/74] Plug a memory leak in buf_dblwr_process(). --- storage/innobase/buf/buf0dblwr.cc | 3 ++- storage/xtradb/buf/buf0dblwr.cc | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index cc5fe52f80aaa..1cf856a731f5d 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -636,6 +636,7 @@ buf_dblwr_process() } } + ut_free(unaligned_read_buf); fil_flush_file_spaces(FIL_TABLESPACE); { diff --git a/storage/xtradb/buf/buf0dblwr.cc b/storage/xtradb/buf/buf0dblwr.cc index 62ed17296f5d1..7f6b6caee9d68 100644 --- a/storage/xtradb/buf/buf0dblwr.cc +++ b/storage/xtradb/buf/buf0dblwr.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2016, MariaDB Corporation. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -635,6 +635,7 @@ buf_dblwr_process() } } + ut_free(unaligned_read_buf); fil_flush_file_spaces(FIL_TABLESPACE); { From f0c19b6a57b699d113e3ae4a67920924bbecae45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 5 Jan 2017 20:13:34 +0200 Subject: [PATCH 32/74] MDEV-11730 Memory leak in innodb.innodb_corrupt_bit Memory was leaked when ALTER TABLE is attempted on a table that contains corrupted indexes. The memory leak was reported by AddressSanitizer for the test innodb.innodb_corrupt_bit. The leak was introduced into MariaDB Server 10.0.26, 10.1.15, 10.2.1 by the following: commit c081c978a2c83b9dc9efa84414cf40232460987d Merge: 1d21b221552 a482e76e65a Author: Sergei Golubchik Date: Tue Jun 21 14:11:02 2016 +0200 Merge branch '5.5' into bb-10.0 --- storage/innobase/handler/handler0alter.cc | 2 +- storage/xtradb/handler/handler0alter.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 3164879877677..89a58d83a60e0 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -3890,7 +3890,7 @@ ha_innobase::prepare_inplace_alter_table( index->name, TRUE); my_error(ER_INDEX_CORRUPT, MYF(0), index_name); - DBUG_RETURN(true); + goto err_exit; } } } diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index c5ac48dc4e376..7c17e9307eff9 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -3904,7 +3904,7 @@ ha_innobase::prepare_inplace_alter_table( index->name, TRUE); my_error(ER_INDEX_CORRUPT, MYF(0), index_name); - DBUG_RETURN(true); + goto err_exit; } } } From b2b6cf492e3c7fd99a2d0b6f152516efd28dadf4 Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Wed, 4 Jan 2017 19:11:13 +0200 Subject: [PATCH 33/74] MDEV-10988 Sphinx test suite refuses to run silently Add diagnostics output if any Sphinx components aren't found --- storage/sphinx/mysql-test/sphinx/suite.pm | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/storage/sphinx/mysql-test/sphinx/suite.pm b/storage/sphinx/mysql-test/sphinx/suite.pm index e4c3c1b9f74e6..fc127ffd6c036 100644 --- a/storage/sphinx/mysql-test/sphinx/suite.pm +++ b/storage/sphinx/mysql-test/sphinx/suite.pm @@ -16,13 +16,26 @@ sub locate_sphinx_binary { for (@list) { return $_ if -x $_; } } -# Look for Sphinx binaries. +# Look for Sphinx binaries my $exe_sphinx_indexer = &locate_sphinx_binary('indexer'); + +unless ($exe_sphinx_indexer) { + mtr_report("Sphinx 'indexer' binary not found, sphinx suite will be skipped"); + return "No Sphinx"; +} my $exe_sphinx_searchd = &locate_sphinx_binary('searchd'); -return "No Sphinx" unless $exe_sphinx_indexer and $exe_sphinx_searchd; -return "No SphinxSE" unless $ENV{HA_SPHINX_SO} or - $::mysqld_variables{'sphinx'} eq "ON"; +unless ($exe_sphinx_searchd) { + mtr_report("Sphinx 'searchd' binary not found, sphinx suite will be skipped"); + return "No Sphinx"; +} + +# Check for Sphinx engine + +unless ($ENV{HA_SPHINX_SO} or $::mysqld_variables{'sphinx'} eq "ON") { + mtr_report("Sphinx engine not found, sphinx suite will be skipped"); + return "No SphinxSE"; +} { local $_ = `"$exe_sphinx_searchd" --help`; From 670b85804ca6d3b0e640a7ddbb5b6e494bec7c2b Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Sun, 1 Jan 2017 15:36:56 +0200 Subject: [PATCH 34/74] Replication tests fail on valgrind due to waiting-related timeouts MTR raises default wait_for_pos_timeout from 300 to 1500 when tests are run with valgrind. The same needs to be done for other replication-related waits. The change should fix one of failures mentioned in MDEV-10653 (rpl.rpl_parallel fails in buildbot with timeout), the one on the valgrind builder; but not all of them --- mysql-test/include/sync_slave_sql_with_io.inc | 4 ++++ mysql-test/include/sync_with_master_gtid.inc | 4 ++++ mysql-test/include/wait_for_slave_param.inc | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/mysql-test/include/sync_slave_sql_with_io.inc b/mysql-test/include/sync_slave_sql_with_io.inc index 8048f7a177c20..9efede9a61f1d 100644 --- a/mysql-test/include/sync_slave_sql_with_io.inc +++ b/mysql-test/include/sync_slave_sql_with_io.inc @@ -26,6 +26,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 300; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1500; + } } --let $_master_log_file= query_get_value(SHOW SLAVE STATUS, Master_Log_File, 1) diff --git a/mysql-test/include/sync_with_master_gtid.inc b/mysql-test/include/sync_with_master_gtid.inc index 97ada8eea2956..777711b979c69 100644 --- a/mysql-test/include/sync_with_master_gtid.inc +++ b/mysql-test/include/sync_with_master_gtid.inc @@ -34,6 +34,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 120; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1200; + } } --let $_result= `SELECT master_gtid_wait('$master_pos', $_slave_timeout)` diff --git a/mysql-test/include/wait_for_slave_param.inc b/mysql-test/include/wait_for_slave_param.inc index d3f7ec56614be..25020d46ed966 100644 --- a/mysql-test/include/wait_for_slave_param.inc +++ b/mysql-test/include/wait_for_slave_param.inc @@ -50,6 +50,10 @@ let $_slave_timeout= $slave_timeout; if (!$_slave_timeout) { let $_slave_timeout= 300; + if ($VALGRIND_TEST) + { + let $_slave_timeout= 1500; + } } if ($slave_error_param == '') From 43378f367c5c01ebc214919006e2072fb2356724 Mon Sep 17 00:00:00 2001 From: Kristian Nielsen Date: Mon, 25 Jul 2016 13:07:50 +0200 Subject: [PATCH 35/74] MDEV-10271: Stopped SQL slave thread doesn't print a message to error log like IO thread does Make the slave SQL thread always output to the error log the message "Slave SQL thread exiting, replication stopped in ..." whenever it previously outputted "Slave SQL thread initialized, starting replication ...". Before this patch, it was somewhat inconsistent in which cases the message would be output and in which not, depending on the exact time and cause of the condition that caused the SQL thread to stop. --- sql/slave.cc | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/sql/slave.cc b/sql/slave.cc index 65bcdc48c6a69..c4817ef47942f 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -4574,11 +4574,11 @@ pthread_handler_t handle_slave_sql(void *arg) { rli->report(ERROR_LEVEL, ER_SLAVE_FATAL_ERROR, NULL, "Error initializing relay log position: %s", errmsg); - goto err; + goto err_before_start; } rli->reset_inuse_relaylog(); if (rli->alloc_inuse_relaylog(rli->group_relay_log_name)) - goto err; + goto err_before_start; strcpy(rli->future_event_master_log_name, rli->group_master_log_name); THD_CHECK_SENTRY(thd); @@ -4738,6 +4738,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, } } + err: if (mi->using_parallel()) rli->parallel.wait_for_done(thd, rli); @@ -4757,15 +4758,7 @@ log '%s' at position %s, relay log '%s' position: %s%s", RPL_LOG_NAME, tmp.c_ptr_safe()); } - err: - - /* - Once again, in case we aborted with an error and skipped the first one. - (We want the first one to be before the printout of stop position to - get the correct position printed.) - */ - if (mi->using_parallel()) - rli->parallel.wait_for_done(thd, rli); + err_before_start: /* Some events set some playgrounds, which won't be cleared because thread From e4978d26b79120c58706e57fc66e4de1ec4b230c Mon Sep 17 00:00:00 2001 From: Dmitry Lenev Date: Mon, 25 Jul 2016 16:06:52 +0300 Subject: [PATCH 36/74] MDEV-9084 Calling a stored function from a nested select from temporary table causes unpredictable behavior Cherry-pick: f4a0af070ce49abae60040f6f32e1074309c27fb Author: Dmitry Lenev Date: Mon Jul 25 16:06:52 2016 +0300 Fix for bug #16672723 "CAN'T FIND TEMPORARY TABLE". Attempt to execute prepared CREATE TABLE SELECT statement which used temporary table in the subquery in FROM clause and stored function failed with unwarranted ER_NO_SUCH_TABLE error. The same happened when such statement was used in stored procedure and this procedure was re-executed. The problem occurred because execution of such prepared statement/its re-execution as part of stored procedure incorrectly set Query_table_list::query_tables_own_last marker, indicating the last table which is directly used by statement. As result temporary table used in the subquery was treated as indirectly used/belonging to prelocking list and was not pre-opened by open_temporary_tables() call before statement execution. Thus causing ER_NO_SUCH_TABLE errors since our code assumes that temporary tables need to be correctly pre-opened before statement execution. This problem became visible only in version 5.6 after patches related to bug 11746602/27480 "EXTEND CREATE TEMPORARY TABLES PRIVILEGE TO ALLOW TEMP TABLE OPERATIONS" since they have introduced pre-opening of temporary tables for statements. Incorrect setting of Query_table_list::query_tables_own_last happened in LEX::first_lists_tables_same() method which is called by CREATE TABLE SELECT implementation as part of LEX::unlink_first_table(), which temporary excludes table list element for table being created from the query table list before handling SELECT part. LEX::first_lists_tables_same() tries to ensure that global table list of the statement starts with the first table list element from the first statement select. To do this it moves such table list element to the head of the global table list. If this table happens to be last directly-used table for the statement, query_tables_own_last marker is pointing to it. Since this marker was not updated when table list element was moved we ended up with all tables except the first table separated by it as if they were not directly used by statement (i.e. belonged to prelocked tables list). This fix changes code of LEX::first_lists_tables_same() to update query_tables_own_last marker in cases when it points to the table being moved. It is set to the table which precedes table being moved in this case. --- mysql-test/r/sp-prelocking.result | 23 +++++++++++++++++++++++ mysql-test/t/sp-prelocking.test | 30 ++++++++++++++++++++++++++++++ sql/sql_lex.cc | 3 +++ 3 files changed, 56 insertions(+) diff --git a/mysql-test/r/sp-prelocking.result b/mysql-test/r/sp-prelocking.result index 5b594e9ea558c..eb47cc21f41de 100644 --- a/mysql-test/r/sp-prelocking.result +++ b/mysql-test/r/sp-prelocking.result @@ -340,3 +340,26 @@ f1() DROP FUNCTION f1; DROP VIEW v1; DROP TABLE t1,t2; +# +# Bug #16672723 "CAN'T FIND TEMPORARY TABLE". +# +CREATE FUNCTION f1() RETURNS INT RETURN 1; +CREATE TEMPORARY TABLE tmp1(a INT); +PREPARE stmt1 FROM "CREATE TEMPORARY TABLE tmp2 AS SELECT b FROM (SELECT f1() AS b FROM tmp1) AS t"; +# The below statement failed before the fix. +EXECUTE stmt1; +DROP TEMPORARY TABLES tmp1, tmp2; +DEALLOCATE PREPARE stmt1; +DROP FUNCTION f1; +create procedure sp1() +begin +drop table if exists t1, t2; +create temporary table t1 select 1 v; +create table t2 (col varchar(45)) select distinct col from (select sf1() as col from t1) t; +end$$ +create function sf1() returns text return 'blah'; +call test.sp1(); +call test.sp1(); +drop procedure sp1; +drop function sf1; +drop table t2; diff --git a/mysql-test/t/sp-prelocking.test b/mysql-test/t/sp-prelocking.test index c1378d59196d6..38cbd5aa11078 100644 --- a/mysql-test/t/sp-prelocking.test +++ b/mysql-test/t/sp-prelocking.test @@ -414,3 +414,33 @@ SELECT f1(); DROP FUNCTION f1; DROP VIEW v1; DROP TABLE t1,t2; + +--echo # +--echo # Bug #16672723 "CAN'T FIND TEMPORARY TABLE". +--echo # +CREATE FUNCTION f1() RETURNS INT RETURN 1; +CREATE TEMPORARY TABLE tmp1(a INT); +PREPARE stmt1 FROM "CREATE TEMPORARY TABLE tmp2 AS SELECT b FROM (SELECT f1() AS b FROM tmp1) AS t"; +--echo # The below statement failed before the fix. +EXECUTE stmt1; +DROP TEMPORARY TABLES tmp1, tmp2; +DEALLOCATE PREPARE stmt1; +DROP FUNCTION f1; + +# +# MDEV-9084 Calling a stored function from a nested select from temporary table causes unpredictable behavior +# +delimiter $$; +create procedure sp1() +begin + drop table if exists t1, t2; + create temporary table t1 select 1 v; + create table t2 (col varchar(45)) select distinct col from (select sf1() as col from t1) t; +end$$ +delimiter ;$$ +create function sf1() returns text return 'blah'; +call test.sp1(); +call test.sp1(); +drop procedure sp1; +drop function sf1; +drop table t2; diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index d20c5ae78aff5..22489913ded42 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -3158,6 +3158,9 @@ void LEX::first_lists_tables_same() if (query_tables_last == &first_table->next_global) query_tables_last= first_table->prev_global; + if (query_tables_own_last == &first_table->next_global) + query_tables_own_last= first_table->prev_global; + if ((next= *first_table->prev_global= first_table->next_global)) next->prev_global= first_table->prev_global; /* include in new place */ From ac0b0efa14aa48c7229b5d6ce6b1ec3a3cf26eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 6 Jan 2017 14:42:28 +0200 Subject: [PATCH 37/74] Post-fix MDEV-11695 Define a reasonable upper limit for innodb_spin_wait_delay Adjust the tests. --- .../r/innodb_spin_wait_delay_basic.result | 46 ++++++++++++------- .../suite/sys_vars/r/sysvars_innodb.result | 2 +- .../t/innodb_spin_wait_delay_basic.test | 31 +++++-------- 3 files changed, 42 insertions(+), 37 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/innodb_spin_wait_delay_basic.result b/mysql-test/suite/sys_vars/r/innodb_spin_wait_delay_basic.result index 621ef56f61f53..88516a854fe3d 100644 --- a/mysql-test/suite/sys_vars/r/innodb_spin_wait_delay_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_spin_wait_delay_basic.result @@ -43,14 +43,40 @@ set global innodb_spin_wait_delay=0; select @@global.innodb_spin_wait_delay; @@global.innodb_spin_wait_delay 0 +set global innodb_spin_wait_delay=5000; +select @@global.innodb_spin_wait_delay; +@@global.innodb_spin_wait_delay +5000 set global innodb_spin_wait_delay=65535; +Warnings: +Warning 1292 Truncated incorrect innodb_spin_wait_delay value: '65535' select @@global.innodb_spin_wait_delay; @@global.innodb_spin_wait_delay -65535 +6000 set global innodb_spin_wait_delay=4294967295; +Warnings: +Warning 1292 Truncated incorrect innodb_spin_wait_delay value: '4294967295' +select @@global.innodb_spin_wait_delay; +@@global.innodb_spin_wait_delay +6000 +set @@global.innodb_spin_wait_delay = 4294967296; +Warnings: +Warning 1292 Truncated incorrect innodb_spin_wait_delay value: '4294967296' +select @@global.innodb_spin_wait_delay; +@@global.innodb_spin_wait_delay +6000 +set @@global.innodb_spin_wait_delay = 12345678901; +Warnings: +Warning 1292 Truncated incorrect innodb_spin_wait_delay value: '12345678901' +select @@global.innodb_spin_wait_delay; +@@global.innodb_spin_wait_delay +6000 +set @@global.innodb_spin_wait_delay = 18446744073709551615; +Warnings: +Warning 1292 Truncated incorrect innodb_spin_wait_delay value: '18446744073709551615' select @@global.innodb_spin_wait_delay; @@global.innodb_spin_wait_delay -4294967295 +6000 set global innodb_spin_wait_delay=1.1; ERROR 42000: Incorrect argument type to variable 'innodb_spin_wait_delay' set global innodb_spin_wait_delay=1e1; @@ -61,12 +87,12 @@ set global innodb_spin_wait_delay=' '; ERROR 42000: Incorrect argument type to variable 'innodb_spin_wait_delay' select @@global.innodb_spin_wait_delay; @@global.innodb_spin_wait_delay -4294967295 +6000 set global innodb_spin_wait_delay=" "; ERROR 42000: Incorrect argument type to variable 'innodb_spin_wait_delay' select @@global.innodb_spin_wait_delay; @@global.innodb_spin_wait_delay -4294967295 +6000 set global innodb_spin_wait_delay=-7; Warnings: Warning 1292 Truncated incorrect innodb_spin_wait_delay value: '-7' @@ -82,18 +108,6 @@ select @@global.innodb_spin_wait_delay; select * from information_schema.global_variables where variable_name='innodb_spin_wait_delay'; VARIABLE_NAME VARIABLE_VALUE INNODB_SPIN_WAIT_DELAY 0 -SET @@global.innodb_spin_wait_delay = 4294967296; -SELECT @@global.innodb_spin_wait_delay IN (4294967296,4294967295); -@@global.innodb_spin_wait_delay IN (4294967296,4294967295) -1 -SET @@global.innodb_spin_wait_delay = 12345678901; -SELECT @@global.innodb_spin_wait_delay IN (12345678901,4294967295); -@@global.innodb_spin_wait_delay IN (12345678901,4294967295) -1 -SET @@global.innodb_spin_wait_delay = 18446744073709551615; -SELECT @@global.innodb_spin_wait_delay IN (18446744073709551615,4294967295); -@@global.innodb_spin_wait_delay IN (18446744073709551615,4294967295) -1 SET @@global.innodb_spin_wait_delay = @start_global_value; SELECT @@global.innodb_spin_wait_delay; @@global.innodb_spin_wait_delay diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index ad6dcc1bb6438..d02dea3dd2d1b 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -2199,7 +2199,7 @@ VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BIGINT UNSIGNED VARIABLE_COMMENT Maximum delay between polling for a spin lock (6 by default) NUMERIC_MIN_VALUE 0 -NUMERIC_MAX_VALUE 18446744073709551615 +NUMERIC_MAX_VALUE 6000 NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO diff --git a/mysql-test/suite/sys_vars/t/innodb_spin_wait_delay_basic.test b/mysql-test/suite/sys_vars/t/innodb_spin_wait_delay_basic.test index ab0b38bb6cec3..d144d24a58b4b 100644 --- a/mysql-test/suite/sys_vars/t/innodb_spin_wait_delay_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_spin_wait_delay_basic.test @@ -46,10 +46,21 @@ select @@global.innodb_spin_wait_delay; # set global innodb_spin_wait_delay=0; select @@global.innodb_spin_wait_delay; +set global innodb_spin_wait_delay=5000; +select @@global.innodb_spin_wait_delay; +# +# invalid values +# set global innodb_spin_wait_delay=65535; select @@global.innodb_spin_wait_delay; set global innodb_spin_wait_delay=4294967295; select @@global.innodb_spin_wait_delay; +set @@global.innodb_spin_wait_delay = 4294967296; +select @@global.innodb_spin_wait_delay; +set @@global.innodb_spin_wait_delay = 12345678901; +select @@global.innodb_spin_wait_delay; +set @@global.innodb_spin_wait_delay = 18446744073709551615; +select @@global.innodb_spin_wait_delay; # # incorrect types @@ -74,26 +85,6 @@ select @@global.innodb_spin_wait_delay; select * from information_schema.global_variables where variable_name='innodb_spin_wait_delay'; --enable_warnings -# -# Check for out of bounds -# - -# With a 64 bit mysqld:18446744073709551615,with a 32 bit mysqld: 4294967295 ---disable_warnings -SET @@global.innodb_spin_wait_delay = 4294967296; ---enable_warnings -SELECT @@global.innodb_spin_wait_delay IN (4294967296,4294967295); - ---disable_warnings -SET @@global.innodb_spin_wait_delay = 12345678901; ---enable_warnings -SELECT @@global.innodb_spin_wait_delay IN (12345678901,4294967295); - ---disable_warnings -SET @@global.innodb_spin_wait_delay = 18446744073709551615; ---enable_warnings -SELECT @@global.innodb_spin_wait_delay IN (18446744073709551615,4294967295); - # # cleanup # From bbd4844a43fc61f207421dcb46299c4dcdce2ed7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 6 Jan 2017 14:52:35 +0200 Subject: [PATCH 38/74] Suppress warnings of NUMA not working. --- mysql-test/suite/sys_vars/r/innodb_numa_interleave_basic.result | 1 + mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test | 2 ++ 2 files changed, 3 insertions(+) diff --git a/mysql-test/suite/sys_vars/r/innodb_numa_interleave_basic.result b/mysql-test/suite/sys_vars/r/innodb_numa_interleave_basic.result index 21ed16c1dab8f..58f3b4b3e3870 100644 --- a/mysql-test/suite/sys_vars/r/innodb_numa_interleave_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_numa_interleave_basic.result @@ -1,3 +1,4 @@ +call mtr.add_suppression("InnoDB: Failed to set NUMA memory policy"); SELECT @@GLOBAL.innodb_numa_interleave; @@GLOBAL.innodb_numa_interleave 1 diff --git a/mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test b/mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test index de89784f7e4a6..fcbf766ac65ab 100644 --- a/mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test +++ b/mysql-test/suite/sys_vars/t/innodb_numa_interleave_basic.test @@ -1,6 +1,8 @@ --source include/have_innodb.inc --source include/have_numa.inc +call mtr.add_suppression("InnoDB: Failed to set NUMA memory policy"); + SELECT @@GLOBAL.innodb_numa_interleave; --error ER_INCORRECT_GLOBAL_LOCAL_VAR From ae6eb7a0020eb4d51f97f924e5320325e5b63934 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 4 Jan 2017 23:04:37 +0100 Subject: [PATCH 39/74] MDEV-11088 Client plugins cannot be loaded by command line tools in default installation. Added plugin-dir to the [client] section of the generated my.ini, so that installed services (MSI or mysql_install_db.exe) would be able to find plugin directory. --- sql/CMakeLists.txt | 1 + sql/mysql_install_db.cc | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 88ec26eb6f626..652664fa43818 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -387,6 +387,7 @@ IF(WIN32) ${CMAKE_CURRENT_BINARY_DIR}/mysql_bootstrap_sql.c COMPONENT Server ) + SET_TARGET_PROPERTIES(mysql_install_db PROPERTIES COMPILE_FLAGS -DINSTALL_PLUGINDIR=${INSTALL_PLUGINDIR}) TARGET_LINK_LIBRARIES(mysql_install_db mysys) ADD_LIBRARY(winservice STATIC winservice.c) diff --git a/sql/mysql_install_db.cc b/sql/mysql_install_db.cc index c39789f7c97a1..c23a20ebac991 100644 --- a/sql/mysql_install_db.cc +++ b/sql/mysql_install_db.cc @@ -233,6 +233,20 @@ static void get_basedir(char *basedir, int size, const char *mysqld_path) } } +#define STR(s) _STR(s) +#define _STR(s) #s + +static char *get_plugindir() +{ + static char plugin_dir[2*MAX_PATH]; + get_basedir(plugin_dir, sizeof(plugin_dir), mysqld_path); + strcat(plugin_dir, "/" STR(INSTALL_PLUGINDIR)); + + if (access(plugin_dir, 0) == 0) + return plugin_dir; + + return NULL; +} /** Allocate and initialize command line for mysqld --bootstrap. @@ -313,6 +327,10 @@ static int create_myini() fprintf(myini,"protocol=pipe\n"); else if (opt_port) fprintf(myini,"port=%d\n",opt_port); + + char *plugin_dir = get_plugindir(); + if (plugin_dir) + fprintf(myini, "plugin-dir=%s\n", plugin_dir); fclose(myini); return 0; } From 82b8741ad00e1ffdcbd0e2cf82f3ab31a7f3ce8b Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 4 Jan 2017 22:50:48 +0100 Subject: [PATCH 40/74] Windows : use meaningful DEFAULT_MYSQL_HOME - base directory for the default installation. It is now defined as "C:/Program Files/MariaDB ${MYSQL_BASE_VERSION}" which is where installer indeed puts it by default. It still does not cover every case -32bit installer on 64 bit Windows would put installation root under "C:/Program Files (x86)", but better than the path used previously C:/MariaDB${MYSQL_BASE_VERSION}, which was never correct. --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0adbb85d431a5..b6ffedabd149b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -303,7 +303,7 @@ ENDIF() # Set commonly used variables IF(WIN32) - SET(DEFAULT_MYSQL_HOME "C:/MariaDB${MYSQL_BASE_VERSION}") + SET(DEFAULT_MYSQL_HOME "C:/Program Files/MariaDB ${MYSQL_BASE_VERSION}") SET(SHAREDIR share) ELSE() SET(DEFAULT_MYSQL_HOME ${CMAKE_INSTALL_PREFIX}) From eaf6b053b83d2a46250834e8367477bd3d85d38f Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 4 Jan 2017 22:41:43 +0100 Subject: [PATCH 41/74] MDEV-11087 Search path for my.ini is wrong for default installation Add /data/my.ini to the search path - this my.ini location is used since MariaDB 5.2 --- mysys/my_default.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mysys/my_default.c b/mysys/my_default.c index 0f9b70ca32658..03cfa973b2c16 100644 --- a/mysys/my_default.c +++ b/mysys/my_default.c @@ -90,7 +90,7 @@ static my_bool defaults_already_read= FALSE; /* Which directories are searched for options (and in which order) */ -#define MAX_DEFAULT_DIRS 6 +#define MAX_DEFAULT_DIRS 7 #define DEFAULT_DIRS_SIZE (MAX_DEFAULT_DIRS + 1) /* Terminate with NULL */ static const char **default_directories = NULL; @@ -1219,7 +1219,12 @@ static const char **init_default_directories(MEM_ROOT *alloc) errors += add_directory(alloc, "C:/", dirs); if (my_get_module_parent(fname_buffer, sizeof(fname_buffer)) != NULL) + { + errors += add_directory(alloc, fname_buffer, dirs); + + strncat(fname_buffer, "/data", sizeof(fname_buffer)); errors += add_directory(alloc, fname_buffer, dirs); + } } #else From 384f4d1e36cf2ba8791b15c83ac601b98f3270c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 7 Jan 2017 15:27:59 +0200 Subject: [PATCH 42/74] Post-push fix for MDEV-11556: Make the debug variable UINT. Sometimes innodb_data_file_size_debug was reported as INT UNSIGNED instead of BIGINT UNSIGNED. Make it uint instead of ulong to get a more deterministic result. --- mysql-test/suite/sys_vars/r/sysvars_innodb.result | 2 +- storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/include/srv0srv.h | 2 +- storage/innobase/srv/srv0start.cc | 2 +- storage/xtradb/handler/ha_innodb.cc | 2 +- storage/xtradb/include/srv0srv.h | 2 +- storage/xtradb/srv/srv0start.cc | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 5274523e3a41b..d3e8977c6bbe3 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -585,7 +585,7 @@ GLOBAL_VALUE 0 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT InnoDB system tablespace size to be set in recovery. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 625ae5f5d2760..227018fd3b6f3 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -19691,7 +19691,7 @@ static MYSQL_SYSVAR_BOOL(trx_purge_view_update_only_debug, "but the each purges were not done yet.", NULL, NULL, FALSE); -static MYSQL_SYSVAR_ULONG(data_file_size_debug, +static MYSQL_SYSVAR_UINT(data_file_size_debug, srv_sys_space_size_debug, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "InnoDB system tablespace size to be set in recovery.", diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 1ba1e3ae9c65d..f404668390f15 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -525,7 +525,7 @@ extern my_bool srv_ibuf_disable_background_merge; #ifdef UNIV_DEBUG extern my_bool srv_purge_view_update_only_debug; -extern ulong srv_sys_space_size_debug; +extern uint srv_sys_space_size_debug; #endif /* UNIV_DEBUG */ #define SRV_SEMAPHORE_WAIT_EXTENSION 7200 diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index ffd97b6109158..40154c11ce11c 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -129,7 +129,7 @@ UNIV_INTERN ibool srv_was_started = FALSE; static ibool srv_start_has_been_called = FALSE; #ifdef UNIV_DEBUG /** InnoDB system tablespace to set during recovery */ -UNIV_INTERN ulong srv_sys_space_size_debug; +UNIV_INTERN uint srv_sys_space_size_debug; #endif /* UNIV_DEBUG */ /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index a361ab17a7caf..a3f1d9c3c85b5 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -21172,7 +21172,7 @@ static MYSQL_SYSVAR_BOOL(trx_purge_view_update_only_debug, "but the each purges were not done yet.", NULL, NULL, FALSE); -static MYSQL_SYSVAR_ULONG(data_file_size_debug, +static MYSQL_SYSVAR_UINT(data_file_size_debug, srv_sys_space_size_debug, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "InnoDB system tablespace size to be set in recovery.", diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index f3b6098a875a1..f65848fdf452d 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -626,7 +626,7 @@ extern my_bool srv_ibuf_disable_background_merge; #ifdef UNIV_DEBUG extern my_bool srv_purge_view_update_only_debug; -extern ulong srv_sys_space_size_debug; +extern uint srv_sys_space_size_debug; #endif /* UNIV_DEBUG */ #define SRV_SEMAPHORE_WAIT_EXTENSION 7200 diff --git a/storage/xtradb/srv/srv0start.cc b/storage/xtradb/srv/srv0start.cc index 31421c7a605ec..77cdfe1affbc7 100644 --- a/storage/xtradb/srv/srv0start.cc +++ b/storage/xtradb/srv/srv0start.cc @@ -132,7 +132,7 @@ UNIV_INTERN ibool srv_was_started = FALSE; static ibool srv_start_has_been_called = FALSE; #ifdef UNIV_DEBUG /** InnoDB system tablespace to set during recovery */ -UNIV_INTERN ulong srv_sys_space_size_debug; +UNIV_INTERN uint srv_sys_space_size_debug; #endif /* UNIV_DEBUG */ /** At a shutdown this value climbs from SRV_SHUTDOWN_NONE to From 8773a5e16199269be090cbed13ff892bac810cef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 7 Jan 2017 15:36:08 +0200 Subject: [PATCH 43/74] Post-fix MDEV-11695 Define a reasonable upper limit for innodb_spin_wait_delay Change the parameter type from ulong to uint, so that 32-bit and 64-bit systems will report an identical result. --- mysql-test/suite/sys_vars/r/sysvars_innodb.result | 2 +- storage/innobase/handler/ha_innodb.cc | 2 +- storage/innobase/include/srv0srv.h | 2 +- storage/innobase/include/ut0mutex.h | 4 ++-- storage/innobase/srv/srv0srv.cc | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index 2365f36ee376b..de2295859c3eb 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -2196,7 +2196,7 @@ GLOBAL_VALUE 6 GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 6 VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE BIGINT UNSIGNED +VARIABLE_TYPE INT UNSIGNED VARIABLE_COMMENT Maximum delay between polling for a spin lock (6 by default) NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 6000 diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index f0e5edb5704a7..ae9363d170935 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -22611,7 +22611,7 @@ static MYSQL_SYSVAR_ULONG(sync_spin_loops, srv_n_spin_wait_rounds, "Count of spin-loop rounds in InnoDB mutexes (30 by default)", NULL, NULL, 30L, 0L, ~0UL, 0); -static MYSQL_SYSVAR_ULONG(spin_wait_delay, srv_spin_wait_delay, +static MYSQL_SYSVAR_UINT(spin_wait_delay, srv_spin_wait_delay, PLUGIN_VAR_OPCMDARG, "Maximum delay between polling for a spin lock (6 by default)", NULL, NULL, 6, 0, 6000, 0); diff --git a/storage/innobase/include/srv0srv.h b/storage/innobase/include/srv0srv.h index 1c546dc2a067a..1e05c2c6e4a88 100644 --- a/storage/innobase/include/srv0srv.h +++ b/storage/innobase/include/srv0srv.h @@ -522,7 +522,7 @@ extern my_bool srv_scrub_log; extern ulong srv_n_spin_wait_rounds; extern ulong srv_n_free_tickets_to_enter; extern ulong srv_thread_sleep_delay; -extern ulong srv_spin_wait_delay; +extern uint srv_spin_wait_delay; extern ibool srv_priority_boost; extern ulint srv_truncated_status_writes; diff --git a/storage/innobase/include/ut0mutex.h b/storage/innobase/include/ut0mutex.h index e4ab671eecee1..197a34e61229d 100644 --- a/storage/innobase/include/ut0mutex.h +++ b/storage/innobase/include/ut0mutex.h @@ -28,7 +28,7 @@ Created 2012-03-24 Sunny Bains. #ifndef ut0mutex_h #define ut0mutex_h -extern ulong srv_spin_wait_delay; +extern uint srv_spin_wait_delay; extern ulong srv_n_spin_wait_rounds; extern ulong srv_force_recovery_crash; @@ -75,7 +75,7 @@ typedef BlockSyncArrayMutex ib_bpmutex_t; #error "ib_mutex_t type is unknown" #endif /* MUTEX_FUTEX */ -extern ulong srv_spin_wait_delay; +extern uint srv_spin_wait_delay; extern ulong srv_n_spin_wait_rounds; #define mutex_create(I, M) mutex_init((M), (I), __FILE__, __LINE__) diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index ab9f52b659612..002477bc4e0d3 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -418,7 +418,7 @@ ulong srv_replication_delay = 0; /*-------------------------------------------*/ UNIV_INTERN ulong srv_n_spin_wait_rounds = 15; -ulong srv_spin_wait_delay = 6; +uint srv_spin_wait_delay; ibool srv_priority_boost = TRUE; static ulint srv_n_rows_inserted_old = 0; From eed319b6fb543849046c8009c38575455e173dc2 Mon Sep 17 00:00:00 2001 From: Monty Date: Sun, 8 Jan 2017 17:51:36 +0200 Subject: [PATCH 44/74] MDEV-11317: `! is_set()' or `!is_set() || (m_status == DA_OK_BULK && is_bulk_op())' fails in Diagnostics_area::set_ok_status on CREATE OR REPLACE with ARCHIVE table Problem was with deleting non existing .frm file for a storage engine that doesn't have .frm files (yet) Fixed by not giving an error for non existing .frm files for storage engines that are using discovery Fixed also valgrind supression related to the given test case --- mysql-test/suite/archive/discover.result | 7 +++++++ mysql-test/suite/archive/discover.test | 10 ++++++++++ mysql-test/valgrind.supp | 15 ++++++++++++++- sql/sql_table.cc | 16 ++++++++++++++-- 4 files changed, 45 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/archive/discover.result b/mysql-test/suite/archive/discover.result index e1ca9cb6a656d..0619ca2051a0f 100644 --- a/mysql-test/suite/archive/discover.result +++ b/mysql-test/suite/archive/discover.result @@ -139,3 +139,10 @@ flush tables; create table t1 (a int) engine=archive; ERROR 42S01: Table 't1' already exists drop table t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +DROP TABLE t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +SELECT * FROM t1; +pk +DROP TABLE t1; diff --git a/mysql-test/suite/archive/discover.test b/mysql-test/suite/archive/discover.test index 20cb69efa005e..4ab35cf11151f 100644 --- a/mysql-test/suite/archive/discover.test +++ b/mysql-test/suite/archive/discover.test @@ -132,3 +132,13 @@ flush tables; create table t1 (a int) engine=archive; drop table t1; +# +# MDEV-11317: Error in deleting non existing .frm for tables with disocvery +# + +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +DROP TABLE t1; +CREATE OR REPLACE TABLE t1 ( pk INT AUTO_INCREMENT PRIMARY KEY ) ENGINE = ARCHIVE; +SELECT * FROM t1; +DROP TABLE t1; diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 77f17cf07ec0c..a7d7f2ee67a38 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -353,10 +353,23 @@ Memcheck:Leak fun:memalign ... - fun:call_init + fun:call_init* + fun:_dl_init +} + +# This one is on OpenSuse 10.3 with gcc 5.4 +{ + memory "loss" from _dl_init 2 + Memcheck:Leak + fun:malloc + fun:pool + ... + fun:call_init* fun:_dl_init } + + # # dlclose can allocate memory for error message, the memory will be # freed by dlerror or other dl* function. diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 62dff1b592889..8569771d5f1db 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -2497,7 +2497,19 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, int frm_delete_error, trigger_drop_error= 0; /* Delete the table definition file */ strmov(end,reg_ext); - frm_delete_error= mysql_file_delete(key_file_frm, path, MYF(MY_WME)); + if (table_type && table_type != view_pseudo_hton && + table_type->discover_table) + { + /* + Table type is using discovery and may not need a .frm file. + Delete it silently if it exists + */ + (void) mysql_file_delete(key_file_frm, path, MYF(0)); + frm_delete_error= 0; + } + else + frm_delete_error= mysql_file_delete(key_file_frm, path, + MYF(MY_WME)); if (frm_delete_error) frm_delete_error= my_errno; else @@ -2513,7 +2525,7 @@ int mysql_rm_table_no_locks(THD *thd, TABLE_LIST *tables, bool if_exists, else if (frm_delete_error && if_exists) thd->clear_error(); } - non_tmp_error= error ? TRUE : non_tmp_error; + non_tmp_error|= MY_TEST(error); } if (error) { From 59ea6456c31e47dd0a2f65c28f6fb97d09f6ee7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 9 Jan 2017 09:12:32 +0200 Subject: [PATCH 45/74] Minor cleanup of innodb.innodb-change-buffer-recovery This should be a non-functional change. I was unable to repeat MDEV-11626 innodb.innodb-change-buffer-recovery fails for xtradb and cannot determine the reason for the failure without having access to the files. The repeatability of MDEV-11626 should not be affected by these changes. --- .../r/innodb-change-buffer-recovery.result | 2 +- .../t/innodb-change-buffer-recovery.test | 26 +++++++------------ 2 files changed, 10 insertions(+), 18 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result b/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result index 96d2a3d462cbd..5b95742d0e8c6 100644 --- a/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result +++ b/mysql-test/suite/innodb/r/innodb-change-buffer-recovery.result @@ -8,7 +8,7 @@ a INT AUTO_INCREMENT PRIMARY KEY, b CHAR(1), c INT, INDEX(b)) -ENGINE=InnoDB; +ENGINE=InnoDB STATS_PERSISTENT=0; INSERT INTO t1 VALUES(0,'x',1); INSERT INTO t1 SELECT 0,b,c FROM t1; INSERT INTO t1 SELECT 0,b,c FROM t1; diff --git a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test index 79f7999d11542..18d7aa3be5850 100644 --- a/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test +++ b/mysql-test/suite/innodb/t/innodb-change-buffer-recovery.test @@ -1,8 +1,3 @@ -if (`select plugin_auth_version < "5.6.17" from information_schema.plugins where plugin_name='innodb'`) -{ - --skip Not fixed in InnoDB before 5.6.17 -} - --echo # --echo # Bug#69122 - INNODB DOESN'T REDO-LOG INSERT BUFFER MERGE --echo # OPERATION IF IT IS DONE IN-PLACE @@ -14,8 +9,9 @@ if (`select plugin_auth_version < "5.6.17" from information_schema.plugins where --source include/not_embedded.inc # DBUG_SUICIDE() hangs under valgrind --source include/not_valgrind.inc -# No windows, need perl ---source include/not_windows.inc + +--let $_server_id= `SELECT @@server_id` +--let $_expect_file_name= $MYSQLTEST_VARDIR/tmp/mysqld.$_server_id.expect # The flag innodb_change_buffering_debug is only available in debug builds. # It instructs InnoDB to try to evict pages from the buffer pool when @@ -29,7 +25,7 @@ CREATE TABLE t1( b CHAR(1), c INT, INDEX(b)) -ENGINE=InnoDB; +ENGINE=InnoDB STATS_PERSISTENT=0; # Create enough rows for the table, so that the change buffer will be # used for modifying the secondary index page. There must be multiple @@ -54,27 +50,23 @@ BEGIN; SELECT b FROM t1 LIMIT 3; connect (con1,localhost,root,,); -connection con1; BEGIN; DELETE FROM t1 WHERE a=1; # This should be buffered, if innodb_change_buffering_debug = 1 is in effect. INSERT INTO t1 VALUES(1,'X',1); SET DEBUG_DBUG='+d,crash_after_log_ibuf_upd_inplace'; ---exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect + +--exec echo "wait" > $_expect_file_name --error 2013 # This should force a change buffer merge SELECT b FROM t1 LIMIT 3; +disconnect con1; +connection default; let SEARCH_PATTERN=Wrote log record for ibuf update in place operation; --source include/search_pattern_in_file.inc - -# Write file to make mysql-test-run.pl start up the server again ---exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect ---enable_reconnect ---source include/wait_until_connected_again.inc +--source include/start_mysqld.inc CHECK TABLE t1; - -# Cleanup DROP TABLE t1; From 4b05d60e62ef6e21d5329a9667813df890034ff0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 9 Jan 2017 09:15:21 +0200 Subject: [PATCH 46/74] Make encryption.innodb_lotoftables more robust. Perform a slow shutdown at the start of the test, and create all InnoDB tables with STATS_PERSISTENT=0, so that any I/O related to background tasks (change buffer merge, purge, persistent statistics) should be eliminated. --- .../encryption/r/innodb_lotoftables.result | 29 ++++++++++--------- .../encryption/t/innodb_lotoftables.test | 13 +++++++-- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_lotoftables.result b/mysql-test/suite/encryption/r/innodb_lotoftables.result index 34f2684253e34..b7cfdd2db9df6 100644 --- a/mysql-test/suite/encryption/r/innodb_lotoftables.result +++ b/mysql-test/suite/encryption/r/innodb_lotoftables.result @@ -1,3 +1,4 @@ +SET GLOBAL innodb_fast_shutdown=0; SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; SHOW VARIABLES LIKE 'innodb_encrypt%'; @@ -11,13 +12,13 @@ create database innodb_encrypted_1; use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 set autocommit=0; set autocommit=1; commit work; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 # should be 100 SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE NAME LIKE 'innodb_encrypted%'; COUNT(*) @@ -87,47 +88,47 @@ Innodb_pages0_read 3 # Restart Success! show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 use test; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 use innodb_encrypted_2; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 use innodb_encrypted_3; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 use innodb_encrypted_1; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 3 +Innodb_pages0_read 1 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 103 +Innodb_pages0_read 101 use innodb_encrypted_2; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 103 +Innodb_pages0_read 101 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 203 +Innodb_pages0_read 201 use innodb_encrypted_3; show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 203 +Innodb_pages0_read 201 show status like 'innodb_pages0_read%'; Variable_name Value -Innodb_pages0_read 303 +Innodb_pages0_read 301 SELECT COUNT(*) FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION WHERE MIN_KEY_VERSION = 0 AND NAME LIKE 'innodb_encrypted%'; COUNT(*) 100 diff --git a/mysql-test/suite/encryption/t/innodb_lotoftables.test b/mysql-test/suite/encryption/t/innodb_lotoftables.test index cad3cb54326e3..8bad356a9e0aa 100644 --- a/mysql-test/suite/encryption/t/innodb_lotoftables.test +++ b/mysql-test/suite/encryption/t/innodb_lotoftables.test @@ -11,6 +11,10 @@ let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; let $innodb_encryption_threads_orig = `SELECT @@global.innodb_encryption_threads`; --enable_query_log +# empty the change buffer and the undo logs to avoid extra reads +SET GLOBAL innodb_fast_shutdown=0; +--source include/restart_mysqld.inc + SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; @@ -29,7 +33,8 @@ let $tables = 100; --disable_query_log while ($tables) { - eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb; + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb + stats_persistent=0; commit; let $rows = 100; while($rows) @@ -64,7 +69,8 @@ set autocommit=0; let $tables = 100; while ($tables) { - eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb encrypted=yes; + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb + stats_persistent=0 encrypted=yes; commit; let $rows = 100; while($rows) @@ -100,7 +106,8 @@ set autocommit=0; let $tables = 100; while ($tables) { - eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb encrypted=no; + eval create table t_$tables (a int not null primary key, b varchar(200)) engine=innodb + stats_persistent=0 encrypted=no; commit; let $rows = 100; while($rows) From 171e59ed479ee347dae04a34855ea3c29b13349d Mon Sep 17 00:00:00 2001 From: Alexey Botchkov Date: Mon, 9 Jan 2017 23:37:42 +0400 Subject: [PATCH 47/74] MDEV-11548 Reproducible server crash after the 2nd ALTER TABLE ADD FOREIGN KEY IF NOT EXISTS. Have to use 'keyname' to check the name uniqueness. --- mysql-test/r/alter_table.result | 15 +++++++++++++++ mysql-test/t/alter_table.test | 19 +++++++++++++++++++ sql/sql_table.cc | 2 +- 3 files changed, 35 insertions(+), 1 deletion(-) diff --git a/mysql-test/r/alter_table.result b/mysql-test/r/alter_table.result index 67de5f3160aca..72e81895816be 100644 --- a/mysql-test/r/alter_table.result +++ b/mysql-test/r/alter_table.result @@ -2076,3 +2076,18 @@ tab1 CREATE TABLE `tab1` ( PRIMARY KEY (`id`) ) ENGINE=MyISAM DEFAULT CHARSET=latin1 DROP TABLE `tab1`; +# +# MDEV-11548 Reproducible server crash after the 2nd ALTER TABLE ADD FOREIGN KEY IF NOT EXISTS +# +CREATE TABLE t1 (id INT UNSIGNED NOT NULL PRIMARY KEY); +CREATE TABLE t2 (id1 INT UNSIGNED NOT NULL); +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); +Warnings: +Note 1061 Duplicate key name 'id1' +DROP TABLE t2; +DROP TABLE t1; diff --git a/mysql-test/t/alter_table.test b/mysql-test/t/alter_table.test index d2b8a6082a623..6fe0f4d30ee57 100644 --- a/mysql-test/t/alter_table.test +++ b/mysql-test/t/alter_table.test @@ -1737,3 +1737,22 @@ SHOW CREATE TABLE `tab1`; ALTER TABLE `tab1` CHANGE COLUMN v_col `v_col` varchar(128) AS (IF(field11='option1',CONCAT_WS(":","field1",field2,field3,field4,field5,field6,field7,field8,field9,field10), CONCAT_WS(":","field1",field11,field2,field3,field4,field5,field6,field7,field8,field9,field10))) PERSISTENT; SHOW CREATE TABLE `tab1`; DROP TABLE `tab1`; + +--echo # +--echo # MDEV-11548 Reproducible server crash after the 2nd ALTER TABLE ADD FOREIGN KEY IF NOT EXISTS +--echo # + +CREATE TABLE t1 (id INT UNSIGNED NOT NULL PRIMARY KEY); +CREATE TABLE t2 (id1 INT UNSIGNED NOT NULL); + +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) + REFERENCES t1 (id); + +ALTER TABLE t2 +ADD FOREIGN KEY IF NOT EXISTS (id1) +REFERENCES t1 (id); + +DROP TABLE t2; +DROP TABLE t1; + diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 8569771d5f1db..31f61301c375e 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -5943,7 +5943,7 @@ handle_if_exists_options(THD *thd, TABLE *table, Alter_info *alter_info) while ((f_key= fk_key_it++)) { if (my_strcasecmp(system_charset_info, f_key->foreign_id->str, - key->name.str) == 0) + keyname) == 0) goto remove_key; } } From 78e6fafcaa93bdd2cf793a82a812137eb7a779a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 10 Jan 2017 14:11:32 +0200 Subject: [PATCH 48/74] Fix an innodb_plugin leak noted in MDEV-11686 buf_flush_init_flush_rbt() was called too early in MariaDB server 10.0, 10.1, MySQL 5.5 and MySQL 5.6. The memory leak has been fixed in the XtraDB storage engine and in MySQL 5.7. As a result, when the server is started to initialize new data files, the buf_pool->flush_rbt will be created unnecessarily and then leaked. This memory leak was noticed in MariaDB server 10.1 when running the test encryption.innodb_first_page. --- storage/innobase/log/log0recv.cc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 85f4f6ea6715d..aed94d0083456 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2,6 +2,7 @@ Copyright (c) 1997, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2017, MariaDB Corporation. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -387,12 +388,6 @@ recv_sys_init( } #ifndef UNIV_HOTBACKUP - /* Initialize red-black tree for fast insertions into the - flush_list during recovery process. - As this initialization is done while holding the buffer pool - mutex we perform it before acquiring recv_sys->mutex. */ - buf_flush_init_flush_rbt(); - mutex_enter(&(recv_sys->mutex)); recv_sys->heap = mem_heap_create_typed(256, @@ -3030,6 +3025,11 @@ recv_recovery_from_checkpoint_start_func( byte* buf; byte log_hdr_buf[LOG_FILE_HDR_SIZE]; dberr_t err; + + /* Initialize red-black tree for fast insertions into the + flush_list during recovery process. */ + buf_flush_init_flush_rbt(); + ut_when_dtor tmp(recv_sys->dblwr); #ifdef UNIV_LOG_ARCHIVE From f516db3500c0c20ff119203308cfc95e8752d62d Mon Sep 17 00:00:00 2001 From: Elena Stepanova Date: Wed, 11 Jan 2017 04:45:47 +0200 Subject: [PATCH 49/74] Updated list of unstable tests for 10.0.29 release --- mysql-test/unstable-tests | 147 +++++++++++++++++++++++--------------- 1 file changed, 91 insertions(+), 56 deletions(-) diff --git a/mysql-test/unstable-tests b/mysql-test/unstable-tests index 3e25115599fdc..e4c57d9f9d7c2 100644 --- a/mysql-test/unstable-tests +++ b/mysql-test/unstable-tests @@ -23,47 +23,65 @@ # ############################################################################## +main.count_distinct2 : MDEV-11768 - timeout +main.create : Modified in 10.0.29 main.create_delayed : MDEV-10605 - failed with timeout -main.ctype_utf32 : Modified on 2016-09-27 (merge) -main.func_group : Modified on 2016-08-08 (MDEV-10468) -main.func_math : Modified on 2016-08-10 (merge) -main.func_misc : Modified on 2016-08-10 (merge) -main.group_min_max_innodb : Modified on 2016-08-25 (MDEV-10595) +main.ctype_ucs2_def : Modified in 10.0.29 +main.ctype_ucs2_query_cache : Modified in 10.0.29 +main.ctype_utf16_def : Modified in 10.0.29 +main.ctype_utf8 : Modified in 10.0.29 +main.ctype_utf8mb4 : Modified in 10.0.29 +main.ctype_utf8mb4 : Modified in 10.0.29 (load file changed) +main.debug_sync : MDEV-10607 - internal error +main.default : Modified in 10.0.29 +main.derived : Modified in 10.0.29 +main.derived_opt : MDEV-11768 - timeout +main.derived_view : Modified in 10.0.29 +main.events_restart : MDEV-11221 - Assertion failure +main.events_slowlog : Added in 10.0.29 +main.fulltext_charsets : Added in 10.0.29 +main.func_time : Modified in 10.0.29 +main.group_by : Modified in 10.0.29 +main.group_by_innodb : Modified in 10.0.29 main.host_cache_size_functionality : MDEV-10606 - sporadic failure on shutdown main.index_intersect_innodb : MDEV-10643 - failed with timeout -main.index_merge_myisam : Modified on 2016-09-05 (include file changed) -main.index_merge_innodb : Modified on 2016-09-05 (MDEV-7142) -main.information_schema_stats : Modified on 2016-07-25 (MDEV-10428) +main.index_merge_innodb : MDEV-7142 - wrong result +main.information_schema_part : Modified in 10.0.29 main.innodb_mysql_lock : MDEV-7861 - sporadic lock detection failure -main.loaddata : Modified on 2016-08-10 (merge) +main.join_cache : Modified in 10.0.29 +main.loaddata : Modified in 10.0.29 +main.log_slow : Modified in 10.0.29 main.mdev-504 : MDEV-10607 - sporadic "can't connect" main.mdev375 : MDEV-10607 - sporadic "can't connect" main.merge : MDEV-10607 - sporadic "can't connect" -main.myisam_enable_keys-10506 : New test, added on 2016-08-10 (MDEV-10506) -main.mysqlcheck : Modified on 2016-08-10 (merge) -main.mysqldump : MDEV-10512 - sporadic assertion failure main.mysqlhotcopy_myisam : MDEV-10995 - test hangs on debug build main.mysqltest : MDEV-9269 - fails on Alpha -main.named_pipe : Modified on 2016-08-02 (MDEV-10383) -main.pool_of_threads : MDEV-10100 - sporadic error on detecting max connections +main.order_by : Modified in 10.0.29 +main.parser : Modified in 10.0.29 +main.pool_of_threads : Modified in 10.0.29 main.ps : MDEV-11017 - sporadic wrong Prepared_stmt_count -main.range : Modified on 2016-08-10 (merge) -main.range_mrr_icp : Modified on 2016-08-10 (merge) -main.query_cache : MDEV-10611 - sporadic mutex problem -main.shutdown : MDEV-10563 - sporadic crashes -main.sp-prelocking : Modified on 2016-08-10 (merge) +main.selectivity : Modified in 10.0.29 +main.signal_demo3 : MDEV-11720 - Thread stack overrun on Solaris +main.sp : Modified in 10.0.29 +main.sp_notembedded : MDEV-10607 - internal error +main.sp-prelocking : Modified in 10.0.29 main.sp-security : MDEV-10607 - sporadic "can't connect" -main.ssl_compress : MDEV-11110 - valgrind failures main.stat_tables_par_innodb : MDEV-10515 - sporadic wrong results +main.subselect : Modified in 10.0.29 +main.subselect2 : Modified in 10.0.29 +main.subselect4 : Modified in 10.0.29 main.subselect_innodb : MDEV-10614 - sporadic wrong results -main.type_date : Modified on 2016-08-10 (merge) -main.type_uint : Modified on 2016-09-27 (merge) -main.view : Modified on 2016-08-10 (merge) -main.xtradb_mrr : Modified on 2016-08-04 (MDEV-9946) +main.subselect_sj2_jcl6 : MDEV-11766 - unexpected warnings +main.type_bit_innodb : MDEV-11766 - unexpected warnings +main.type_decimal : Modified in 10.0.29 +main.union : Modified in 10.0.29 +main.view : Modified in 10.0.29 +main.xa : MDEV-11769 - lock wait timeout #---------------------------------------------------------------- archive.archive-big : MDEV-10615 - table is marked as crashed +archive.archive_bitfield : MDEV-11771 - table is marked as crashed archive.discover : MDEV-10510 - table is marked as crashed archive.mysqlhotcopy_archive : MDEV-10995 - test hangs on debug build @@ -74,6 +92,8 @@ binlog.binlog_xa_recover : MDEV-8517 - Extra checkpoint #---------------------------------------------------------------- +connect.jdbc : re-enabled in 10.0.29 +connect.jdbc_new : re-enabled in 10.0.29 connect.tbl : MDEV-9844, MDEV-10179 - sporadic crashes, valgrind warnings, wrong results #---------------------------------------------------------------- @@ -82,10 +102,6 @@ engines/rr_trx.* : MDEV-10998 - tests not maintained #---------------------------------------------------------------- -extra/binlog_tests.database : Modified on 2016-10-21 (Upstream MIPS test fixes) - -#---------------------------------------------------------------- - federated.federatedx : MDEV-10617 - Wrong checksum, timeouts federated.federated_innodb : MDEV-10617, MDEV-10417 - Wrong checksum, timeouts, fails on Mips federated.federated_partition : MDEV-10417 - Fails on Mips @@ -93,19 +109,23 @@ federated.federated_transactions : MDEV-10617, MDEV-10417 - Wrong checksum, time #---------------------------------------------------------------- -funcs_2/charset.* : MDEV-10999 - test not maintained +funcs_1.memory_views : MDEV-11773 - timeout +funcs_2/charset.* : MDEV-10999 - test not maintained #---------------------------------------------------------------- innodb.binlog_consistent : MDEV-10618 - Server fails to start +innodb.group_commit_crash_no_optimize_thread : MDEV-11770 - checksum mismatch innodb.innodb-alter-table : MDEV-10619 - Testcase timeout -innodb.innodb-alter-tempfile : Modified on 2016-08-09 (MDEV-10469) innodb.innodb_bug30423 : MDEV-7311 - Wrong number of rows in the plan -innodb.innodb_bug54044 : Modified on 2016-09-27 (merge) +innodb.innodb_bug53290 : MDEV-11767 - timeout +innodb.innodb_bug56143 : MDEV-11766 - unexpected warnings innodb.innodb_monitor : MDEV-10939 - Testcase timeout -innodb.innodb-wl5522 : rdiff file modified on 2016-08-10 (merge) -innodb.innodb-wl5522-debug-zip : MDEV-10427 - Warning: database page corruption -innodb.system_tables : Added on 2016-09-23 (MDEV-10775) +innodb.innodb-wl5522-debug-zip : Modified in 10.0.29 +innodb.table_index_statistics : Modified in 10.0.29 + +innodb_fts.innodb_fts_misc : MDEV-11767 - timeout +innodb_fts.innodb_fts_plugin : MDEV-11766 - unexpected warnings #---------------------------------------------------------------- @@ -125,29 +145,41 @@ multi_source.status_vars : MDEV-4632 - failed while waiting for Slave_received_h #---------------------------------------------------------------- +parts.partition_exch_qa_10 : MDEV-11765 - wrong result parts.partition_float_myisam : MDEV-10621 - Testcase timeout parts.partition_int_myisam : MDEV-10621 - Testcase timeout #---------------------------------------------------------------- +percona.percona_xtradb_bug317074 : MDEV-11767 - timeout + +#---------------------------------------------------------------- + perfschema.func_file_io : MDEV-5708 - fails for s390x perfschema.func_mutex : MDEV-5708 - fails for s390x perfschema.hostcache_ipv6_ssl : MDEV-10696 - crash on shutdown +perfschema.table_name : MDEV-11764 - wrong result perfschema.socket_summary_by_event_name_func : MDEV-10622 - Socket summary tables do not match perfschema_stress.* : MDEV-10996 - tests not maintained #---------------------------------------------------------------- -plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url, MDEV-11112 - valgrind warnings -plugins.pam : Modified on 2016-08-03 (MDEV-7329) -plugins.pam_cleartext : Modified on 2016-08-03 +plugins.feedback_plugin_send : MDEV-7932 - ssl failed for url, MDEV-11118 - wrong result plugins.server_audit : MDEV-9562 - crashes on sol10-sparc plugins.thread_pool_server_audit : MDEV-9562 - crashes on sol10-sparc + +#---------------------------------------------------------------- + +roles.create_and_drop_role : Modified in 10.0.29 +roles.create_and_grant_role : MDEV-11772 - wrong result +roles.role_case_sensitive-10744 : Added in 10.0.29 + #---------------------------------------------------------------- rpl.last_insert_id : MDEV-10625 - warnings in error log +rpl.rpl_alter_extra_persistent : Added in 10.0.29 rpl.rpl_auto_increment : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_bug45679 : MDEV-10417 - Fails on Mips rpl.rpl_auto_increment_update_failure : MDEV-10625 - warnings in error log @@ -155,29 +187,31 @@ rpl.rpl_binlog_index : MDEV-9501 - Warning: failed registering rpl.rpl_checksum_cache : MDEV-10626 - Testcase timeout rpl.rpl_circular_for_4_hosts : MDEV-10627 - Testcase timeout rpl.rpl_ddl : MDEV-10417 - Fails on Mips -rpl.rpl_drop_db : Modified on 2016-10-21 (Upstream MIPS test fixes) rpl.rpl_gtid_crash : MDEV-9501 - Warning: failed registering on master rpl.rpl_gtid_master_promote : MDEV-10628 - Timeout in sync_with_master rpl.rpl_gtid_stop_start : MDEV-10629 - Crash on shutdown rpl.rpl_gtid_until : MDEV-10625 - warnings in error log +rpl.rpl_heartbeat_basic : MDEV-11668 - Wrong result rpl.rpl_innodb_bug30888 : MDEV-10417 - Fails on Mips rpl.rpl_insert : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_insert_delayed : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_invoked_features : MDEV-10417 - Fails on Mips +rpl.rpl_mdev10863 : Added in 10.0.29 rpl.rpl_mdev6020 : MDEV-10630, MDEV-10417 - Timeouts, fails on Mips rpl.rpl_mdev6386 : MDEV-10631 - Wrong result on slave rpl.rpl_parallel : MDEV-10632, MDEV-10653 - Failures to sync, timeouts rpl.rpl_parallel_temptable : MDEV-10356 - Crash in close_thread_tables rpl.rpl_partition_innodb : MDEV-10417 - Fails on Mips rpl.rpl_row_drop_create_temp_table : MDEV-10626 - Testcase timeout +rpl.rpl_row_mysqlbinlog : Modified in 10.0.29 rpl.rpl_row_sp001 : MDEV-9329 - Fails on Ubuntu/s390x rpl.rpl_semi_sync_uninstall_plugin : MDEV-7140 - Wrong plugin status rpl.rpl_slave_grp_exec : MDEV-10514 - Unexpected deadlock -rpl.rpl_switch_stm_row_mixed : MDEV-10611 - Wrong usage of mutex +rpl.rpl_special_charset : Modified in 10.0.29 rpl.rpl_sync : MDEV-10633 - Database page corruption rpl.rpl_temporary_error2 : MDEV-10634 - Wrong number of retries -rpl.sec_behind_master-5114 : MDEV-8518 - Wrong value of Seconds_Behind_Master rpl.rpl_skip_replication : MDEV-9268 - Fails with timeout in sync_slave_with_master on Alpha +rpl.sec_behind_master-5114 : Modified in 10.0.29 rpl/extra/rpl_tests.* : MDEV-10994 - tests not maintained @@ -187,6 +221,8 @@ spider.* : MDEV-9329 - tests are too memory-consuming spider/bg.direct_aggregate : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.direct_aggregate_part : MDEV-7098 - Trying to unlock mutex that wasn't locked +spider/bg.spider3_fixes : MDEV-7098 - Trying to unlock mutex that wasn't locked +spider/bg.spider_fixes_part : MDEV-7098 - Trying to unlock mutex that wasn't locked spider/bg.ha : MDEV-7914, MDEV-9329 - Crash, failures on s390x spider/bg.ha_part : MDEV-9329 - Fails on Ubuntu/s390x spider/bg.spider_fixes : MDEV-7098, MDEV-9329 - Mutex problem, failures on s390x @@ -194,7 +230,7 @@ spider/bg.vp_fixes : MDEV-9329 - Fails on Ubuntu/s390x #---------------------------------------------------------------- -sphinx.* : MDEV-10747 - tests are not run in buildbot, they can't be stable +sphinx.* : MDEV-10986 - sphinx tests fail in buildbot and outside #---------------------------------------------------------------- @@ -203,34 +239,33 @@ stress.ddl_innodb : MDEV-10635 - Testcase timeout #---------------------------------------------------------------- sys_vars.autocommit_func2 : MDEV-9329 - Fails on Ubuntu/s390x -sys_vars.general_log_file_basic : Modified on 2016-08-09 (MDEV-10465) -sys_vars.slow_query_log_file_basic : Modified on 2016-08-09 (MDEV-10465) sys_vars.innodb_buffer_pool_dump_pct_basic : MDEV-10651 - sporadic failure on file_exists +sys_vars.replicate_do_db_basic : Modified in 10.0.29 +sys_vars.replicate_do_table_basic : Modified in 10.0.29 +sys_vars.replicate_ignore_db_basic : Modified in 10.0.29 +sys_vars.replicate_ignore_table_basic : Modified in 10.0.29 +sys_vars.replicate_wild_do_table_basic : Modified in 10.0.29 +sys_vars.replicate_wild_ignore_table_basic : Modified in 10.0.29 +sys_vars.thread_cache_size_func : MDEV-11775 - wrong result #---------------------------------------------------------------- -tokudb.background_job_manager : MDEV-10327 - Assertion failure on server shutdown tokudb.cluster_filter_unpack_varchar : MDEV-10636 - Wrong execution plan -tokudb.* : MDEV-9891 - massive crashes on shutdown -tokudb_alter_table.* : MDEV-9891 - massive crashes on shutdown +tokudb.dir_per_db : MDEV-11537 - wrong result +tokudb.locks-select-update-3 : MDEV-11774 - lock wait timeout +tokudb.table_index_statistics : Added in 10.0.29 + tokudb_backup.* : MDEV-11001 - tests don't work tokudb_bugs.checkpoint_lock : MDEV-10637 - Wrong processlist output tokudb_bugs.checkpoint_lock_3 : MDEV-10637 - Wrong processlist output -tokudb_bugs.* : MDEV-9891 - massive crashes on shutdown -tokudb_parts.* : MDEV-9891 - massive crashes on shutdown -tokudb_rpl_suites.* : MDEV-11001 - tests don't work -tokudb_sys_vars.* : MDEV-11001 - tests don't work -rpl-tokudb.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_add_index.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_backup.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_mariadb.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_sys_vars.* : MDEV-9891 - massive crashes on shutdown -tokudb/tokudb_rpl.* : MDEV-9891 - massive crashes on shutdown +tokudb_rpl.* : MDEV-11001 - tests don't work +tokudb_sys_vars.* : MDEV-11001 - tests don't work #---------------------------------------------------------------- unit.ma_test_loghandler : MDEV-10638 - record read not ok +unit.pfs : MySQL:84457 - unittest pft-t failing #---------------------------------------------------------------- From 67034b6d5265621d67ce589cdee537571ccacfa9 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 6 Dec 2016 14:05:09 +0200 Subject: [PATCH 50/74] Fixes for running with gcov --- .gitignore | 2 ++ BUILD/SETUP.sh | 2 +- BUILD/compile-pentium64-gcov | 2 +- debian/mariadb-test.install | 2 +- mysql-test/{README.gcov => README-gcov} | 10 ++++++---- mysql-test/lib/mtr_gcov.pl | 2 +- mysql-test/mysql-test-run.pl | 2 +- 7 files changed, 13 insertions(+), 9 deletions(-) rename mysql-test/{README.gcov => README-gcov} (56%) diff --git a/.gitignore b/.gitignore index dce5b5ea9342b..3cec6585ca8de 100644 --- a/.gitignore +++ b/.gitignore @@ -80,6 +80,8 @@ mysql-test/lib/My/SafeProcess/my_safe_process mysql-test/mtr mysql-test/mysql-test-run mysql-test/var +mysql-test-gcov.err +mysql-test-gcov.msg mysys/thr_lock mysys/thr_timer packaging/rpm-oel/mysql.spec diff --git a/BUILD/SETUP.sh b/BUILD/SETUP.sh index 01d654dba53ba..5f5b02500881c 100755 --- a/BUILD/SETUP.sh +++ b/BUILD/SETUP.sh @@ -293,7 +293,7 @@ gcov_compile_flags="$gcov_compile_flags -DMYSQL_SERVER_SUFFIX=-gcov -DHAVE_gcov" # GCC4 needs -fprofile-arcs -ftest-coverage on the linker command line (as well # as on the compiler command line), and this requires setting LDFLAGS for BDB. -gcov_link_flags="-fprofile-arcs -ftest-coverage" +gcov_link_flags="-fprofile-arcs -ftest-coverage -lgcov" gcov_configs="--with-gcov" diff --git a/BUILD/compile-pentium64-gcov b/BUILD/compile-pentium64-gcov index 36d4b6192ecbe..9587c51b4e0cd 100755 --- a/BUILD/compile-pentium64-gcov +++ b/BUILD/compile-pentium64-gcov @@ -28,6 +28,6 @@ export LDFLAGS="$gcov_link_flags" extra_flags="$pentium64_cflags $max_cflags $gcov_compile_flags" c_warnings="$c_warnings $debug_extra_warnings" cxx_warnings="$cxx_warnings $debug_extra_warnings" -extra_configs="$pentium_configs $debug_configs $gcov_configs $max_configs --with-zlib-dir=bundled" +extra_configs="$pentium_configs $debug_configs $gcov_configs $max_configs" . "$path/FINISH.sh" diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 5e6c3717d1f7f..605620dc28c7c 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -17,7 +17,7 @@ usr/lib/mysql/plugin/qa_auth_client.so usr/lib/mysql/plugin/qa_auth_interface.so usr/lib/mysql/plugin/qa_auth_server.so usr/share/mysql/mysql-test/README -usr/share/mysql/mysql-test/README.gcov +usr/share/mysql/mysql-test/README-gcov usr/share/mysql/mysql-test/README.stress usr/share/mysql/mysql-test/disabled.def usr/share/mysql/mysql-test/lib diff --git a/mysql-test/README.gcov b/mysql-test/README-gcov similarity index 56% rename from mysql-test/README.gcov rename to mysql-test/README-gcov index 6d2852e8ca054..ba22a7964234d 100644 --- a/mysql-test/README.gcov +++ b/mysql-test/README-gcov @@ -2,12 +2,14 @@ To be able to see the level of coverage with the current test suite, do the following: - Make sure gcov is installed - - Compile the MySQL distribution with BUILD/compile-pentium-gcov (if your + - Compile the MySQL distribution with BUILD/compile-pentium64-gcov (if your machine does not have a pentium CPU, hack this script, or just live with the pentium-specific stuff) - In the mysql-test directory, run this command: ./mysql-test-run -gcov - To see the level of coverage for a given source file: - grep source_file_name /tmp/gcov.out + grep -1 source_file_name ../mysql-test-gcov.msg - To see which lines are not yet covered, look at source_file_name.gcov in - the source tree. Then think hard about a test case that will cover those - lines, and write one! + the source tree. You can find this by doing something like: + find source-directory -name "mysqld.cc.gcov" + Then think hard about a test case that will cover those lines, and write + one! diff --git a/mysql-test/lib/mtr_gcov.pl b/mysql-test/lib/mtr_gcov.pl index a6e1f8efd5ffe..4c260d089b2d2 100644 --- a/mysql-test/lib/mtr_gcov.pl +++ b/mysql-test/lib/mtr_gcov.pl @@ -27,7 +27,7 @@ ($) print "Purging gcov information from '$dir'...\n"; system("find $dir -name \*.gcov -o -name \*.da" - . " -o -name \*.gcda | grep -v 'README.gcov\$' | xargs rm"); + . " -o -name \*.gcda | xargs rm"); } # diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 58aedf169d882..9efc9035555de 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -248,7 +248,7 @@ END our $opt_clean_vardir= $ENV{'MTR_CLEAN_VARDIR'}; our $opt_gcov; -our $opt_gcov_src_dir; +our $opt_gcov_src_dir="."; our $opt_gcov_exe= "gcov"; our $opt_gcov_err= "mysql-test-gcov.err"; our $opt_gcov_msg= "mysql-test-gcov.msg"; From e80ad58de8bce0923b91c08d12959c42e9e213a5 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 19 Dec 2016 22:25:42 +0200 Subject: [PATCH 51/74] Improve mysys/hash by caching hash_nr This is done without using any additional memory Added internal test case This is similar to MDEV-7716 --- .gitignore | 1 + include/hash.h | 2 +- mysys/CMakeLists.txt | 4 + mysys/hash.c | 238 ++++++++++++++++++++++++++++++------------- 4 files changed, 175 insertions(+), 70 deletions(-) diff --git a/.gitignore b/.gitignore index 3cec6585ca8de..ee52f228ea2e6 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,7 @@ mysql-test/mysql-test-run mysql-test/var mysql-test-gcov.err mysql-test-gcov.msg +mysys/test_hash mysys/thr_lock mysys/thr_timer packaging/rpm-oel/mysql.spec diff --git a/include/hash.h b/include/hash.h index fde7fc30d3825..892922d81a341 100644 --- a/include/hash.h +++ b/include/hash.h @@ -42,7 +42,7 @@ extern "C" { #define HASH_UNIQUE 1 /* hash_insert fails on duplicate key */ #define HASH_THREAD_SPECIFIC 2 /* Mark allocated memory THREAD_SPECIFIC */ -typedef uint my_hash_value_type; +typedef uint32 my_hash_value_type; typedef uchar *(*my_hash_get_key)(const uchar *,size_t*,my_bool); typedef my_hash_value_type (*my_hash_function)(CHARSET_INFO *, const uchar *, size_t); diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 892928ca69b96..6456db6c0bffb 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -96,6 +96,10 @@ ADD_EXECUTABLE(thr_timer thr_timer.c) TARGET_LINK_LIBRARIES(thr_timer mysys) SET_TARGET_PROPERTIES(thr_timer PROPERTIES COMPILE_FLAGS "-DMAIN") +ADD_EXECUTABLE(test_hash hash.c) +TARGET_LINK_LIBRARIES(test_hash mysys) +SET_TARGET_PROPERTIES(test_hash PROPERTIES COMPILE_FLAGS "-DMAIN") + IF(MSVC) INSTALL_DEBUG_TARGET(mysys DESTINATION ${INSTALL_LIBDIR}/debug) ENDIF() diff --git a/mysys/hash.c b/mysys/hash.c index dc03ea9a4dcb5..2b8130ee47ff6 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -23,14 +23,15 @@ #include #include "hash.h" -#define NO_RECORD ((uint) -1) +#define NO_RECORD ~((my_hash_value_type) 0) #define LOWFIND 1 #define LOWUSED 2 #define HIGHFIND 4 #define HIGHUSED 8 typedef struct st_hash_info { - uint next; /* index to next key */ + uint32 next; /* index to next key */ + my_hash_value_type hash_nr; uchar *data; /* data for current entry */ } HASH_LINK; @@ -196,13 +197,10 @@ static uint my_hash_mask(my_hash_value_type hashnr, size_t buffmax, return (uint) (hashnr & ((buffmax >> 1) -1)); } -static uint my_hash_rec_mask(const HASH *hash, HASH_LINK *pos, - size_t buffmax, size_t maxlength) +static inline uint my_hash_rec_mask(HASH_LINK *pos, + size_t buffmax, size_t maxlength) { - size_t length; - uchar *key= (uchar*) my_hash_key(hash, pos->data, &length, 0); - return my_hash_mask(hash->hash_function(hash->charset, key, length), buffmax, - maxlength); + return my_hash_mask(pos->hash_nr, buffmax, maxlength); } @@ -266,14 +264,13 @@ uchar* my_hash_first_from_hash_value(const HASH *hash, HASH_SEARCH_STATE *current_record) { HASH_LINK *pos; - uint flag,idx; DBUG_ENTER("my_hash_first_from_hash_value"); - flag=1; if (hash->records) { - idx= my_hash_mask(hash_value, - hash->blength, hash->records); + uint flag= 1; + uint idx= my_hash_mask(hash_value, + hash->blength, hash->records); do { pos= dynamic_element(&hash->array,idx,HASH_LINK*); @@ -286,7 +283,7 @@ uchar* my_hash_first_from_hash_value(const HASH *hash, if (flag) { flag=0; /* Reset flag */ - if (my_hash_rec_mask(hash, pos, hash->blength, hash->records) != idx) + if (my_hash_rec_mask(pos, hash->blength, hash->records) != idx) break; /* Wrong link */ } } @@ -378,15 +375,19 @@ static int hashcmp(const HASH *hash, HASH_LINK *pos, const uchar *key, my_bool my_hash_insert(HASH *info, const uchar *record) { int flag; - size_t idx,halfbuff,first_index; - my_hash_value_type hash_nr; - uchar *UNINIT_VAR(ptr_to_rec),*UNINIT_VAR(ptr_to_rec2); + uint idx, halfbuff, first_index; + size_t length; + my_hash_value_type current_hash_nr, UNINIT_VAR(rec_hash_nr), + UNINIT_VAR(rec2_hash_nr); + uchar *UNINIT_VAR(rec_data),*UNINIT_VAR(rec2_data), *key; HASH_LINK *data,*empty,*UNINIT_VAR(gpos),*UNINIT_VAR(gpos2),*pos; + key= (uchar*) my_hash_key(info, record, &length, 1); + current_hash_nr= info->hash_function(info->charset, key, length); + if (info->flags & HASH_UNIQUE) { - uchar *key= (uchar*) my_hash_key(info, record, &idx, 1); - if (my_hash_search(info, key, idx)) + if (my_hash_search_using_hash_value(info, current_hash_nr, key, length)) return(TRUE); /* Duplicate entry */ } @@ -402,8 +403,9 @@ my_bool my_hash_insert(HASH *info, const uchar *record) { do { + my_hash_value_type hash_nr; pos=data+idx; - hash_nr=rec_hashnr(info,pos->data); + hash_nr= pos->hash_nr; if (flag == 0) /* First loop; Check if ok */ if (my_hash_mask(hash_nr, info->blength, info->records) != first_index) break; @@ -413,17 +415,19 @@ my_bool my_hash_insert(HASH *info, const uchar *record) { if (flag & HIGHFIND) { - flag=LOWFIND | HIGHFIND; + flag= LOWFIND | HIGHFIND; /* key shall be moved to the current empty position */ - gpos=empty; - ptr_to_rec=pos->data; + gpos= empty; + rec_data= pos->data; + rec_hash_nr= pos->hash_nr; empty=pos; /* This place is now free */ } else { - flag=LOWFIND | LOWUSED; /* key isn't changed */ - gpos=pos; - ptr_to_rec=pos->data; + flag= LOWFIND | LOWUSED; /* key isn't changed */ + gpos= pos; + rec_data= pos->data; + rec_hash_nr= pos->hash_nr; } } else @@ -431,12 +435,14 @@ my_bool my_hash_insert(HASH *info, const uchar *record) if (!(flag & LOWUSED)) { /* Change link of previous LOW-key */ - gpos->data=ptr_to_rec; - gpos->next= (uint) (pos-data); + gpos->data= rec_data; + gpos->hash_nr= rec_hash_nr; + gpos->next= (uint) (pos-data); flag= (flag & HIGHFIND) | (LOWFIND | LOWUSED); } - gpos=pos; - ptr_to_rec=pos->data; + gpos= pos; + rec_data= pos->data; + rec_hash_nr= pos->hash_nr; } } else @@ -445,20 +451,24 @@ my_bool my_hash_insert(HASH *info, const uchar *record) { flag= (flag & LOWFIND) | HIGHFIND; /* key shall be moved to the last (empty) position */ - gpos2 = empty; empty=pos; - ptr_to_rec2=pos->data; + gpos2= empty; + empty= pos; + rec2_data= pos->data; + rec2_hash_nr= pos->hash_nr; } else { if (!(flag & HIGHUSED)) { /* Change link of previous hash-key and save */ - gpos2->data=ptr_to_rec2; - gpos2->next=(uint) (pos-data); + gpos2->data= rec2_data; + gpos2->hash_nr= rec2_hash_nr; + gpos2->next= (uint) (pos-data); flag= (flag & LOWFIND) | (HIGHFIND | HIGHUSED); } - gpos2=pos; - ptr_to_rec2=pos->data; + gpos2= pos; + rec2_data= pos->data; + rec2_hash_nr= pos->hash_nr; } } } @@ -466,41 +476,44 @@ my_bool my_hash_insert(HASH *info, const uchar *record) if ((flag & (LOWFIND | LOWUSED)) == LOWFIND) { - gpos->data=ptr_to_rec; - gpos->next=NO_RECORD; + gpos->data= rec_data; + gpos->hash_nr= rec_hash_nr; + gpos->next= NO_RECORD; } if ((flag & (HIGHFIND | HIGHUSED)) == HIGHFIND) { - gpos2->data=ptr_to_rec2; - gpos2->next=NO_RECORD; + gpos2->data= rec2_data; + gpos2->hash_nr= rec2_hash_nr; + gpos2->next= NO_RECORD; } } - /* Check if we are at the empty position */ - idx= my_hash_mask(rec_hashnr(info, record), info->blength, info->records + 1); - pos=data+idx; + idx= my_hash_mask(current_hash_nr, info->blength, info->records + 1); + pos= data+idx; + /* Check if we are at the empty position */ if (pos == empty) { - pos->data=(uchar*) record; pos->next=NO_RECORD; } else { - /* Check if more records in same hash-nr family */ - empty[0]=pos[0]; - gpos= data + my_hash_rec_mask(info, pos, info->blength, info->records + 1); + /* Move conflicting record to empty position (last) */ + empty[0]= pos[0]; + /* Check if the moved record was in same hash-nr family */ + gpos= data + my_hash_rec_mask(pos, info->blength, info->records + 1); if (pos == gpos) { - pos->data=(uchar*) record; - pos->next=(uint) (empty - data); + /* Point to moved record */ + pos->next= (uint32) (empty - data); } else { - pos->data=(uchar*) record; - pos->next=NO_RECORD; + pos->next= NO_RECORD; movelink(data,(uint) (pos-data),(uint) (gpos-data),(uint) (empty-data)); } } + pos->data= (uchar*) record; + pos->hash_nr= current_hash_nr; if (++info->records == info->blength) info->blength+= info->blength; return(0); @@ -557,15 +570,14 @@ my_bool my_hash_delete(HASH *hash, uchar *record) else if (pos->next != NO_RECORD) { empty=data+(empty_index=pos->next); - pos->data=empty->data; - pos->next=empty->next; + pos[0]= empty[0]; } - if (empty == lastpos) /* last key at wrong pos or no next link */ + if (empty == lastpos) /* last key at wrong pos or no next link */ goto exit; /* Move the last key (lastpos) */ - lastpos_hashnr=rec_hashnr(hash,lastpos->data); + lastpos_hashnr= lastpos->hash_nr; /* pos is where lastpos should be */ pos= data + my_hash_mask(lastpos_hashnr, hash->blength, hash->records); if (pos == empty) /* Move to empty position. */ @@ -573,7 +585,7 @@ my_bool my_hash_delete(HASH *hash, uchar *record) empty[0]=lastpos[0]; goto exit; } - pos_hashnr=rec_hashnr(hash,pos->data); + pos_hashnr= pos->hash_nr; /* pos3 is where the pos should be */ pos3= data + my_hash_mask(pos_hashnr, hash->blength, hash->records); if (pos != pos3) @@ -616,23 +628,30 @@ my_bool my_hash_delete(HASH *hash, uchar *record) my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, size_t old_key_length) { - uint new_index,new_pos_index,records; - size_t idx, empty, blength; + uint new_index, new_pos_index, org_index, records, idx; + size_t length, empty, blength; + my_hash_value_type hash_nr; HASH_LINK org_link,*data,*previous,*pos; + uchar *new_key; DBUG_ENTER("my_hash_update"); + + new_key= (uchar*) my_hash_key(hash, record, &length, 1); + hash_nr= hash->hash_function(hash->charset, new_key, length); if (HASH_UNIQUE & hash->flags) { HASH_SEARCH_STATE state; - uchar *found, *new_key= (uchar*) my_hash_key(hash, record, &idx, 1); - if ((found= my_hash_first(hash, new_key, idx, &state))) + uchar *found; + + if ((found= my_hash_first_from_hash_value(hash, hash_nr, new_key, length, + &state))) { do { if (found != record) DBUG_RETURN(1); /* Duplicate entry */ } - while ((found= my_hash_next(hash, new_key, idx, &state))); + while ((found= my_hash_next(hash, new_key, length, &state))); } } @@ -645,19 +664,24 @@ my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, (old_key_length ? old_key_length : hash->key_length)), blength, records); - new_index= my_hash_mask(rec_hashnr(hash, record), blength, records); - if (idx == new_index) - DBUG_RETURN(0); /* Nothing to do (No record check) */ + org_index= idx; + new_index= my_hash_mask(hash_nr, blength, records); previous=0; for (;;) { - if ((pos= data+idx)->data == record) break; previous=pos; if ((idx=pos->next) == NO_RECORD) DBUG_RETURN(1); /* Not found in links */ } + + if (org_index == new_index) + { + data[idx].hash_nr= hash_nr; /* Hash number may have changed */ + DBUG_RETURN(0); /* Record is in right position */ + } + org_link= *pos; empty=idx; @@ -692,21 +716,24 @@ my_bool my_hash_update(HASH *hash, uchar *record, uchar *old_key, data[empty]= org_link; } data[empty].next= NO_RECORD; + data[empty].hash_nr= hash_nr; DBUG_RETURN(0); } pos=data+new_index; - new_pos_index= my_hash_rec_mask(hash, pos, blength, records); + new_pos_index= my_hash_rec_mask(pos, blength, records); if (new_index != new_pos_index) { /* Other record in wrong position */ - data[empty] = *pos; + data[empty]= *pos; movelink(data,new_index,new_pos_index, (uint) empty); org_link.next=NO_RECORD; data[new_index]= org_link; + data[new_index].hash_nr= hash_nr; } else { /* Link in chain at right position */ org_link.next=data[new_index].next; data[empty]=org_link; + data[empty].hash_nr= hash_nr; data[new_index].next= (uint) empty; } DBUG_RETURN(0); @@ -765,7 +792,7 @@ my_bool my_hash_iterate(HASH *hash, my_hash_walk_action action, void *argument) } -#ifndef DBUG_OFF +#if !defined(DBUG_OFF) || defined(MAIN) my_bool my_hash_check(HASH *hash) { @@ -781,7 +808,15 @@ my_bool my_hash_check(HASH *hash) for (i=found=max_links=seek=0 ; i < records ; i++) { - if (my_hash_rec_mask(hash, data + i, blength, records) == i) + size_t length; + uchar *key= (uchar*) my_hash_key(hash, data[i].data, &length, 0); + if (data[i].hash_nr != hash->hash_function(hash->charset, key, length)) + { + DBUG_PRINT("error", ("record at %d has wrong hash", i)); + error= 1; + } + + if (my_hash_rec_mask(data + i, blength, records) == i) { found++; seek++; links=1; for (idx=data[i].next ; @@ -797,7 +832,7 @@ my_bool my_hash_check(HASH *hash) } hash_info=data+idx; seek+= ++links; - if ((rec_link= my_hash_rec_mask(hash, hash_info, + if ((rec_link= my_hash_rec_mask(hash_info, blength, records)) != i) { DBUG_PRINT("error", ("Record in wrong link at %d: Start %d " @@ -820,6 +855,71 @@ my_bool my_hash_check(HASH *hash) DBUG_PRINT("info", ("records: %u seeks: %d max links: %d hitrate: %.2f", records,seek,max_links,(float) seek / (float) records)); + DBUG_ASSERT(error == 0); return error; } #endif + +#ifdef MAIN + +#define RECORDS 1000 + +uchar *test_get_key(uchar *data, size_t *length, + my_bool not_used __attribute__((unused))) +{ + *length= 2; + return data; +} + + +int main(int argc __attribute__((unused)),char **argv __attribute__((unused))) +{ + uchar records[RECORDS][2], copy[2]; + HASH hash_test; + uint i; + MY_INIT(argv[0]); + DBUG_PUSH("d:t:O,/tmp/test_hash.trace"); + + printf("my_hash_init\n"); + if (my_hash_init2(&hash_test, 100, &my_charset_bin, 20, + 0, 0, (my_hash_get_key) test_get_key, 0, 0, HASH_UNIQUE)) + { + fprintf(stderr, "hash init failed\n"); + exit(1); + } + + printf("my_hash_insert\n"); + for (i= 0 ; i < RECORDS ; i++) + { + int2store(records[i],i); + my_hash_insert(&hash_test, records[i]); + my_hash_check(&hash_test); + } + printf("my_hash_update\n"); + for (i= 0 ; i < RECORDS ; i+=2) + { + memcpy(copy, records[i], 2); + int2store(records[i],i + RECORDS); + if (my_hash_update(&hash_test, records[i], copy, 2)) + { + fprintf(stderr, "hash update failed\n"); + exit(1); + } + my_hash_check(&hash_test); + } + printf("my_hash_delete\n"); + for (i= 0 ; i < RECORDS ; i++) + { + if (my_hash_delete(&hash_test, records[i])) + { + fprintf(stderr, "hash delete failed\n"); + exit(1); + } + my_hash_check(&hash_test); + } + my_hash_free(&hash_test); + printf("ok\n"); + my_end(MY_CHECK_ERROR); + return(0); +} +#endif /* MAIN */ From ed0bc17bee591599c988df21b8d5a264f08eb885 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 20 Dec 2016 13:03:45 +0200 Subject: [PATCH 52/74] Removed usage of my_hash_search() with uninitialized HASH. - Not documented on intened usage - Extra checking takes time for all HASH usage --- mysys/hash.c | 14 +++++++------- sql/item_func.cc | 3 ++- sql/sql_acl.cc | 2 ++ sql/sql_handler.cc | 13 ++++++++----- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/mysys/hash.c b/mysys/hash.c index 2b8130ee47ff6..1d6b3d5daaade 100644 --- a/mysys/hash.c +++ b/mysys/hash.c @@ -246,13 +246,13 @@ uchar* my_hash_first(const HASH *hash, const uchar *key, size_t length, HASH_SEARCH_STATE *current_record) { uchar *res; - if (my_hash_inited(hash)) - res= my_hash_first_from_hash_value(hash, - hash->hash_function(hash->charset, key, - length ? length : hash->key_length), - key, length, current_record); - else - res= 0; + DBUG_ASSERT(my_hash_inited(hash)); + + res= my_hash_first_from_hash_value(hash, + hash->hash_function(hash->charset, key, + length ? length : + hash->key_length), + key, length, current_record); return res; } diff --git a/sql/item_func.cc b/sql/item_func.cc index c38bdba05c2ad..8e912fe83c8fd 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -4302,7 +4302,8 @@ longlong Item_func_release_lock::val_int() User_level_lock *ull; - if (!(ull= + if (!my_hash_inited(&thd->ull_hash) || + !(ull= (User_level_lock*) my_hash_search(&thd->ull_hash, ull_key.ptr(), ull_key.length()))) { diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index ce7de2ed72b10..d2840a81e7ef4 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -4365,6 +4365,8 @@ table_hash_search(const char *host, const char *ip, const char *db, static GRANT_COLUMN * column_hash_search(GRANT_TABLE *t, const char *cname, uint length) { + if (!my_hash_inited(&t->hash_columns)) + return (GRANT_COLUMN*) 0; return (GRANT_COLUMN*) my_hash_search(&t->hash_columns, (uchar*) cname, length); } diff --git a/sql/sql_handler.cc b/sql/sql_handler.cc index 58db104ec96fd..4a27244f5b9eb 100644 --- a/sql/sql_handler.cc +++ b/sql/sql_handler.cc @@ -458,9 +458,10 @@ bool mysql_ha_close(THD *thd, TABLE_LIST *tables) my_error(ER_LOCK_OR_ACTIVE_TRANSACTION, MYF(0)); DBUG_RETURN(TRUE); } - if ((handler= (SQL_HANDLER*) my_hash_search(&thd->handler_tables_hash, - (uchar*) tables->alias, - strlen(tables->alias) + 1))) + if ((my_hash_inited(&thd->handler_tables_hash)) && + (handler= (SQL_HANDLER*) my_hash_search(&thd->handler_tables_hash, + (uchar*) tables->alias, + strlen(tables->alias) + 1))) { mysql_ha_close_table(handler); my_hash_delete(&thd->handler_tables_hash, (uchar*) handler); @@ -497,8 +498,10 @@ bool mysql_ha_close(THD *thd, TABLE_LIST *tables) SQL_HANDLER *mysql_ha_find_handler(THD *thd, const char *name) { SQL_HANDLER *handler; - if ((handler= (SQL_HANDLER*) my_hash_search(&thd->handler_tables_hash, - (uchar*) name, strlen(name) + 1))) + if ((my_hash_inited(&thd->handler_tables_hash)) && + (handler= (SQL_HANDLER*) my_hash_search(&thd->handler_tables_hash, + (uchar*) name, + strlen(name) + 1))) { DBUG_PRINT("info-in-hash",("'%s'.'%s' as '%s' table: %p", handler->db.str, From ed008a74cf4cfe8619595ec71a6073a9e94f984c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 31 Dec 2016 15:11:52 +0100 Subject: [PATCH 53/74] Make atomic writes general - Atomic writes are enabled by default - Automatically detect if device supports atomic write and use it if atomic writes are enabled - Remove ATOMIC WRITE options from CREATE TABLE - Atomic write is a device option, not a table options as the table may crash if the media changes - Add support for SHANNON SSD cards --- include/my_sys.h | 8 + .../r/innodb_use_atomic_writes_basic.result | 10 +- .../suite/sys_vars/r/sysvars_innodb.result | 6 +- mysys/CMakeLists.txt | 1 + mysys/my_atomic_writes.c | 333 ++++++++++++++++++ sql/mysqld.cc | 3 + storage/innobase/buf/buf0flu.cc | 4 +- storage/innobase/dict/dict0dict.cc | 7 - storage/innobase/fil/fil0fil.cc | 118 +++---- storage/innobase/fsp/fsp0fsp.cc | 12 +- storage/innobase/fsp/fsp0space.cc | 9 +- storage/innobase/fsp/fsp0sysspace.cc | 20 +- storage/innobase/handler/ha_innodb.cc | 66 ++-- storage/innobase/handler/handler0alter.cc | 3 +- storage/innobase/include/dict0dict.h | 4 +- storage/innobase/include/dict0dict.ic | 43 +-- storage/innobase/include/dict0pagecompress.h | 20 -- storage/innobase/include/dict0pagecompress.ic | 36 -- storage/innobase/include/dict0types.h | 7 - storage/innobase/include/fil0fil.h | 21 +- storage/innobase/include/fil0pagecompress.h | 2 +- storage/innobase/include/fsp0file.h | 8 - storage/innobase/include/fsp0fsp.h | 2 +- storage/innobase/include/fsp0fsp.ic | 10 +- storage/innobase/include/fsp0pagecompress.h | 9 - storage/innobase/include/fsp0pagecompress.ic | 30 +- storage/innobase/os/os0file.cc | 45 --- storage/innobase/srv/srv0start.cc | 11 +- support-files/build-tags | 4 +- 29 files changed, 474 insertions(+), 378 deletions(-) create mode 100644 mysys/my_atomic_writes.c diff --git a/include/my_sys.h b/include/my_sys.h index c8f3e1bf3a4c5..2e9f842d06e6a 100644 --- a/include/my_sys.h +++ b/include/my_sys.h @@ -192,6 +192,14 @@ extern void my_large_free(uchar *ptr); #define my_large_free(A) my_free_lock((A)) #endif /* HAVE_LARGE_PAGES */ +void my_init_atomic_write(void); +#ifdef __linux__ +my_bool my_test_if_atomic_write(File handle, int pagesize); +#else +#define my_test_if_atomic_write(A, B) 0 +#endif /* __linux__ */ +extern my_bool my_may_have_atomic_write; + #if defined(HAVE_ALLOCA) && !defined(HAVE_valgrind) #if defined(_AIX) && !defined(__GNUC__) && !defined(_AIX43) #pragma alloca diff --git a/mysql-test/suite/sys_vars/r/innodb_use_atomic_writes_basic.result b/mysql-test/suite/sys_vars/r/innodb_use_atomic_writes_basic.result index fa6379fbe1c45..e420d6e516185 100644 --- a/mysql-test/suite/sys_vars/r/innodb_use_atomic_writes_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_use_atomic_writes_basic.result @@ -1,20 +1,20 @@ select @@global.innodb_use_atomic_writes; @@global.innodb_use_atomic_writes -0 +1 select @@session.innodb_use_atomic_writes; ERROR HY000: Variable 'innodb_use_atomic_writes' is a GLOBAL variable show global variables like 'innodb_use_atomic_writes'; Variable_name Value -innodb_use_atomic_writes OFF +innodb_use_atomic_writes ON show session variables like 'innodb_use_atomic_writes'; Variable_name Value -innodb_use_atomic_writes OFF +innodb_use_atomic_writes ON select * from information_schema.global_variables where variable_name='innodb_use_atomic_writes'; VARIABLE_NAME VARIABLE_VALUE -INNODB_USE_ATOMIC_WRITES OFF +INNODB_USE_ATOMIC_WRITES ON select * from information_schema.session_variables where variable_name='innodb_use_atomic_writes'; VARIABLE_NAME VARIABLE_VALUE -INNODB_USE_ATOMIC_WRITES OFF +INNODB_USE_ATOMIC_WRITES ON set global innodb_use_atomic_writes=1; ERROR HY000: Variable 'innodb_use_atomic_writes' is a read only variable set session innodb_use_atomic_writes=1; diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb.result b/mysql-test/suite/sys_vars/r/sysvars_innodb.result index de2295859c3eb..8f655c91427b6 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb.result +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb.result @@ -2584,12 +2584,12 @@ READ_ONLY YES COMMAND_LINE_ARGUMENT REQUIRED VARIABLE_NAME INNODB_USE_ATOMIC_WRITES SESSION_VALUE NULL -GLOBAL_VALUE OFF +GLOBAL_VALUE ON GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE OFF +DEFAULT_VALUE ON VARIABLE_SCOPE GLOBAL VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Prevent partial page writes, via atomic writes.The option is used to prevent partial writes in case of a crash/poweroff, as faster alternative to doublewrite buffer.Currently this option works only on Linux only with FusionIO device, and directFS filesystem. +VARIABLE_COMMENT Enable atomic writes, instead of using the doublewrite buffer, for files on devices that supports atomic writes. To use this option one must use file_per_table=1, flush_method=O_DIRECT and use_fallocate=1. This option only works on Linux with either FusionIO cards using the directFS filesystem or with Shannon cards using any file system. NUMERIC_MIN_VALUE NULL NUMERIC_MAX_VALUE NULL NUMERIC_BLOCK_SIZE NULL diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 6456db6c0bffb..6c43c29a75853 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -42,6 +42,7 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c my_atomic.c my_getncpus.c my_safehash.c my_chmod.c my_rnd.c my_uuid.c wqueue.c waiting_threads.c ma_dyncol.c ../sql-common/my_time.c my_rdtsc.c my_context.c psi_noop.c + my_atomic_writes.c file_logger.c) IF (WIN32) diff --git a/mysys/my_atomic_writes.c b/mysys/my_atomic_writes.c new file mode 100644 index 0000000000000..0b54a20771338 --- /dev/null +++ b/mysys/my_atomic_writes.c @@ -0,0 +1,333 @@ +/* Copyright (c) 2016, MariaDB Corporation + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +#include "mysys_priv.h" + +my_bool my_may_have_atomic_write= 0; + +#ifdef __linux__ + +my_bool has_shannon_atomic_write= 0, has_fusion_io_atomic_write= 0; + +#include + + +/*********************************************************************** + FUSION_IO +************************************************************************/ + +/** FusionIO atomic write control info */ +#define DFS_IOCTL_ATOMIC_WRITE_SET _IOW(0x95, 2, uint) + + +/** + Check if the system has a funsion_io card + @return TRUE Card exists +*/ + +static my_bool test_if_fusion_io_card_exists() +{ + /* Fusion card requires fallocate to exists */ +#ifndef HAVE_POSIX_FALLOCATE + return 0; +#else + return (access("/dev/fcta", F_OK)) == 0; +#endif +} + + +/** + Check if a file is on a Fusion_IO device and that it supports atomic_write + @param[in] file OS file handle + @param[in] page_size page size + @return TRUE Atomic write supported +*/ + +static my_bool fusion_io_has_atomic_write(File file, int page_size) +{ + int atomic= 1; + if (page_size <= 32768 && + ioctl(file, DFS_IOCTL_ATOMIC_WRITE_SET, &atomic) != -1) + return(TRUE); + return(FALSE); +} + + +/*********************************************************************** + SHANNON +************************************************************************/ + +#define SHANNON_IOMAGIC 'x' +#define SHANNON_IOCQATOMIC_SIZE _IO(SHANNON_IOMAGIC, 22) + +#define SHANNON_MAX_DEVICES 32 +#define SHANNON_NO_ATOMIC_SIZE_YET -2 + +struct shannon_dev +{ + char dev_name[32]; + dev_t st_dev; + int atomic_size; +}; + + +static struct shannon_dev shannon_devices[SHANNON_MAX_DEVICES+1]; + +/** + Check if the system has a Shannon card + If card exists, record device numbers to allow us to later check if + a given file is on this device. + @return TRUE Card exists +*/ + +static my_bool test_if_shannon_card_exists() +{ + uint shannon_found_devices= 0; + char dev_part; + uint dev_no; + + if (access("/dev/scta", F_OK) < 0) + return 0; + + /* + The Shannon devices are /dev/dfX, where X can be from a-z. + We have to check all of them as some may be missing if the user + removed one with the U.2 interface. + */ + + for (dev_part= 'a' ; dev_part < 'z' ; dev_part++) + { + char path[32]; + struct stat stat_buff; + + sprintf(path, "/dev/df%c", dev_part); +#ifdef TEST_SHANNON + if (lstat(path, &stat_buff) < 0) + { + printf("%s(): lstat failed.\n", __func__); + break; + } +#endif + shannon_devices[shannon_found_devices].st_dev= stat_buff.st_rdev; + sprintf(shannon_devices[shannon_found_devices].dev_name, "/dev/sct%c", + dev_part); + +#ifdef TEST_SHANNON + printf("%s(): i=%d, stat_buff.st_dev=0x%lx, stat_buff.st_rdev=0x%lx, st_rdev=0x%lx, dev_name=%s\n", + __func__, + shannon_found_devices, + (ulong) stat_buff.st_dev, + (ulong) stat_buff.st_rdev, + (ulong) shannon_devices[shannon_found_devices].st_dev, + shannon_devices[shannon_found_devices].dev_name); +#endif + + /* + The atomic size will be checked on first access. This is needed + as a normal user can't open the /dev/scta file + */ + shannon_devices[shannon_found_devices].atomic_size= + SHANNON_NO_ATOMIC_SIZE_YET; + if (++shannon_found_devices== SHANNON_MAX_DEVICES) + goto end; + + for (dev_no= 1 ; dev_no < 9 ; dev_no++) + { + sprintf(path, "/dev/df%c%d", dev_part, dev_no); + if (lstat(path, &stat_buff) < 0) + break; + + shannon_devices[shannon_found_devices].st_dev= stat_buff.st_rdev; + sprintf(shannon_devices[shannon_found_devices].dev_name, "/dev/sct%c%d", + dev_part, dev_no); + +#ifdef TEST_SHANNON + printf("%s(): i=%d, st_dev=0x%lx, st_rdev=0x%lx, dev_name=%s\n", + __func__, + shannon_found_devices, + (ulong) stat_buff.st_dev, + (ulong) shannon_devices[shannon_found_devices].st_dev, + shannon_devices[shannon_found_devices].dev_name); +#endif + + /* + The atomic size will be checked on first access. This is needed + as a normal user can't open the /dev/scta file + */ + shannon_devices[shannon_found_devices].atomic_size= + SHANNON_NO_ATOMIC_SIZE_YET; + if (++shannon_found_devices == SHANNON_MAX_DEVICES) + goto end; + } + } +end: + shannon_devices[shannon_found_devices].st_dev= 0; + return shannon_found_devices > 0; +} + + +static my_bool shannon_dev_has_atomic_write(struct shannon_dev *dev, + int page_size) +{ +#ifdef TEST_SHANNON + printf("%s: enter: page_size=%d, atomic_size=%d, dev_name=%s\n", + __func__, + page_size, + dev->atomic_size, + dev->dev_name); +#endif + if (dev->atomic_size == SHANNON_NO_ATOMIC_SIZE_YET) + { + int fd= open(dev->dev_name, 0); + if (fd < 0) + { + perror("open() failed!"); + dev->atomic_size= 0; /* Don't try again */ + return FALSE; + } + dev->atomic_size= ioctl(fd, SHANNON_IOCQATOMIC_SIZE); + close(fd); + } + +#ifdef TEST_SHANNON + printf("%s: exit: page_size=%d, atomic_size=%d, dev_name=%s\n", + __func__, + page_size, + dev->atomic_size, + dev->dev_name); +#endif + return (page_size <= dev->atomic_size); +} + + +/** + Check if a file is on a Shannon device and that it supports atomic_write + @param[in] file OS file handle + @param[in] page_size page size + @return TRUE Atomic write supported + + @notes + This is called only at first open of a file. In this case it's doesn't + matter so much that we loop over all cards. + We update the atomic size on first access. +*/ + +static my_bool shannon_has_atomic_write(File file, int page_size) +{ + struct shannon_dev *dev; + struct stat stat_buff; + + if (fstat(file, &stat_buff) < 0) + { +#ifdef TEST_SHANNON + printf("%s(): fstat failed\n", __func__); +#endif + return 0; + } + +#ifdef TEST_SHANNON + printf("%s(): st_dev=0x%lx, st_rdev=0x%lx\n", __func__, + (ulong) stat_buff.st_dev, (ulong) stat_buff.st_rdev); +#endif + + for (dev= shannon_devices ; dev->st_dev; dev++) + { +#ifdef TEST_SHANNON + printf("%s(): st_rdev=0x%lx\n", __func__, (ulong) dev->st_dev); +#endif + if (stat_buff.st_dev == dev->st_dev) + return shannon_dev_has_atomic_write(dev, page_size); + } + return 0; +} + + +/*********************************************************************** + Generic atomic write code +************************************************************************/ + +/* + Initalize automic write sub systems. + Checks if we have any devices that supports atomic write +*/ + +void my_init_atomic_write(void) +{ + if ((has_shannon_atomic_write= test_if_shannon_card_exists()) || + (has_fusion_io_atomic_write= test_if_fusion_io_card_exists())) + my_may_have_atomic_write= 1; +#ifdef TEST_SHANNON + printf("%s(): has_shannon_atomic_write=%d, my_may_have_atomic_write=%d\n", + __func__, + has_shannon_atomic_write, + my_may_have_atomic_write); +#endif +} + + +/** + Check if a file supports atomic write + + @return FALSE No atomic write support + TRUE File supports atomic write +*/ + +my_bool my_test_if_atomic_write(File handle, int page_size) +{ +#ifdef TEST_SHANNON + printf("%s(): has_shannon_atomic_write=%d, my_may_have_atomic_write=%d\n", + __func__, + has_shannon_atomic_write, + my_may_have_atomic_write); +#endif + if (!my_may_have_atomic_write) + return 0; + if (has_shannon_atomic_write && + shannon_has_atomic_write(handle, page_size)) + return 1; + + if (has_fusion_io_atomic_write && + fusion_io_has_atomic_write(handle, page_size)) + return 1; + + return 0; +} + +#ifdef TEST_SHANNON +int main() +{ + int fd, ret; + + my_init_atomic_write(); + fd= open("/u01/1.file", O_RDWR); + ret= my_test_if_atomic_write(fd, 4096); + if (ret) + printf("support atomic_write\n"); + else + printf("do not support atomic_write\n"); + close(fd); + return 0; +} +#endif + + +#else /* __linux__ */ + +/* Dummy functions to provide the interfaces for other systems */ + +void my_init_atomic_write(void) +{ +} +#endif /* __linux__ */ diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 84d6e3b582fc5..4b79ea43c7c42 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5862,6 +5862,9 @@ int mysqld_main(int argc, char **argv) if (my_setwd(mysql_real_data_home, opt_abort ? 0 : MYF(MY_WME)) && !opt_abort) unireg_abort(1); /* purecov: inspected */ + /* Atomic write initialization must be done as root */ + my_init_atomic_write(); + if ((user_info= check_user(mysqld_user))) { #if defined(HAVE_MLOCKALL) && defined(MCL_CURRENT) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index b82c4db18ad27..0f901363596fe 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1008,7 +1008,7 @@ buf_flush_write_block_low( { page_t* frame = NULL; ulint space_id = bpage->id.space(); - atomic_writes_t awrites = fil_space_get_atomic_writes(space_id); + bool atomic_writes = fil_space_get_atomic_writes(space_id); #ifdef UNIV_DEBUG buf_pool_t* buf_pool = buf_pool_from_bpage(bpage); @@ -1086,7 +1086,7 @@ buf_flush_write_block_low( || buf_dblwr == NULL || srv_read_only_mode || fsp_is_system_temporary(bpage->id.space()) - || awrites == ATOMIC_WRITES_ON) { + || atomic_writes) { ut_ad(!srv_read_only_mode || fsp_is_system_temporary(bpage->id.space())); diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 37c6341a29368..0013a5f690f8c 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -7273,7 +7273,6 @@ dict_tf_to_fsp_flags( bool is_shared = DICT_TF_HAS_SHARED_SPACE(table_flags); bool page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); ut_ad(!page_size.is_compressed() || has_atomic_blobs); @@ -7305,12 +7304,6 @@ dict_tf_to_fsp_flags( fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); } - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - if (atomic_writes) { - fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); - } - ut_ad(fsp_flags_is_valid(fsp_flags)); return(fsp_flags); diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index a2865141a4d6f..f104112eb7ac7 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -475,34 +475,6 @@ fil_space_is_flushed( return(true); } -#ifdef UNIV_LINUX - -#include -/** FusionIO atomic write control info */ -#define DFS_IOCTL_ATOMIC_WRITE_SET _IOW(0x95, 2, uint) - -/** -Try and enable FusionIO atomic writes. -@param[in] file OS file handle -@return true if successful */ -bool -fil_fusionio_enable_atomic_write(os_file_t file) -{ - if (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT) { - - uint atomic = 1; - - ut_a(file != -1); - - if (ioctl(file, DFS_IOCTL_ATOMIC_WRITE_SET, &atomic) != -1) { - - return(true); - } - } - - return(false); -} -#endif /* UNIV_LINUX */ /** Append a file to the chain of files of a space. @param[in] name file name of a file that is not open @@ -510,7 +482,7 @@ fil_fusionio_enable_atomic_write(os_file_t file) @param[in,out] space tablespace from fil_space_create() @param[in] is_raw whether this is a raw device or partition @param[in] punch_hole true if supported for this node -@param[in] atomic_write true if the file has atomic write enabled +@param[in] atomic_write true if the file could use atomic write @param[in] max_pages maximum number of pages in file, ULINT_MAX means the file size is unlimited. @return pointer to the file name @@ -606,7 +578,7 @@ fil_node_create_low( an integer @param[in,out] space space where to append @param[in] is_raw true if a raw device or a raw disk partition -@param[in] atomic_write true if the file has atomic write enabled +@param[in] atomic_write true if the file could use atomic write @param[in] max_pages maximum number of pages in file, ULINT_MAX means the file size is unlimited. @return pointer to the file name @@ -829,7 +801,27 @@ fil_node_open_file( node->handle = os_file_create( innodb_data_file_key, node->name, OS_FILE_OPEN, OS_FILE_AIO, OS_DATA_FILE, read_only_mode, &success); - } + + if (!space->atomic_write_tested) + { + const page_size_t page_size(space->flags); + + space->atomic_write_tested= 1; + /* + Atomic writes is supported if the file can be used + with atomic_writes (not log file), O_DIRECT is + used (tested in ha_innodbc.cc) and the file is + device and file system that supports atomic writes + for the given block size + */ + space->atomic_write_supported= + srv_use_atomic_writes && + node->atomic_write && + my_test_if_atomic_write(node->handle, + page_size.physical()) ? + true : false; + } + } ut_a(success); @@ -3855,37 +3847,34 @@ fil_ibd_create( return(DB_ERROR); } -#ifdef UNIV_LINUX - const bool atomic_write = fil_fusionio_enable_atomic_write(file); - - if (atomic_write) { - /* This is required by FusionIO HW/Firmware */ - int ret = posix_fallocate(file, 0, size * UNIV_PAGE_SIZE); - - if (ret != 0) { - - ib::error() << - "posix_fallocate(): Failed to preallocate" - " data for file " << path - << ", desired size " - << size * UNIV_PAGE_SIZE - << " Operating system error number " << ret - << ". Check" - " that the disk is not full or a disk quota" - " exceeded. Make sure the file system supports" - " this function. Some operating system error" - " numbers are described at " REFMAN - " operating-system-error-codes.html"; - - success = false; + success= false; +#ifdef HAVE_POSIX_FALLOCATE + /* + Extend the file using posix_fallocate(). This is required by + FusionIO HW/Firmware but should also be the prefered way to extend + a file. + */ + int ret = posix_fallocate(file, 0, size * UNIV_PAGE_SIZE); + + if (ret != 0) { + ib::error() << + "posix_fallocate(): Failed to preallocate" + " data for file " << path + << ", desired size " + << size * UNIV_PAGE_SIZE + << " Operating system error number " << ret + << ". Check" + " that the disk is not full or a disk quota" + " exceeded. Make sure the file system supports" + " this function. Some operating system error" + " numbers are described at " REFMAN + " operating-system-error-codes.html"; } else { success = true; } - } else -#else - const bool atomic_write = false; -#endif /* UNIV_LINUX */ - { +#endif /* HAVE_POSIX_FALLOCATE */ + if (!success) + { success = os_file_set_size( path, file, size * UNIV_PAGE_SIZE, srv_read_only_mode); } @@ -4022,7 +4011,7 @@ fil_ibd_create( crypt_data, true); if (!fil_node_create_low( - path, size, space, false, punch_hole, atomic_write)) { + path, size, space, false, punch_hole, TRUE)) { if (crypt_data) { free(crypt_data); @@ -4234,13 +4223,6 @@ fil_ibd_open( df_dict.close(); } -#ifdef UNIV_LINUX - const bool atomic_write = !srv_use_doublewrite_buf && df_default.is_open() - && fil_fusionio_enable_atomic_write(df_default.handle()); -#else - const bool atomic_write = false; -#endif /* UNIV_LINUX */ - /* We have now checked all possible tablespace locations and have a count of how many unique files we found. If things are normal, we only found 1. */ @@ -4443,7 +4425,7 @@ fil_ibd_open( df_remote.is_open() ? df_remote.filepath() : df_dict.is_open() ? df_dict.filepath() : df_default.filepath(), 0, space, false, - true, atomic_write) == NULL) { + true, TRUE) == NULL) { err = DB_ERROR; } diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index 54070c124067e..4fa1d92a6fdcb 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -200,7 +200,6 @@ fsp_flags_to_dict_tf( bool shared_space = FSP_FLAGS_GET_SHARED(fsp_flags); bool page_compressed = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); ulint comp_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - bool atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); /* FSP_FLAGS_GET_TEMPORARY(fsp_flags) does not have an equivalent flag position in the table flags. But it would go into flags2 if @@ -208,7 +207,7 @@ fsp_flags_to_dict_tf( ulint flags = dict_tf_init(post_antelope | compact, zip_ssize, atomic_blobs, data_dir, shared_space, - page_compressed, comp_level, atomic_writes); + page_compressed, comp_level, 0); return(flags); } @@ -235,7 +234,6 @@ fsp_flags_is_valid( ulint unused = FSP_FLAGS_GET_UNUSED(flags); bool page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); const char *file; ulint line; @@ -301,11 +299,6 @@ fsp_flags_is_valid( } } - if (atomic_writes > ATOMIC_WRITES_OFF) { - GOTO_ERROR; - return (false); - } - #if UNIV_FORMAT_MAX != UNIV_FORMAT_B # error UNIV_FORMAT_MAX != UNIV_FORMAT_B, Add more validations. #endif @@ -329,8 +322,7 @@ fsp_flags_is_valid( << " is_temp: " << is_temp << " is_encryption: " << is_encryption << " page_compressed: " << page_compression - << " page_compression_level: " << page_compression_level - << " atomic_writes: " << atomic_writes; + << " page_compression_level: " << page_compression_level; return (false); } diff --git a/storage/innobase/fsp/fsp0space.cc b/storage/innobase/fsp/fsp0space.cc index 45942f58dd340..b72590e48e8c1 100644 --- a/storage/innobase/fsp/fsp0space.cc +++ b/storage/innobase/fsp/fsp0space.cc @@ -118,13 +118,6 @@ Tablespace::open_or_create(bool is_temp) break; } -#ifdef UNIV_LINUX - const bool atomic_write = fil_fusionio_enable_atomic_write( - it->m_handle); -#else - const bool atomic_write = false; -#endif - /* We can close the handle now and open the tablespace the proper way. */ it->close(); @@ -149,7 +142,7 @@ Tablespace::open_or_create(bool is_temp) /* Create the tablespace node entry for this data file. */ if (!fil_node_create( it->m_filepath, it->m_size, space, false, - atomic_write)) { + TRUE)) { err = DB_ERROR; break; diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index fa6a46890dbbe..9a9e7051403fd 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -906,24 +906,6 @@ SysTablespace::open_or_create( return(err); } -#ifdef UNIV_LINUX - /* Note: This should really be per node and not per - tablespace because a tablespace can contain multiple - files (nodes). The implication is that all files of - the tablespace should be on the same medium. */ - - it->m_atomic_write - = fil_fusionio_enable_atomic_write(it->m_handle); - - if (it->m_atomic_write && srv_use_doublewrite_buf) { - ib::info() << "FusionIO atomic IO enabled," - " disabling the double write buffer"; - - srv_use_doublewrite_buf = false; - } -#else - it->m_atomic_write = false; -#endif } if (!create_new_db && flush_lsn) { @@ -975,7 +957,7 @@ SysTablespace::open_or_create( if (!fil_node_create( it->m_filepath, it->m_size, space, it->m_type != SRV_NOT_RAW, - it->m_atomic_write, max_size)) { + TRUE, max_size)) { err = DB_ERROR; break; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index ae9363d170935..ff80d9670922b 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -249,7 +249,7 @@ values */ static ulong innobase_fast_shutdown = 1; static my_bool innobase_file_format_check = TRUE; -static my_bool innobase_use_atomic_writes = FALSE; +static my_bool innobase_use_atomic_writes = TRUE; static my_bool innobase_use_fallocate; static my_bool innobase_use_doublewrite = TRUE; static my_bool innobase_use_checksums = TRUE; @@ -791,8 +791,6 @@ ha_create_table_option innodb_table_option_list[]= /* With this option user can set zip compression level for page compression for this table*/ HA_TOPTION_NUMBER("PAGE_COMPRESSION_LEVEL", page_compression_level, 0, 1, 9, 1), - /* With this option user can enable atomic writes feature for this table */ - HA_TOPTION_ENUM("ATOMIC_WRITES", atomic_writes, "DEFAULT,ON,OFF", 0), /* With this option the user can enable encryption for the table */ HA_TOPTION_ENUM("ENCRYPTED", encryption, "DEFAULT,YES,NO", 0), /* With this option the user defines the key identifier using for the encryption */ @@ -4332,7 +4330,7 @@ innobase_init( /* Create the filespace flags. */ fsp_flags = fsp_flags_init( - univ_page_size, false, false, false, false, false, 0, ATOMIC_WRITES_DEFAULT); + univ_page_size, false, false, false, false, false, 0, 0); srv_sys_space.set_flags(fsp_flags); srv_sys_space.set_name(reserved_system_space_name); @@ -4358,7 +4356,7 @@ innobase_init( /* Create the filespace flags with the temp flag set. */ fsp_flags = fsp_flags_init( - univ_page_size, false, false, false, true, false, 0, ATOMIC_WRITES_DEFAULT); + univ_page_size, false, false, false, true, false, 0, 0); srv_tmp_space.set_flags(fsp_flags); if (!srv_tmp_space.parse_params(innobase_temp_data_file_path, false)) { @@ -4647,17 +4645,20 @@ innobase_init( " It will be removed in MariaDB 10.3."; } - srv_use_atomic_writes = (ibool) innobase_use_atomic_writes; - if (innobase_use_atomic_writes) { - fprintf(stderr, "InnoDB: using atomic writes.\n"); - /* Force doublewrite buffer off, atomic writes replace it. */ - if (srv_use_doublewrite_buf) { - fprintf(stderr, "InnoDB: Switching off doublewrite buffer " - "because of atomic writes.\n"); - innobase_use_doublewrite = srv_use_doublewrite_buf = FALSE; - } + srv_use_atomic_writes = (ibool) (innobase_use_atomic_writes && + my_may_have_atomic_write); + if (srv_use_atomic_writes && !srv_file_per_table) + { + fprintf(stderr, "InnoDB: Disabling atomic_writes as file_per_table is not used.\n"); + srv_use_atomic_writes= 0; + } - /* Force O_DIRECT on Unixes (on Windows writes are always unbuffered)*/ + if (srv_use_atomic_writes) { + fprintf(stderr, "InnoDB: using atomic writes.\n"); + /* + Force O_DIRECT on Unixes (on Windows writes are always + unbuffered) + */ #ifndef _WIN32 if (!innobase_file_flush_method || !strstr(innobase_file_flush_method, "O_DIRECT")) { @@ -13153,7 +13154,6 @@ create_table_info_t::check_table_options() { enum row_type row_format = m_form->s->row_type; ha_table_option_struct *options= m_form->s->option_struct; - atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; fil_encryption_t encrypt = (fil_encryption_t)options->encryption; if (encrypt != FIL_SPACE_ENCRYPTION_DEFAULT && !m_allow_file_per_table) { @@ -13287,19 +13287,6 @@ create_table_info_t::check_table_options() } } - /* Check atomic writes requirements */ - if (awrites == ATOMIC_WRITES_ON || - (awrites == ATOMIC_WRITES_DEFAULT && srv_use_atomic_writes)) { - if (!m_allow_file_per_table) { - push_warning( - m_thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: ATOMIC_WRITES requires" - " innodb_file_per_table."); - return "ATOMIC_WRITES"; - } - } - return NULL; } @@ -13712,7 +13699,7 @@ create_table_info_t::innobase_table_flags() options->page_compressed, options->page_compression_level == 0 ? default_compression_level : options->page_compression_level, - options->atomic_writes); + 0); if (m_use_file_per_table) { ut_ad(!m_use_shared_space); @@ -15032,7 +15019,7 @@ innobase_create_tablespace( false, /* Temporary General Tablespaces not allowed */ false, /* Page compression is not used. */ 0, /* Page compression level 0 */ - ATOMIC_WRITES_DEFAULT); /* No atomic writes yet */ + 0); tablespace.set_flags(fsp_flags); @@ -19482,8 +19469,8 @@ ha_innobase::check_if_incompatible_data( /* Changes on engine specific table options requests a rebuild of the table. */ if (param_new->page_compressed != param_old->page_compressed || - param_new->page_compression_level != param_old->page_compression_level || - param_new->atomic_writes != param_old->atomic_writes) { + param_new->page_compression_level != param_old->page_compression_level) + { return(COMPATIBLE_DATA_NO); } @@ -21950,12 +21937,13 @@ static MYSQL_SYSVAR_BOOL(doublewrite, innobase_use_doublewrite, static MYSQL_SYSVAR_BOOL(use_atomic_writes, innobase_use_atomic_writes, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, - "Prevent partial page writes, via atomic writes." - "The option is used to prevent partial writes in case of a crash/poweroff, " - "as faster alternative to doublewrite buffer." - "Currently this option works only " - "on Linux only with FusionIO device, and directFS filesystem.", - NULL, NULL, FALSE); + "Enable atomic writes, instead of using the doublewrite buffer, for files " + "on devices that supports atomic writes. " + "To use this option one must use " + "file_per_table=1, flush_method=O_DIRECT and use_fallocate=1. " + "This option only works on Linux with either FusionIO cards using " + "the directFS filesystem or with Shannon cards using any file system.", + NULL, NULL, TRUE); static MYSQL_SYSVAR_BOOL(use_fallocate, innobase_use_fallocate, PLUGIN_VAR_NOCMDARG | PLUGIN_VAR_READONLY, diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index b1bdf29c057b9..53157258d2e53 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -614,8 +614,7 @@ ha_innobase::check_if_supported_inplace_alter( ha_table_option_struct *old_options= table->s->option_struct; if (new_options->page_compressed != old_options->page_compressed || - new_options->page_compression_level != old_options->page_compression_level || - new_options->atomic_writes != old_options->atomic_writes) { + new_options->page_compression_level != old_options->page_compression_level) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index be6f6b1d9a599..ca3951dd3090e 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -1003,7 +1003,7 @@ dict_tf_set( bool shared_space, bool page_compressed, ulint page_compression_level, - ulint atomic_writes); + ulint not_used); /** Initialize a dict_table_t::flags pointer. @param[in] compact, Table uses Compact or greater @@ -1021,7 +1021,7 @@ dict_tf_init( bool shared_space, bool page_compressed, ulint page_compression_level, - ulint atomic_writes); + ulint not_used); /** Convert a 32 bit integer table flags to the 32 bit FSP Flags. Fsp Flags are written into the tablespace header at the offset diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index b99cb421ab260..c7d553f5daa14 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -664,7 +664,6 @@ dict_tf_is_valid( ulint unused = DICT_TF_GET_UNUSED(flags); bool page_compression = DICT_TF_GET_PAGE_COMPRESSION(flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(flags); bool flags_corrupt = false; /* Make sure there are no bits that we do not know about. */ @@ -711,12 +710,6 @@ dict_tf_is_valid( } } - if (atomic_writes) { - - if(atomic_writes > ATOMIC_WRITES_OFF) { - flags_corrupt = true; - } - } /* HAS_DATA_DIR and SHARED_SPACE are mutually exclusive. */ if (data_dir && shared_space) { @@ -734,7 +727,6 @@ dict_tf_is_valid( << " zip_ssize:" << zip_ssize << " page_compression:" << page_compression << " page_compression_level:" << page_compression_level - << " atomic_writes:" << atomic_writes << " shared_space:" << shared_space; return (false); } else { @@ -789,9 +781,6 @@ dict_sys_tables_type_validate( ulint unused = DICT_TF_GET_UNUSED(type); bool page_compression = DICT_TF_GET_PAGE_COMPRESSION(type); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(type); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(type); - - ut_a(atomic_writes <= ATOMIC_WRITES_OFF); /* The low order bit of SYS_TABLES.TYPE is always set to 1. If the format is UNIV_FORMAT_B or higher, this field is the same @@ -875,13 +864,6 @@ dict_sys_tables_type_validate( } } - /* Validate that the atomic writes number is within allowed range. */ - if (atomic_writes > ATOMIC_WRITES_OFF) { - ib::error() << "SYS_TABLES::TYPE=" << type - << " atomic_writes:" << atomic_writes; - return(ULINT_UNDEFINED); - } - /* Return the validated SYS_TABLES.TYPE. */ return(type); } @@ -949,11 +931,10 @@ dict_table_get_format( @param[in] format File Format @param[in] zip_ssize Zip Shift Size @param[in] use_data_dir Table uses DATA DIRECTORY -@param[in] atomic_writes Does table use atomic writes @param[in] shared_space Table uses a General Shared Tablespace @param[in] page_compressed Table uses page compression @param[in] page_compression_level Page compression level -@param[in] atomic_writes Table uses atomic writes */ +@param[in] not_used For future */ UNIV_INLINE void dict_tf_set( @@ -965,7 +946,7 @@ dict_tf_set( bool shared_space, bool page_compressed, ulint page_compression_level, - ulint atomic_writes) + ulint not_used) { switch (format) { case REC_FORMAT_REDUNDANT: @@ -1005,11 +986,6 @@ dict_tf_set( ut_ad(dict_tf_get_page_compression(*flags) == TRUE); ut_ad(dict_tf_get_page_compression_level(*flags) == page_compression_level); } - - if (atomic_writes) { - *flags |= (atomic_writes << DICT_TF_POS_ATOMIC_WRITES); - ut_a(dict_tf_get_atomic_writes(*flags) == atomic_writes); - } } /** Initialize a dict_table_t::flags pointer. @@ -1020,7 +996,7 @@ dict_tf_set( @param[in] shared_space Table uses a General Shared Tablespace @param[in] page_compression Table uses page compression @param[in] page_compression_level used compression level -@param[in] atomic_writes Table atomic writes option */ +@param[in] not_used For future */ UNIV_INLINE ulint dict_tf_init( @@ -1031,7 +1007,7 @@ dict_tf_init( bool shared_space, bool page_compressed, ulint page_compression_level, - ulint atomic_writes) + ulint not_used) { ulint flags = 0; @@ -1065,11 +1041,6 @@ dict_tf_init( ut_ad(dict_tf_get_page_compression_level(flags) == page_compression_level); } - if (atomic_writes) { - flags |= (atomic_writes << DICT_TF_POS_ATOMIC_WRITES); - ut_a(dict_tf_get_atomic_writes(flags) == atomic_writes); - } - return(flags); } @@ -1097,13 +1068,12 @@ dict_sys_tables_type_to_tf( flags = redundant ? 0 : 1; /* ZIP_SSIZE, ATOMIC_BLOBS, DATA_DIR, PAGE_COMPRESSION, - PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES are the same. */ + PAGE_COMPRESSION_LEVEL are the same. */ flags |= type & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES | DICT_TF_MASK_SHARED_SPACE); ut_ad(!DICT_TF_GET_ZIP_SSIZE(flags) || DICT_TF_HAS_ATOMIC_BLOBS(flags)); @@ -1134,13 +1104,12 @@ dict_tf_to_sys_tables_type( type = 1; /* ZIP_SSIZE, ATOMIC_BLOBS, DATA_DIR, PAGE_COMPRESSION, - PAGE_COMPRESSION_LEVEL, ATOMIC_WRITES are the same. */ + PAGE_COMPRESSION_LEVEL are the same. */ type |= flags & (DICT_TF_MASK_ZIP_SSIZE | DICT_TF_MASK_ATOMIC_BLOBS | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES | DICT_TF_MASK_SHARED_SPACE); return(type); diff --git a/storage/innobase/include/dict0pagecompress.h b/storage/innobase/include/dict0pagecompress.h index 19a2a6c52f3d8..f8873aec9659e 100644 --- a/storage/innobase/include/dict0pagecompress.h +++ b/storage/innobase/include/dict0pagecompress.h @@ -67,26 +67,6 @@ dict_tf_verify_flags( ulint fsp_flags) /*!< in: fil_space_t::flags */ __attribute__((const)); -/********************************************************************//** -Extract the atomic writes flag from table flags. -@return true if atomic writes are used, false if not used */ -UNIV_INLINE -atomic_writes_t -dict_tf_get_atomic_writes( -/*======================*/ - ulint flags) /*!< in: flags */ - __attribute__((const)); - -/********************************************************************//** -Check whether the table uses the atomic writes. -@return true if atomic writes is used, false if not */ -UNIV_INLINE -atomic_writes_t -dict_table_get_atomic_writes( -/*=========================*/ - const dict_table_t* table); /*!< in: table */ - - #ifndef UNIV_NONINL #include "dict0pagecompress.ic" #endif diff --git a/storage/innobase/include/dict0pagecompress.ic b/storage/innobase/include/dict0pagecompress.ic index 811976434a83b..05a26f007110e 100644 --- a/storage/innobase/include/dict0pagecompress.ic +++ b/storage/innobase/include/dict0pagecompress.ic @@ -41,7 +41,6 @@ dict_tf_verify_flags( ulint data_dir = DICT_TF_HAS_DATA_DIR(table_flags); ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); - ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); @@ -49,7 +48,6 @@ dict_tf_verify_flags( ulint fsp_unused = FSP_FLAGS_GET_UNUSED(fsp_flags); ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); - ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", return(ULINT_UNDEFINED);); @@ -97,16 +95,6 @@ dict_tf_verify_flags( return (FALSE); } - if (atomic_writes != fsp_atomic_writes) { - fprintf(stderr, - "InnoDB: Error: table flags has atomic writes %ld" - " in the data dictionary\n" - "InnoDB: but the flags in file has atomic_writes %ld\n", - atomic_writes, fsp_atomic_writes); - - return (FALSE); - } - return(TRUE); } @@ -165,27 +153,3 @@ dict_table_is_page_compressed( { return (dict_tf_get_page_compression(table->flags)); } - -/********************************************************************//** -Extract the atomic writes flag from table flags. -@return enumerated value of atomic writes */ -UNIV_INLINE -atomic_writes_t -dict_tf_get_atomic_writes( -/*======================*/ - ulint flags) /*!< in: flags */ -{ - return((atomic_writes_t)DICT_TF_GET_ATOMIC_WRITES(flags)); -} - -/********************************************************************//** -Check whether the table uses the atomic writes. -@return enumerated value of atomic writes */ -UNIV_INLINE -atomic_writes_t -dict_table_get_atomic_writes( -/*=========================*/ - const dict_table_t* table) /*!< in: table */ -{ - return ((atomic_writes_t)dict_tf_get_atomic_writes(table->flags)); -} diff --git a/storage/innobase/include/dict0types.h b/storage/innobase/include/dict0types.h index ae002dd9487fb..b0623f82bb7a1 100644 --- a/storage/innobase/include/dict0types.h +++ b/storage/innobase/include/dict0types.h @@ -80,13 +80,6 @@ enum ib_quiesce_t { QUIESCE_COMPLETE /*!< All done */ }; -/** Enum values for atomic_writes table option */ -typedef enum { - ATOMIC_WRITES_DEFAULT = 0, - ATOMIC_WRITES_ON = 1, - ATOMIC_WRITES_OFF = 2 -} atomic_writes_t; - #ifndef UNIV_INNOCHECKSUM typedef ib_mutex_t DictSysMutex; #endif /* !UNIV_INNOCHECKSUM */ diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 30d79a52b4d8f..0197dfaabb446 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -190,6 +190,14 @@ struct fil_space_t { /** True if we have already printed compression failure */ bool printed_compression_failure; + /** True if page 0 of tablespace is read */ + bool read_page0; + + /** True if we have tested if this filespace supports atomic writes */ + bool atomic_write_tested; + /** True if the device this filespace is on supports atomic writes */ + bool atomic_write_supported; + /** Release the reserved free extents. @param[in] n_reserved number of reserved extents */ void release_free_extents(ulint n_reserved); @@ -244,7 +252,7 @@ struct fil_node_t { /** block size to use for punching holes */ ulint block_size; - /** whether atomic write is enabled for this file */ + /** whether this file could use atomic write (data file) */ bool atomic_write; /** FIL_NODE_MAGIC_N */ @@ -663,7 +671,7 @@ MY_ATTRIBUTE((warn_unused_result, pure)); @param[in] size file size in entire database blocks @param[in,out] space tablespace from fil_space_create() @param[in] is_raw whether this is a raw device or partition -@param[in] atomic_write true if atomic write enabled +@param[in] atomic_write true if atomic write could be enabled @param[in] max_pages maximum number of pages in file, ULINT_MAX means the file size is unlimited. @return pointer to the file name @@ -1730,15 +1738,6 @@ fil_names_clear( lsn_t lsn, bool do_write); -#ifdef UNIV_LINUX -/** -Try and enable FusionIO atomic writes. -@param[in] file OS file handle -@return true if successful */ -bool -fil_fusionio_enable_atomic_write(os_file_t file); -#endif /* UNIV_LINUX */ - /** Note that the file system where the file resides doesn't support PUNCH HOLE @param[in,out] node Node to set */ void fil_no_punch_hole(fil_node_t* node); diff --git a/storage/innobase/include/fil0pagecompress.h b/storage/innobase/include/fil0pagecompress.h index d4cc54c7b2a8b..e65d34911555c 100644 --- a/storage/innobase/include/fil0pagecompress.h +++ b/storage/innobase/include/fil0pagecompress.h @@ -62,7 +62,7 @@ Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. @return atomic write table option value */ UNIV_INLINE -atomic_writes_t +bool fil_space_get_atomic_writes( /*=========================*/ ulint id); /*!< in: space id */ diff --git a/storage/innobase/include/fsp0file.h b/storage/innobase/include/fsp0file.h index 83aa370abf09d..82d086bcf7a8c 100644 --- a/storage/innobase/include/fsp0file.h +++ b/storage/innobase/include/fsp0file.h @@ -65,7 +65,6 @@ class Datafile { m_is_valid(), m_first_page_buf(), m_first_page(), - m_atomic_write(), m_last_os_error(), m_file_info(), m_encryption_key(NULL), @@ -91,7 +90,6 @@ class Datafile { m_is_valid(), m_first_page_buf(), m_first_page(), - m_atomic_write(), m_last_os_error(), m_file_info(), m_encryption_key(NULL), @@ -115,7 +113,6 @@ class Datafile { m_is_valid(file.m_is_valid), m_first_page_buf(), m_first_page(), - m_atomic_write(file.m_atomic_write), m_last_os_error(), m_file_info(), m_encryption_key(NULL), @@ -183,8 +180,6 @@ class Datafile { /* Do not copy crypt info it is read from first page */ m_crypt_info = NULL; - m_atomic_write = file.m_atomic_write; - return(*this); } @@ -475,9 +470,6 @@ class Datafile { /** Pointer to the first page held in the buffer above */ byte* m_first_page; - /** true if atomic writes enabled for this file */ - bool m_atomic_write; - protected: /** Last OS error received so it can be reported if needed. */ ulint m_last_os_error; diff --git a/storage/innobase/include/fsp0fsp.h b/storage/innobase/include/fsp0fsp.h index f503580a1a72c..6ddcec78a0198 100644 --- a/storage/innobase/include/fsp0fsp.h +++ b/storage/innobase/include/fsp0fsp.h @@ -708,7 +708,7 @@ fsp_flags_init( bool is_temporary, bool page_compression, ulint page_compression_level, - ulint atomic_writes, + ulint not_used, bool is_encrypted = false); /** Convert a 32 bit integer tablespace flags to the 32 bit table flags. diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index c675c6302a6e1..31317969bf01a 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -184,7 +184,7 @@ fsp_flags_set_page_size( @param[in] is_encrypted This tablespace is encrypted. @param[in] page_compressed Table uses page compression @param[in] page_compression_level Page compression level -@param[in] atomic_writes Table uses atomic writes +@param[in] not_used For future @@return tablespace flags after initialization */ UNIV_INLINE ulint @@ -196,7 +196,7 @@ fsp_flags_init( bool is_temporary, bool page_compression, ulint page_compression_level, - ulint atomic_writes, + ulint not_used, bool is_encrypted) { ut_ad(page_size.physical() <= page_size.logical()); @@ -247,12 +247,6 @@ fsp_flags_init( flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, page_compression_level); } - /* In addition, tablespace flags also contain flag if atomic writes - is used for this table */ - if (atomic_writes) { - flags |= FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomic_writes); - } - ut_ad(fsp_flags_is_valid(flags)); return(flags); diff --git a/storage/innobase/include/fsp0pagecompress.h b/storage/innobase/include/fsp0pagecompress.h index 44bdddfa3bfc5..9038aa0fdef64 100644 --- a/storage/innobase/include/fsp0pagecompress.h +++ b/storage/innobase/include/fsp0pagecompress.h @@ -68,15 +68,6 @@ fsp_flags_get_page_compression_level( /*=================================*/ ulint flags); /*!< in: tablespace flags */ -/********************************************************************//** -Determine the tablespace is using atomic writes from dict_table_t::flags. -@return true if atomic writes is used, false if not */ -UNIV_INLINE -atomic_writes_t -fsp_flags_get_atomic_writes( -/*========================*/ - ulint flags); /*!< in: tablespace flags */ - #ifndef UNIV_NONINL #include "fsp0pagecompress.ic" #endif diff --git a/storage/innobase/include/fsp0pagecompress.ic b/storage/innobase/include/fsp0pagecompress.ic index a3971da6772f5..0915fae4b920f 100644 --- a/storage/innobase/include/fsp0pagecompress.ic +++ b/storage/innobase/include/fsp0pagecompress.ic @@ -49,17 +49,6 @@ fsp_flags_get_page_compression_level( return(FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags)); } -/********************************************************************//** -Determine the tablespace is using atomic writes from dict_table_t::flags. -@return true if atomic writes is used, false if not */ -UNIV_INLINE -atomic_writes_t -fsp_flags_get_atomic_writes( -/*========================*/ - ulint flags) /*!< in: tablespace flags */ -{ - return((atomic_writes_t)FSP_FLAGS_GET_ATOMIC_WRITES(flags)); -} /*******************************************************************//** Find out wheather the page is index page or not @@ -186,25 +175,28 @@ fil_get_compression_alg_name( /*******************************************************************//** Returns the atomic writes flag of the space, or false if the space is not using atomic writes. The tablespace must be cached in the memory cache. -@return atomic writes table option value */ +@return 1 if atomic writes can be used for the file */ UNIV_INLINE -atomic_writes_t +bool fil_space_get_atomic_writes( /*========================*/ ulint id) /*!< in: space id */ { - ulint flags; + struct fil_space_t* space; + bool ret= 0; - flags = fil_space_get_flags(id); + ut_ad(fil_system); - if (flags && flags != ULINT_UNDEFINED) { + mutex_enter(&fil_system->mutex); - return((atomic_writes_t)fsp_flags_get_atomic_writes(flags)); - } + if ((space = fil_space_get_by_id(id))) + ret= space->atomic_write_supported; - return((atomic_writes_t)0); + mutex_exit(&fil_system->mutex); + return(ret); } + /*******************************************************************//** Find out wheather the page is page compressed with lzo method @return true if page is page compressed with lzo method, false if not */ diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 5755707d710ef..58cc1ff8d3ed9 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -79,13 +79,6 @@ Created 10/21/1995 Heikki Tuuri bool innodb_calling_exit; #endif /* UNIV_DEBUG */ -#if defined(UNIV_LINUX) && defined(HAVE_SYS_IOCTL_H) -# include -# ifndef DFS_IOCTL_ATOMIC_WRITE_SET -# define DFS_IOCTL_ATOMIC_WRITE_SET _IOW(0x95, 2, uint) -# endif -#endif - #if defined(UNIV_LINUX) && defined(HAVE_SYS_STATVFS_H) #include #endif @@ -3310,26 +3303,6 @@ os_file_create_simple_func( } #endif /* USE_FILE_LOCK */ - /* If we have proper file handle and atomic writes should be used, - try to set atomic writes and if that fails when creating a new - table, produce a error. If atomic writes are used on existing - file, ignore error and use traditional writes for that file */ - /* JAN: TODO: ATOMIC WRITES - if (file != -1 - && (awrites == ATOMIC_WRITES_ON || - (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) - && !os_file_set_atomic_writes(name, file)) { - if (create_mode == OS_FILE_CREATE) { - fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); - close(file); - os_file_delete_if_exists_func(name); - *success = FALSE; - file = -1; - } - } - */ - - return(file); } @@ -3682,24 +3655,6 @@ os_file_create_func( } #endif /* USE_FILE_LOCK */ - /* If we have proper file handle and atomic writes should be used, - try to set atomic writes and if that fails when creating a new - table, produce a error. If atomic writes are used on existing - file, ignore error and use traditional writes for that file */ - /* JAN: TODO: ATOMIC WRITES - if (file != -1 && type == OS_DATA_FILE - && (awrites == ATOMIC_WRITES_ON || - (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) - && !os_file_set_atomic_writes(name, file)) { - if (create_mode == OS_FILE_CREATE) { - fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); - close(file); - os_file_delete_if_exists_func(name); - *success = FALSE; - file = -1; - } - } - */ return(file); } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 708356512be08..56576c2f29e44 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -675,13 +675,6 @@ srv_undo_tablespace_open( os_offset_t size; fil_space_t* space; -#ifdef UNIV_LINUX - const bool atomic_write = !srv_use_doublewrite_buf - && fil_fusionio_enable_atomic_write(fh); -#else - const bool atomic_write = false; -#endif - size = os_file_get_size(fh); ut_a(size != (os_offset_t) -1); @@ -699,7 +692,7 @@ srv_undo_tablespace_open( /* Set the compressed page size to 0 (non-compressed) */ flags = fsp_flags_init( - univ_page_size, false, false, false, false, false, 0, ATOMIC_WRITES_DEFAULT); + univ_page_size, false, false, false, false, false, 0, 0); space = fil_space_create( undo_name, space_id, flags, FIL_TYPE_TABLESPACE, NULL, true); @@ -713,7 +706,7 @@ srv_undo_tablespace_open( the unit has been scaled to pages and page number is always 32 bits. */ if (fil_node_create( - name, (ulint) n_pages, space, false, atomic_write)) { + name, (ulint) n_pages, space, false, TRUE)) { err = DB_SUCCESS; } diff --git a/support-files/build-tags b/support-files/build-tags index 87f320ba5f536..03b243ee8cc6e 100755 --- a/support-files/build-tags +++ b/support-files/build-tags @@ -8,9 +8,9 @@ then echo client storage dbug libmysql sql-common \ sql extra mysys mysys_ssl strings regex pcre vio include \ tools unittest plugin libmysqld | \ - xargs -n1 git ls-files | \ + xargs -n1 git ls-files | grep -v '\.jar$' | \ xargs etags -o TAGS --append else - find . -type f | + find . -type f ! -name "*.jar" | xargs etags -o TAGS --append fi From 1afb17047a61d7666d4c3d6e5fae97ec526693ba Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 20 Dec 2016 18:06:49 +0200 Subject: [PATCH 54/74] Fixed bugs found by mysql-test-run: - privilege_table_io.test didn't properly reset roles_mapping - Fixed memory allocation problem with CHECK CONSTRAINT, found when running --valgrind main.check_constraint --- mysql-test/suite/perfschema/r/privilege_table_io.result | 1 + mysql-test/suite/perfschema/t/privilege_table_io.test | 1 + sql/table.cc | 3 ++- 3 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/perfschema/r/privilege_table_io.result b/mysql-test/suite/perfschema/r/privilege_table_io.result index f1f0946cb901d..aeacd0ba0112b 100644 --- a/mysql-test/suite/perfschema/r/privilege_table_io.result +++ b/mysql-test/suite/perfschema/r/privilege_table_io.result @@ -15,6 +15,7 @@ optimize table mysql.proxies_priv; optimize table mysql.tables_priv; optimize table mysql.procs_priv; optimize table mysql.servers; +optimize table mysql.roles_mapping; update performance_schema.setup_consumers set enabled='YES'; update performance_schema.setup_objects set enabled='YES' where object_type='TABLE' and object_schema= 'mysql'; diff --git a/mysql-test/suite/perfschema/t/privilege_table_io.test b/mysql-test/suite/perfschema/t/privilege_table_io.test index 35c49bf33fb03..8f06be5075e3b 100644 --- a/mysql-test/suite/perfschema/t/privilege_table_io.test +++ b/mysql-test/suite/perfschema/t/privilege_table_io.test @@ -24,6 +24,7 @@ optimize table mysql.proxies_priv; optimize table mysql.tables_priv; optimize table mysql.procs_priv; optimize table mysql.servers; +optimize table mysql.roles_mapping; --enable_result_log # Start recording events diff --git a/sql/table.cc b/sql/table.cc index 6d11ac3d3eb98..5c9d480594314 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -3110,7 +3110,8 @@ enum open_frm_error open_table_from_share(THD *thd, TABLE_SHARE *share, share->default_expressions +1)* sizeof(Field*)), &check_constraint_ptr, - (uint) ((share->table_check_constraints + 1)* + (uint) ((share->table_check_constraints + + share->field_check_constraints + 1)* sizeof(Virtual_column_info*)), NullS)) goto err; From 1628a2ae27cfc8b96e4b50d85d430955625e2d88 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 27 Dec 2016 01:30:20 +0200 Subject: [PATCH 55/74] Fixed issues found by buildbot - MDEV-11621 rpl.rpl_gtid_stop_start fails sporadically in buildbot - MDEV-11620 rpl.rpl_upgrade_master_info fails sporadically in buildbot The issue above was probably that the build machine was overworked and the shutdown took longer than 30 resp 10 seconds, which caused MyISAM tables to be marked as crashed. Fixed by flushing myisam tables before doing a forced shutdown/kill. I also increased timeout for forced shutdown from 10 seconds to 60 seconds to fix other possible issues on slow machines. Fixed also some compiler warnings --- client/mysqltest.cc | 1 + mysql-test/include/rpl_stop_server.inc | 4 ++-- mysql-test/suite/rpl/r/rpl_gtid_stop_start.result | 6 ++++++ mysql-test/suite/rpl/r/rpl_upgrade_master_info.result | 5 +++++ mysql-test/suite/rpl/t/rpl_gtid_stop_start.test | 6 ++++++ mysql-test/suite/rpl/t/rpl_upgrade_master_info.test | 5 +++++ sql/compat56.cc | 6 +++--- sql/item_jsonfunc.cc | 2 +- sql/log_event.cc | 8 ++++---- storage/innobase/row/row0ftsort.cc | 4 ---- storage/mroonga/vendor/groonga/lib/proc.c | 2 +- 11 files changed, 34 insertions(+), 15 deletions(-) diff --git a/client/mysqltest.cc b/client/mysqltest.cc index f0c56bc639c74..702098fc5bc7c 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -5156,6 +5156,7 @@ uint get_errcode_from_name(const char *error_name, const char *error_end) handler_error_names))) return tmp; die("Unknown SQL error name '%s'", error_name); + return 0; // Keep compiler happy } const char *unknown_error= ""; diff --git a/mysql-test/include/rpl_stop_server.inc b/mysql-test/include/rpl_stop_server.inc index e1f8839dd69d4..978cfec18856a 100644 --- a/mysql-test/include/rpl_stop_server.inc +++ b/mysql-test/include/rpl_stop_server.inc @@ -47,8 +47,8 @@ if ($rpl_debug) --exec echo "wait" > $MYSQLTEST_VARDIR/tmp/mysqld.$rpl_server_number.expect # Send shutdown to the connected server and give -# it 10 seconds to die before zapping it -shutdown_server 10; +# it 60 seconds to die before zapping it +shutdown_server 60; --source include/wait_until_disconnected.inc diff --git a/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result b/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result index db2f40fd495cf..3ba178223192d 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result @@ -55,6 +55,7 @@ INSERT INTO t1 VALUES(5); include/save_master_gtid.inc connection server_2; include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; a 1 @@ -87,6 +88,7 @@ a connection server_1; INSERT INTO t1 VALUES (7); connection server_2; +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; a 1 @@ -116,6 +118,7 @@ include/start_slave.inc connection server_1; INSERT INTO t1 VALUES (8); connection server_2; +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; a 1 @@ -142,6 +145,7 @@ Error 1286 Unknown storage engine 'InnoDB' connection server_1; INSERT INTO t1 VALUES (9); connection server_2; +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; a 1 @@ -171,6 +175,7 @@ domain_id COUNT(*) connection server_1; INSERT INTO t1 VALUES (11); connection server_2; +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT domain_id, COUNT(*) FROM mysql.gtid_slave_pos GROUP BY domain_id; domain_id COUNT(*) 0 2 @@ -180,6 +185,7 @@ connection server_1; INSERT INTO t1 VALUES (12); INSERT INTO t1 VALUES (13); connection server_2; +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT domain_id, COUNT(*) FROM mysql.gtid_slave_pos GROUP BY domain_id; domain_id COUNT(*) 0 2 diff --git a/mysql-test/suite/rpl/r/rpl_upgrade_master_info.result b/mysql-test/suite/rpl/r/rpl_upgrade_master_info.result index 3e737267fbd03..ffa041efd2718 100644 --- a/mysql-test/suite/rpl/r/rpl_upgrade_master_info.result +++ b/mysql-test/suite/rpl/r/rpl_upgrade_master_info.result @@ -14,6 +14,7 @@ connection slave; CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; include/start_slave.inc include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1; a 1 @@ -27,6 +28,7 @@ connection slave; CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; include/start_slave.inc include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; a 1 @@ -41,6 +43,7 @@ connection slave; CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; include/start_slave.inc include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; a 1 @@ -56,6 +59,7 @@ connection slave; CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; include/start_slave.inc include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; a 1 @@ -72,6 +76,7 @@ connection slave; CHANGE MASTER TO master_host='127.0.0.1', master_port=SERVER_MYPORT_1; include/start_slave.inc include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; a 1 diff --git a/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test b/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test index b57714aaa5725..65b13b57f2d60 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test @@ -92,6 +92,7 @@ INSERT INTO t1 VALUES(5); --connection server_2 --source include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; --echo *** Test that @@gtid_slave_pos and @@gtid_current_pos are correctly loaded even if slave threads have not started. *** @@ -136,6 +137,7 @@ INSERT INTO t1 VALUES (7); --connection server_2 --sync_with_master +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; # Now we restart the slave server. When it restarts, there is nothing new @@ -177,6 +179,7 @@ INSERT INTO t1 VALUES (8); --connection server_2 --sync_with_master +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; --source include/stop_slave.inc @@ -210,6 +213,7 @@ INSERT INTO t1 VALUES (9); --connection server_2 --sync_with_master +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; # Put things back as they were. @@ -248,6 +252,7 @@ INSERT INTO t1 VALUES (11); --connection server_2 --sync_with_master +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT domain_id, COUNT(*) FROM mysql.gtid_slave_pos GROUP BY domain_id; --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect @@ -270,6 +275,7 @@ INSERT INTO t1 VALUES (13); --connection server_2 --sync_with_master +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT domain_id, COUNT(*) FROM mysql.gtid_slave_pos GROUP BY domain_id; diff --git a/mysql-test/suite/rpl/t/rpl_upgrade_master_info.test b/mysql-test/suite/rpl/t/rpl_upgrade_master_info.test index e81e7c0d71499..42b375c4579ab 100644 --- a/mysql-test/suite/rpl/t/rpl_upgrade_master_info.test +++ b/mysql-test/suite/rpl/t/rpl_upgrade_master_info.test @@ -29,6 +29,7 @@ INSERT INTO t1 VALUES (1); eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; --source include/start_slave.inc --source include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1; --source include/stop_slave.inc @@ -54,6 +55,7 @@ INSERT INTO t1 VALUES (2); eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; --source include/start_slave.inc --source include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; --source include/stop_slave.inc @@ -79,6 +81,7 @@ INSERT INTO t1 VALUES (3); eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; --source include/start_slave.inc --source include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; --source include/stop_slave.inc @@ -104,6 +107,7 @@ INSERT INTO t1 VALUES (4); eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; --source include/start_slave.inc --source include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; --source include/stop_slave.inc @@ -129,6 +133,7 @@ INSERT INTO t1 VALUES (5); eval CHANGE MASTER TO master_host='127.0.0.1', master_port=$SERVER_MYPORT_1; --source include/start_slave.inc --source include/sync_with_master_gtid.inc +FLUSH NO_WRITE_TO_BINLOG TABLES; SELECT * FROM t1 ORDER BY a; --source include/stop_slave.inc diff --git a/sql/compat56.cc b/sql/compat56.cc index 3bd6b21a15413..704d1db9a989f 100644 --- a/sql/compat56.cc +++ b/sql/compat56.cc @@ -65,7 +65,7 @@ void TIME_from_longlong_time_packed(MYSQL_TIME *ltime, longlong tmp) long hms; if ((ltime->neg= (tmp < 0))) tmp= -tmp; - hms= MY_PACKED_TIME_GET_INT_PART(tmp); + hms= (long) MY_PACKED_TIME_GET_INT_PART(tmp); ltime->year= (uint) 0; ltime->month= (uint) 0; ltime->day= (uint) 0; @@ -264,11 +264,11 @@ void TIME_from_longlong_datetime_packed(MYSQL_TIME *ltime, longlong tmp) ltime->day= ymd % (1 << 5); ltime->month= ym % 13; - ltime->year= ym / 13; + ltime->year= (uint) (ym / 13); ltime->second= hms % (1 << 6); ltime->minute= (hms >> 6) % (1 << 6); - ltime->hour= (hms >> 12); + ltime->hour= (uint) (hms >> 12); ltime->time_type= MYSQL_TIMESTAMP_DATETIME; } diff --git a/sql/item_jsonfunc.cc b/sql/item_jsonfunc.cc index 9b9490a0cbca7..a500170ead092 100644 --- a/sql/item_jsonfunc.cc +++ b/sql/item_jsonfunc.cc @@ -2307,7 +2307,7 @@ bool Item_func_json_search::fix_fields(THD *thd, Item **ref) } -static const uint SQR_MAX_BLOB_WIDTH= sqrt(MAX_BLOB_WIDTH); +static const uint SQR_MAX_BLOB_WIDTH= (uint) sqrt(MAX_BLOB_WIDTH); void Item_func_json_search::fix_length_and_dec() { diff --git a/sql/log_event.cc b/sql/log_event.cc index 85dce217743e7..0af348ab4537e 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -1794,7 +1794,7 @@ int Log_event::read_log_event(IO_CACHE* file, String* packet, if (fdle->crypto_data.scheme) { uchar iv[BINLOG_IV_LENGTH]; - fdle->crypto_data.set_iv(iv, my_b_tell(file) - data_len); + fdle->crypto_data.set_iv(iv, (uint32) (my_b_tell(file) - data_len)); char *newpkt= (char*)my_malloc(data_len + ev_offset + 1, MYF(MY_WME)); if (!newpkt) @@ -7058,7 +7058,7 @@ Binlog_checkpoint_log_event::Binlog_checkpoint_log_event( uint8 header_size= description_event->common_header_len; uint8 post_header_len= description_event->post_header_len[BINLOG_CHECKPOINT_EVENT-1]; - if (event_len < header_size + post_header_len || + if (event_len < (uint) header_size + (uint) post_header_len || post_header_len < BINLOG_CHECKPOINT_HEADER_LEN) return; buf+= header_size; @@ -7096,7 +7096,7 @@ Gtid_log_event::Gtid_log_event(const char *buf, uint event_len, { uint8 header_size= description_event->common_header_len; uint8 post_header_len= description_event->post_header_len[GTID_EVENT-1]; - if (event_len < header_size + post_header_len || + if (event_len < (uint) header_size + (uint) post_header_len || post_header_len < GTID_HEADER_LEN) return; @@ -7421,7 +7421,7 @@ Gtid_list_log_event::Gtid_list_log_event(const char *buf, uint event_len, uint32 val; uint8 header_size= description_event->common_header_len; uint8 post_header_len= description_event->post_header_len[GTID_LIST_EVENT-1]; - if (event_len < header_size + post_header_len || + if (event_len < (uint) header_size + (uint) post_header_len || post_header_len < GTID_LIST_HEADER_LEN) return; diff --git a/storage/innobase/row/row0ftsort.cc b/storage/innobase/row/row0ftsort.cc index 0d216d584d74d..3179c63f162f3 100644 --- a/storage/innobase/row/row0ftsort.cc +++ b/storage/innobase/row/row0ftsort.cc @@ -793,7 +793,6 @@ fts_parallel_tokenization( mem_heap_t* blob_heap = NULL; fts_doc_t doc; dict_table_t* table = psort_info->psort_common->new_table; - dict_field_t* idx_field; fts_tokenize_ctx_t t_ctx; ulint retried = 0; dberr_t error = DB_SUCCESS; @@ -822,9 +821,6 @@ fts_parallel_tokenization( doc.charset = fts_index_get_charset( psort_info->psort_common->dup->index); - idx_field = dict_index_get_nth_field( - psort_info->psort_common->dup->index, 0); - block = psort_info->merge_block; crypt_block = psort_info->crypt_block; crypt_data = psort_info->psort_common->crypt_data; diff --git a/storage/mroonga/vendor/groonga/lib/proc.c b/storage/mroonga/vendor/groonga/lib/proc.c index 86b0fd58c0211..ba858de6f2a0c 100644 --- a/storage/mroonga/vendor/groonga/lib/proc.c +++ b/storage/mroonga/vendor/groonga/lib/proc.c @@ -713,7 +713,7 @@ grn_select_drilldown(grn_ctx *ctx, grn_obj *table, { uint32_t i; for (i = 0; i < n_keys; i++) { - grn_table_group_result g = {NULL, 0, 0, 1, GRN_TABLE_GROUP_CALC_COUNT, 0}; + grn_table_group_result g = {NULL, 0, 0, 1, GRN_TABLE_GROUP_CALC_COUNT, 0, 0 ,0}; uint32_t n_hits; int offset; int limit; From 7454087d071d1c1954c51f08ccd7a29682cd9da8 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 27 Dec 2016 17:18:10 +0200 Subject: [PATCH 56/74] Revert "bugfix: UPDATE and virtual BLOBs" This reverts commit f73bdb685d77b944f4565e0a97778faa30999145. --- sql/field.h | 3 -- sql/sql_class.h | 99 +---------------------------------------------- sql/sql_insert.cc | 36 ++--------------- sql/sql_insert.h | 1 + sql/sql_load.cc | 6 --- sql/sql_update.cc | 18 --------- 6 files changed, 7 insertions(+), 156 deletions(-) diff --git a/sql/field.h b/sql/field.h index fd62218f1444f..0d278968eb5a6 100644 --- a/sql/field.h +++ b/sql/field.h @@ -3339,9 +3339,6 @@ class Field_blob :public Field_longstr { uint max_packed_col_length(uint max_length); void free() { value.free(); } inline void clear_temporary() { bzero((uchar*) &value, sizeof(value)); } - inline bool owns_ptr(uchar* p) const { return p == (uchar*)value.ptr(); } - inline void own_value_ptr() - { value.reset((char*)get_ptr(), get_length(), get_length(), value.charset()); } uint size_of() const { return sizeof(*this); } bool has_charset(void) const { return charset() == &my_charset_bin ? FALSE : TRUE; } diff --git a/sql/sql_class.h b/sql/sql_class.h index ef10d7e40530e..a55e17393186d 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -201,99 +201,6 @@ typedef struct st_user_var_events bool unsigned_flag; } BINLOG_USER_VAR_EVENT; - -/* - When updating a table with virtual BLOB columns, the following might happen: - - an old record is read from the table, it has no vcol blob. - - update_virtual_fields() is run, vcol blob gets its value into the - record. But only a pointer to the value is in the table->record[0], - the value is in Field_blob::value String (or, it can be elsewhere!) - - store_record(table,record[1]), old record now is in record[1] - - fill_record() prepares new values in record[0], vcol blob is updated, - new value replaces the old one in the Field_blob::value - - now both record[1] and record[0] have a pointer that points to the - *new* vcol blob value. Or record[1] has a pointer to nowhere if - Field_blob::value had to realloc. - - To resolve this we unlink vcol blobs from the pointer to the - data (in the record[1]). The orphan memory must be freed manually - (but, again, only if it was owned by Field_blob::value String). - - With REPLACE and INSERT ... ON DUP KEY UPATE it's even more complex. - There is no store_record(table,record[1]), instead the row is read - directly into record[1]. -*/ -struct BLOB_VALUE_ORPHANAGE { - MY_BITMAP map; - TABLE *table; - BLOB_VALUE_ORPHANAGE() { map.bitmap= NULL; } - ~BLOB_VALUE_ORPHANAGE() { free(); } - bool init(TABLE *table_arg) - { - table= table_arg; - if (table->s->virtual_fields && table->s->blob_fields) - return bitmap_init(&map, NULL, table->s->virtual_fields, FALSE); - map.bitmap= NULL; - return 0; - } - void free() { bitmap_free(&map); } - - /** Remove blob's ownership from blob value memory - - @note the memory becomes orphaned, it needs to be freed using - free_orphans() or re-attached back to blobs using adopt_orphans() - */ - void make_orphans() - { - DBUG_ASSERT(!table || !table->s->virtual_fields || !table->s->blob_fields || map.bitmap); - if (!map.bitmap) - return; - for (Field **ptr=table->vfield; *ptr; ptr++) - { - Field_blob *vb= (Field_blob*)(*ptr); - if (!(vb->flags & BLOB_FLAG) || !vb->owns_ptr(vb->get_ptr())) - continue; - bitmap_set_bit(&map, ptr - table->vfield); - vb->clear_temporary(); - } - } - - /** Frees orphaned blob values - - @note It is assumed that value pointers are in table->record[1], while - Field_blob::ptr's point to table->record[0] as usual - */ - void free_orphans() - { - DBUG_ASSERT(!table || !table->s->virtual_fields || !table->s->blob_fields || map.bitmap); - if (!map.bitmap) - return; - for (Field **ptr=table->vfield; *ptr; ptr++) - { - Field_blob *vb= (Field_blob*)(*ptr); - if (vb->flags & BLOB_FLAG && bitmap_fast_test_and_clear(&map, ptr - table->vfield)) - my_free(vb->get_ptr(table->s->rec_buff_length)); - } - DBUG_ASSERT(bitmap_is_clear_all(&map)); - } - - /** Restores blob's ownership over previously orphaned values */ - void adopt_orphans() - { - DBUG_ASSERT(!table || !table->s->virtual_fields || !table->s->blob_fields || map.bitmap); - if (!map.bitmap) - return; - for (Field **ptr=table->vfield; *ptr; ptr++) - { - Field_blob *vb= (Field_blob*)(*ptr); - if (vb->flags & BLOB_FLAG && bitmap_fast_test_and_clear(&map, ptr - table->vfield)) - vb->own_value_ptr(); - } - DBUG_ASSERT(bitmap_is_clear_all(&map)); - } -}; - - /* The COPY_INFO structure is used by INSERT/REPLACE code. The schema of the row counting by the INSERT/INSERT ... ON DUPLICATE KEY @@ -306,7 +213,7 @@ struct BLOB_VALUE_ORPHANAGE { of the INSERT ... ON DUPLICATE KEY UPDATE no matter whether the row was actually changed or not. */ -struct COPY_INFO { +typedef struct st_copy_info { ha_rows records; /**< Number of processed records */ ha_rows deleted; /**< Number of deleted records */ ha_rows updated; /**< Number of updated records */ @@ -322,8 +229,7 @@ struct COPY_INFO { /* for VIEW ... WITH CHECK OPTION */ TABLE_LIST *view; TABLE_LIST *table_list; /* Normal table */ - BLOB_VALUE_ORPHANAGE vblobs0, vblobs1; // vcol blobs of record[0] and record[1] -}; +} COPY_INFO; class Key_part_spec :public Sql_alloc { @@ -5435,7 +5341,6 @@ class multi_update :public select_result_interceptor TABLE_LIST *update_tables, *table_being_updated; TABLE **tmp_tables, *main_table, *table_to_update; TMP_TABLE_PARAM *tmp_table_param; - BLOB_VALUE_ORPHANAGE *vblobs; ha_rows updated, found; List *fields, *values; List **fields_for_table, **values_for_table; diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 9c21cb74802fb..7b583e1ddec2e 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -812,11 +812,6 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, info.update_values= &update_values; info.view= (table_list->view ? table_list : 0); info.table_list= table_list; - if (duplic != DUP_ERROR) - { - info.vblobs0.init(table); - info.vblobs1.init(table); - } /* Count warnings for all inserts. @@ -1184,6 +1179,7 @@ bool mysql_insert(THD *thd,TABLE_LIST *table_list, thd->lex->current_select->save_leaf_tables(thd); thd->lex->current_select->first_cond_optimization= 0; } + DBUG_RETURN(FALSE); abort: @@ -1695,12 +1691,9 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) } if (table->vfield) { - info->vblobs0.make_orphans(); table->move_fields(table->field, table->record[1], table->record[0]); table->update_virtual_fields(VCOL_UPDATE_INDEXED); - info->vblobs1.make_orphans(); table->move_fields(table->field, table->record[0], table->record[1]); - info->vblobs0.adopt_orphans(); } if (info->handle_duplicates == DUP_UPDATE) { @@ -1861,7 +1854,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) trg_error= 1; goto ok_or_after_trg_err; } - info->vblobs1.free_orphans(); /* Let us attempt do write_row() once more */ } } @@ -1912,7 +1904,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) my_safe_afree(key,table->s->max_unique_length); if (!table->file->has_transactions()) thd->transaction.stmt.modified_non_trans_table= TRUE; - info->vblobs1.free_orphans(); DBUG_RETURN(trg_error); err: @@ -1924,7 +1915,6 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) if (key) my_safe_afree(key, table->s->max_unique_length); table->column_bitmaps_set(save_read_set, save_write_set); - info->vblobs1.free_orphans(); DBUG_RETURN(1); } @@ -3106,14 +3096,10 @@ static void free_delayed_insert_blobs(register TABLE *table) { for (Field **ptr=table->field ; *ptr ; ptr++) { - Field_blob *f= (Field_blob*)(*ptr); - if (f->flags & BLOB_FLAG) + if ((*ptr)->flags & BLOB_FLAG) { - if (f->vcol_info) - f->free(); - else - my_free(f->get_ptr()); - f->reset(); + my_free(((Field_blob *) (*ptr))->get_ptr()); + ((Field_blob *) (*ptr))->reset(); } } } @@ -3135,9 +3121,6 @@ bool Delayed_insert::handle_inserts(void) table->next_number_field=table->found_next_number_field; table->use_all_columns(); - info.vblobs0.init(table); - info.vblobs1.init(table); - THD_STAGE_INFO(&thd, stage_upgrading_lock); if (thr_upgrade_write_delay_lock(*thd.lock->locks, delayed_lock, thd.variables.lock_wait_timeout)) @@ -3244,8 +3227,6 @@ bool Delayed_insert::handle_inserts(void) if (info.handle_duplicates == DUP_UPDATE) table->file->extra(HA_EXTRA_INSERT_WITH_UPDATE); thd.clear_error(); // reset error for binlog - if (table->vfield) - table->update_virtual_fields(VCOL_UPDATE_FOR_WRITE); if (write_record(&thd, table, &info)) { info.error_count++; // Ignore errors @@ -3352,8 +3333,6 @@ bool Delayed_insert::handle_inserts(void) DBUG_PRINT("error", ("HA_EXTRA_NO_CACHE failed after loop")); goto err; } - info.vblobs0.free(); - info.vblobs1.free(); query_cache_invalidate3(&thd, table, 1); mysql_mutex_lock(&mutex); DBUG_RETURN(0); @@ -3362,8 +3341,6 @@ bool Delayed_insert::handle_inserts(void) #ifndef DBUG_OFF max_rows= 0; // For DBUG output #endif - info.vblobs0.free(); - info.vblobs1.free(); /* Remove all not used rows */ mysql_mutex_lock(&mutex); while ((row=rows.get())) @@ -3611,11 +3588,6 @@ select_insert::prepare(List &values, SELECT_LEX_UNIT *u) restore_record(table,s->default_values); // Get empty record table->reset_default_fields(); table->next_number_field=table->found_next_number_field; - if (info.handle_duplicates != DUP_ERROR) - { - info.vblobs0.init(table); - info.vblobs1.init(table); - } #ifdef HAVE_REPLICATION if (thd->rgi_slave && diff --git a/sql/sql_insert.h b/sql/sql_insert.h index 5ec1846425dd0..cbfc1ea9dcdb2 100644 --- a/sql/sql_insert.h +++ b/sql/sql_insert.h @@ -21,6 +21,7 @@ /* Instead of including sql_lex.h we add this typedef here */ typedef List List_item; +typedef struct st_copy_info COPY_INFO; bool mysql_prepare_insert(THD *thd, TABLE_LIST *table_list, TABLE *table, List &fields, List_item *values, diff --git a/sql/sql_load.cc b/sql/sql_load.cc index c25e73e73462f..6f0e97a61c982 100644 --- a/sql/sql_load.cc +++ b/sql/sql_load.cc @@ -537,12 +537,6 @@ int mysql_load(THD *thd,sql_exchange *ex,TABLE_LIST *table_list, !(thd->variables.sql_mode & MODE_NO_BACKSLASH_ESCAPES))) ? (*escaped)[0] : INT_MAX; - if (handle_duplicates != DUP_ERROR) - { - info.vblobs0.init(table); - info.vblobs1.init(table); - } - READ_INFO read_info(thd, file, tot_length, ex->cs ? ex->cs : thd->variables.collation_database, *field_term,*ex->line_start, *ex->line_term, *enclosed, diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 1f1af7f266067..05f7080609ad5 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -273,7 +273,6 @@ int mysql_update(THD *thd, SORT_INFO *file_sort= 0; READ_RECORD info; SELECT_LEX *select_lex= &thd->lex->select_lex; - BLOB_VALUE_ORPHANAGE vblobs; ulonglong id; List all_fields; killed_state killed_status= NOT_KILLED; @@ -725,8 +724,6 @@ int mysql_update(THD *thd, table->reset_default_fields(); - vblobs.init(table); - /* We can use compare_record() to optimize away updates if the table handler is returning all columns OR if @@ -748,9 +745,6 @@ int mysql_update(THD *thd, explain->tracker.on_record_after_where(); store_record(table,record[1]); - - vblobs.make_orphans(); - if (fill_record_n_invoke_before_triggers(thd, table, fields, values, 0, TRG_EVENT_UPDATE)) break; /* purecov: inspected */ @@ -910,9 +904,7 @@ int mysql_update(THD *thd, error= 1; break; } - vblobs.free_orphans(); } - vblobs.free_orphans(); ANALYZE_STOP_TRACKING(&explain->command_tracker); table->auto_increment_field_not_null= FALSE; dup_key_found= 0; @@ -1760,8 +1752,6 @@ int multi_update::prepare(List ¬_used_values, table_count); values_for_table= (List_item **) thd->alloc(sizeof(List_item *) * table_count); - vblobs= (BLOB_VALUE_ORPHANAGE *)thd->calloc(sizeof(*vblobs) * table_count); - if (thd->is_fatal_error) DBUG_RETURN(1); for (i=0 ; i < table_count ; i++) @@ -1794,7 +1784,6 @@ int multi_update::prepare(List ¬_used_values, TABLE *table= ((Item_field*)(fields_for_table[i]->head()))->field->table; switch_to_nullable_trigger_fields(*fields_for_table[i], table); switch_to_nullable_trigger_fields(*values_for_table[i], table); - vblobs[i].init(table); } } copy_field= new Copy_field[max_fields]; @@ -2076,8 +2065,6 @@ multi_update::~multi_update() free_tmp_table(thd, tmp_tables[cnt]); tmp_table_param[cnt].cleanup(); } - vblobs[cnt].free_orphans(); - vblobs[cnt].free(); } } if (copy_field) @@ -2123,9 +2110,7 @@ int multi_update::send_data(List ¬_used_values) can_compare_record= records_are_comparable(table); table->status|= STATUS_UPDATED; - vblobs[offset].free_orphans(); store_record(table,record[1]); - vblobs[offset].make_orphans(); if (fill_record_n_invoke_before_triggers(thd, table, *fields_for_table[offset], *values_for_table[offset], 0, @@ -2342,7 +2327,6 @@ int multi_update::do_updates() goto err; } table->file->extra(HA_EXTRA_NO_CACHE); - empty_record(table); check_opt_it.rewind(); while(TABLE *tbl= check_opt_it++) @@ -2416,9 +2400,7 @@ int multi_update::do_updates() goto err2; table->status|= STATUS_UPDATED; - vblobs[offset].free_orphans(); store_record(table,record[1]); - vblobs[offset].make_orphans(); /* Copy data from temporary table to current table */ for (copy_field_ptr=copy_field; From ea1b25046c81db8fdf59130126d57cfb42737ba5 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 30 Dec 2016 11:07:44 +0200 Subject: [PATCH 57/74] New simpler bugfix for UPDATE and virtual BLOBs When updating a table with virtual BLOB columns, the following might happen: - an old record is read from the table, it has no virtual blob values - update_virtual_fields() is run, vcol blob gets its value into the record. But only a pointer to the value is in the table->record[0], the value is in Field_blob::value String (but it doesn't have to be! it can be in the record, if the column is just a copy of another columns: ... b VARCHAR, c BLOB AS (b) ...) - store_record(table,record[1]), old record now is in record[1] - fill_record() prepares new values in record[0], vcol blob is updated, new value replaces the old one in the Field_blob::value - now both record[1] and record[0] have a pointer that points to the *new* vcol blob value. Or record[1] has a pointer to nowhere if Field_blob::value had to realloc. To fix this I have introduced a new String object 'read_value' in Field_blob. When updating virtual columns when a row has been read, the allocated value is stored in 'read_value' instead of 'value'. The allocated blobs for the new row is stored in 'value' as before. I also made, as a safety precaution, the insert delayed handling of blobs more general by using value to store strings instead of the record. This ensures that virtual functions on delayed insert should work in as in the case of normal insert. Triggers are now properly updating the read, write and vcol maps for used fields. This means that we don't need VCOL_UPDATE_FOR_READ_WRITE anymore and there is no need for any other special handling of triggers in update_virtual_fields(). To be able to test how many times virtual fields are invoked, I also relaxed rules that one can use local (@) variables in DEFAULT and non persistent virtual field expressions. --- mysql-test/r/default.result | 8 +- mysql-test/suite/vcol/inc/vcol_trigger_sp.inc | 132 +++++++++++++ mysql-test/suite/vcol/r/not_supported.result | 4 +- .../vcol/r/vcol_trigger_sp_innodb.result | 179 ++++++++++++++++++ .../vcol/r/vcol_trigger_sp_myisam.result | 179 ++++++++++++++++++ mysql-test/suite/vcol/t/not_supported.test | 8 +- mysql-test/t/default.test | 8 +- sql/field.h | 40 +++- sql/item_func.cc | 2 +- sql/sql_base.cc | 13 +- sql/sql_insert.cc | 41 +++- sql/sql_trigger.cc | 6 + sql/sql_update.cc | 15 +- sql/table.cc | 46 ++++- sql/table.h | 4 +- storage/innobase/handler/handler0alter.cc | 2 +- storage/myisam/ha_myisam.cc | 8 +- 17 files changed, 643 insertions(+), 52 deletions(-) diff --git a/mysql-test/r/default.result b/mysql-test/r/default.result index dc7a33d6b9c18..ac449d44fee95 100644 --- a/mysql-test/r/default.result +++ b/mysql-test/r/default.result @@ -480,6 +480,10 @@ a b 2 2 3 4 drop table t1; +CREATE OR REPLACE TABLE t1 (a INT DEFAULT @v); +drop table t1; +CREATE TABLE t1 (a INT DEFAULT @v:=1); +drop table t1; # # Error handling # @@ -516,10 +520,6 @@ CREATE TABLE t1 (a INT DEFAULT(?)); Got one of the listed errors CREATE TABLE t1 (a INT DEFAULT (b), b INT DEFAULT(a)); ERROR 01000: Expression for field `a` is refering to uninitialized field `b` -CREATE TABLE t1 (a INT DEFAULT @v); -ERROR HY000: Function or expression '@v' cannot be used in the DEFAULT clause of `a` -CREATE TABLE t1 (a INT DEFAULT @v:=1); -ERROR HY000: Function or expression '@v' cannot be used in the DEFAULT clause of `a` CREATE TABLE t1 (a INT DEFAULT(NAME_CONST('xxx', 'yyy')); ERROR HY000: Function or expression 'name_const()' cannot be used in the DEFAULT clause of `a` CREATE TABLE t1 (a INT DEFAULT COUNT(*)); diff --git a/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc b/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc index eb7e6ad32b932..f807405d18d34 100644 --- a/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc +++ b/mysql-test/suite/vcol/inc/vcol_trigger_sp.inc @@ -149,3 +149,135 @@ DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +--echo # +--echo # Examine the number of times triggers are recalculated for updates +--echo # + +CREATE TABLE t1 ( + a INTEGER UNSIGNED NULL DEFAULT NULL, + b CHAR(10) NULL DEFAULT NULL, + c blob NULL DEFAULT NULL, + blob_a blob GENERATED ALWAYS AS (last_value(@a:=@a+1,a)) VIRTUAL, + blob_b blob GENERATED ALWAYS AS (last_value(@b:=@b+1,b)) VIRTUAL, + blob_c blob GENERATED ALWAYS AS (last_value(@c:=@c+1,c)) VIRTUAL +); + +DELIMITER |; +CREATE TRIGGER t1_ins + BEFORE INSERT + ON t1 + FOR EACH ROW +BEGIN + IF NEW.b IS NULL THEN + SET NEW.b="generated before insert"; + END IF; +END | + +CREATE TRIGGER t1_update + BEFORE UPDATE + ON t1 + FOR EACH ROW +BEGIN + IF NEW.b IS NULL or NEW.c IS NULL THEN + SET NEW.b="generated before update"; + SET NEW.c="generated before update"; + END IF; +END | + +DELIMITER ;| + +--echo # Inserts +set @a=0,@b=0,@c=0; + +insert into t1 (a) values(1); +insert into t1 (a,b) values(2, "*2*"); +insert into t1 (a,b,c) values(3, "*3*", "**3**"); +insert into t1 (a,c) values(4, "**4**"); +select * from t1; +select @a,@b,@c; +select * from t1; +select @a,@b,@c; +select a,b,c from t1; +select @a,@b,@c; +select a,b,c,blob_a from t1; +select @a,@b,@c; + +--echo # updates +set @a=0,@b=0,@c=0; + +update t1 set a=a+100 where a=1; +update t1 set a=a+100, b="*102*" where a=2; +update t1 set a=a+100, b=NULL where a=3; +update t1 set a=a+100, b="invisible", c=NULL where a=4; +select @a,@b,@c; +select * from t1; + +drop trigger t1_ins; +drop trigger t1_update; +drop table t1; + +--echo # +--echo # Same test, but with virtual keys +--echo # + +CREATE TABLE t1 ( + a INTEGER UNSIGNED NULL DEFAULT NULL, + b CHAR(10) NULL DEFAULT NULL, + c blob NULL DEFAULT NULL, + blob_a blob GENERATED ALWAYS AS (a) VIRTUAL, + blob_b blob GENERATED ALWAYS AS (b) VIRTUAL, + blob_c blob GENERATED ALWAYS AS (c) VIRTUAL, + key (a), + key (blob_a(10)), + key (blob_b(10)), + key (blob_c(10)) +); + +DELIMITER |; +CREATE TRIGGER t1_ins + BEFORE INSERT + ON t1 + FOR EACH ROW +BEGIN + IF NEW.b IS NULL THEN + SET NEW.b="generated before insert"; + END IF; +END | + +CREATE TRIGGER t1_update + BEFORE UPDATE + ON t1 + FOR EACH ROW +BEGIN + IF NEW.b IS NULL or NEW.c IS NULL THEN + SET NEW.b="generated before update"; + SET NEW.c="generated before update"; + END IF; +END | + +DELIMITER ;| + +--echo # Inserts +insert into t1 (a) values(1); +insert into t1 (a,b) values(2, "*2*"); +insert into t1 (a,b,c) values(3, "*3*", "**3**"); +insert into t1 (a,c) values(4, "**4**"); +select * from t1; +select @a,@b,@c; +select * from t1; +select @a,@b,@c; +select a,b,c from t1; +select @a,@b,@c; +select a,b,c,blob_a from t1; +select @a,@b,@c; + +--echo # updates +update t1 set a=a+100 where a=1; +update t1 set a=a+100, b="*102*" where a=2; +update t1 set a=a+100, b=NULL where a=3; +update t1 set a=a+100, b="invisible", c=NULL where a=4; +select * from t1; + +drop trigger t1_ins; +drop trigger t1_update; +drop table t1; diff --git a/mysql-test/suite/vcol/r/not_supported.result b/mysql-test/suite/vcol/r/not_supported.result index f2c98706dd64f..c804cf220d2c0 100644 --- a/mysql-test/suite/vcol/r/not_supported.result +++ b/mysql-test/suite/vcol/r/not_supported.result @@ -4,14 +4,14 @@ set time_zone='+10:00'; set div_precision_increment=20; create table t1 (a int, b int, v decimal(20,19) as (a/3)); create table t2 (a int, b int, v int as (a+@a)); -ERROR HY000: Function or expression '@a' cannot be used in the GENERATED ALWAYS AS clause of `v` +drop table t2; create table t2 (a int, b int, v int as (a+@a) PERSISTENT); ERROR HY000: Function or expression '@a' cannot be used in the GENERATED ALWAYS AS clause of `v` create table t3_ok (a int, b int, v int as (a+@@error_count)); create table t3 (a int, b int, v int as (a+@@error_count) PERSISTENT); ERROR HY000: Function or expression '@@error_count' cannot be used in the GENERATED ALWAYS AS clause of `v` create table t4 (a int, b int, v int as (@a:=a)); -ERROR HY000: Function or expression '@a' cannot be used in the GENERATED ALWAYS AS clause of `v` +drop table t4; create table t4 (a int, b int, v int as (@a:=a) PERSISTENT); ERROR HY000: Function or expression '@a' cannot be used in the GENERATED ALWAYS AS clause of `v` create table t8 (a int, b int, v varchar(100) as (from_unixtime(a))); diff --git a/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result b/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result index 1d78bbf50e4c0..0a82f1006e7f9 100644 --- a/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result +++ b/mysql-test/suite/vcol/r/vcol_trigger_sp_innodb.result @@ -125,3 +125,182 @@ c DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +# +# Examine the number of times triggers are recalculated for updates +# +CREATE TABLE t1 ( +a INTEGER UNSIGNED NULL DEFAULT NULL, +b CHAR(10) NULL DEFAULT NULL, +c blob NULL DEFAULT NULL, +blob_a blob GENERATED ALWAYS AS (last_value(@a:=@a+1,a)) VIRTUAL, +blob_b blob GENERATED ALWAYS AS (last_value(@b:=@b+1,b)) VIRTUAL, +blob_c blob GENERATED ALWAYS AS (last_value(@c:=@c+1,c)) VIRTUAL +); +CREATE TRIGGER t1_ins +BEFORE INSERT +ON t1 +FOR EACH ROW +BEGIN +IF NEW.b IS NULL THEN +SET NEW.b="generated before insert"; +END IF; +END | +CREATE TRIGGER t1_update +BEFORE UPDATE +ON t1 +FOR EACH ROW +BEGIN +IF NEW.b IS NULL or NEW.c IS NULL THEN +SET NEW.b="generated before update"; +SET NEW.c="generated before update"; +END IF; +END | +# Inserts +set @a=0,@b=0,@c=0; +insert into t1 (a) values(1); +insert into t1 (a,b) values(2, "*2*"); +insert into t1 (a,b,c) values(3, "*3*", "**3**"); +insert into t1 (a,c) values(4, "**4**"); +select * from t1; +a b c blob_a blob_b blob_c +1 generated NULL 1 generated NULL +2 *2* NULL 2 *2* NULL +3 *3* **3** 3 *3* **3** +4 generated **4** 4 generated **4** +select @a,@b,@c; +@a @b @c +4 4 4 +select * from t1; +a b c blob_a blob_b blob_c +1 generated NULL 1 generated NULL +2 *2* NULL 2 *2* NULL +3 *3* **3** 3 *3* **3** +4 generated **4** 4 generated **4** +select @a,@b,@c; +@a @b @c +8 8 8 +select a,b,c from t1; +a b c +1 generated NULL +2 *2* NULL +3 *3* **3** +4 generated **4** +select @a,@b,@c; +@a @b @c +8 8 8 +select a,b,c,blob_a from t1; +a b c blob_a +1 generated NULL 1 +2 *2* NULL 2 +3 *3* **3** 3 +4 generated **4** 4 +select @a,@b,@c; +@a @b @c +12 8 8 +# updates +set @a=0,@b=0,@c=0; +update t1 set a=a+100 where a=1; +update t1 set a=a+100, b="*102*" where a=2; +update t1 set a=a+100, b=NULL where a=3; +update t1 set a=a+100, b="invisible", c=NULL where a=4; +select @a,@b,@c; +@a @b @c +0 0 0 +select * from t1; +a b c blob_a blob_b blob_c +101 generated generated before update 101 generated generated before update +102 generated generated before update 102 generated generated before update +103 generated generated before update 103 generated generated before update +104 generated generated before update 104 generated generated before update +drop trigger t1_ins; +drop trigger t1_update; +drop table t1; +# +# Same test, but with virtual keys +# +CREATE TABLE t1 ( +a INTEGER UNSIGNED NULL DEFAULT NULL, +b CHAR(10) NULL DEFAULT NULL, +c blob NULL DEFAULT NULL, +blob_a blob GENERATED ALWAYS AS (a) VIRTUAL, +blob_b blob GENERATED ALWAYS AS (b) VIRTUAL, +blob_c blob GENERATED ALWAYS AS (c) VIRTUAL, +key (a), +key (blob_a(10)), +key (blob_b(10)), +key (blob_c(10)) +); +CREATE TRIGGER t1_ins +BEFORE INSERT +ON t1 +FOR EACH ROW +BEGIN +IF NEW.b IS NULL THEN +SET NEW.b="generated before insert"; +END IF; +END | +CREATE TRIGGER t1_update +BEFORE UPDATE +ON t1 +FOR EACH ROW +BEGIN +IF NEW.b IS NULL or NEW.c IS NULL THEN +SET NEW.b="generated before update"; +SET NEW.c="generated before update"; +END IF; +END | +# Inserts +insert into t1 (a) values(1); +insert into t1 (a,b) values(2, "*2*"); +insert into t1 (a,b,c) values(3, "*3*", "**3**"); +insert into t1 (a,c) values(4, "**4**"); +select * from t1; +a b c blob_a blob_b blob_c +1 generated NULL 1 generated NULL +2 *2* NULL 2 *2* NULL +3 *3* **3** 3 *3* **3** +4 generated **4** 4 generated **4** +select @a,@b,@c; +@a @b @c +4 4 4 +select * from t1; +a b c blob_a blob_b blob_c +1 generated NULL 1 generated NULL +2 *2* NULL 2 *2* NULL +3 *3* **3** 3 *3* **3** +4 generated **4** 4 generated **4** +select @a,@b,@c; +@a @b @c +4 4 4 +select a,b,c from t1; +a b c +1 generated NULL +2 *2* NULL +3 *3* **3** +4 generated **4** +select @a,@b,@c; +@a @b @c +4 4 4 +select a,b,c,blob_a from t1; +a b c blob_a +1 generated NULL 1 +2 *2* NULL 2 +3 *3* **3** 3 +4 generated **4** 4 +select @a,@b,@c; +@a @b @c +4 4 4 +# updates +update t1 set a=a+100 where a=1; +update t1 set a=a+100, b="*102*" where a=2; +update t1 set a=a+100, b=NULL where a=3; +update t1 set a=a+100, b="invisible", c=NULL where a=4; +select * from t1; +a b c blob_a blob_b blob_c +101 generated generated before update 101 generated generated before update +102 generated generated before update 102 generated generated before update +103 generated generated before update 103 generated generated before update +104 generated generated before update 104 generated generated before update +drop trigger t1_ins; +drop trigger t1_update; +drop table t1; diff --git a/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result b/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result index 77efa8fe6b91e..edafd4742868f 100644 --- a/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_trigger_sp_myisam.result @@ -125,3 +125,182 @@ c DROP TRIGGER t1_ins_aft; DROP TRIGGER t1_del_bef; DROP TABLE t1,t2; +# +# Examine the number of times triggers are recalculated for updates +# +CREATE TABLE t1 ( +a INTEGER UNSIGNED NULL DEFAULT NULL, +b CHAR(10) NULL DEFAULT NULL, +c blob NULL DEFAULT NULL, +blob_a blob GENERATED ALWAYS AS (last_value(@a:=@a+1,a)) VIRTUAL, +blob_b blob GENERATED ALWAYS AS (last_value(@b:=@b+1,b)) VIRTUAL, +blob_c blob GENERATED ALWAYS AS (last_value(@c:=@c+1,c)) VIRTUAL +); +CREATE TRIGGER t1_ins +BEFORE INSERT +ON t1 +FOR EACH ROW +BEGIN +IF NEW.b IS NULL THEN +SET NEW.b="generated before insert"; +END IF; +END | +CREATE TRIGGER t1_update +BEFORE UPDATE +ON t1 +FOR EACH ROW +BEGIN +IF NEW.b IS NULL or NEW.c IS NULL THEN +SET NEW.b="generated before update"; +SET NEW.c="generated before update"; +END IF; +END | +# Inserts +set @a=0,@b=0,@c=0; +insert into t1 (a) values(1); +insert into t1 (a,b) values(2, "*2*"); +insert into t1 (a,b,c) values(3, "*3*", "**3**"); +insert into t1 (a,c) values(4, "**4**"); +select * from t1; +a b c blob_a blob_b blob_c +1 generated NULL 1 generated NULL +2 *2* NULL 2 *2* NULL +3 *3* **3** 3 *3* **3** +4 generated **4** 4 generated **4** +select @a,@b,@c; +@a @b @c +4 4 4 +select * from t1; +a b c blob_a blob_b blob_c +1 generated NULL 1 generated NULL +2 *2* NULL 2 *2* NULL +3 *3* **3** 3 *3* **3** +4 generated **4** 4 generated **4** +select @a,@b,@c; +@a @b @c +8 8 8 +select a,b,c from t1; +a b c +1 generated NULL +2 *2* NULL +3 *3* **3** +4 generated **4** +select @a,@b,@c; +@a @b @c +8 8 8 +select a,b,c,blob_a from t1; +a b c blob_a +1 generated NULL 1 +2 *2* NULL 2 +3 *3* **3** 3 +4 generated **4** 4 +select @a,@b,@c; +@a @b @c +12 8 8 +# updates +set @a=0,@b=0,@c=0; +update t1 set a=a+100 where a=1; +update t1 set a=a+100, b="*102*" where a=2; +update t1 set a=a+100, b=NULL where a=3; +update t1 set a=a+100, b="invisible", c=NULL where a=4; +select @a,@b,@c; +@a @b @c +0 0 0 +select * from t1; +a b c blob_a blob_b blob_c +101 generated generated before update 101 generated generated before update +102 generated generated before update 102 generated generated before update +103 generated generated before update 103 generated generated before update +104 generated generated before update 104 generated generated before update +drop trigger t1_ins; +drop trigger t1_update; +drop table t1; +# +# Same test, but with virtual keys +# +CREATE TABLE t1 ( +a INTEGER UNSIGNED NULL DEFAULT NULL, +b CHAR(10) NULL DEFAULT NULL, +c blob NULL DEFAULT NULL, +blob_a blob GENERATED ALWAYS AS (a) VIRTUAL, +blob_b blob GENERATED ALWAYS AS (b) VIRTUAL, +blob_c blob GENERATED ALWAYS AS (c) VIRTUAL, +key (a), +key (blob_a(10)), +key (blob_b(10)), +key (blob_c(10)) +); +CREATE TRIGGER t1_ins +BEFORE INSERT +ON t1 +FOR EACH ROW +BEGIN +IF NEW.b IS NULL THEN +SET NEW.b="generated before insert"; +END IF; +END | +CREATE TRIGGER t1_update +BEFORE UPDATE +ON t1 +FOR EACH ROW +BEGIN +IF NEW.b IS NULL or NEW.c IS NULL THEN +SET NEW.b="generated before update"; +SET NEW.c="generated before update"; +END IF; +END | +# Inserts +insert into t1 (a) values(1); +insert into t1 (a,b) values(2, "*2*"); +insert into t1 (a,b,c) values(3, "*3*", "**3**"); +insert into t1 (a,c) values(4, "**4**"); +select * from t1; +a b c blob_a blob_b blob_c +1 generated NULL 1 generated NULL +2 *2* NULL 2 *2* NULL +3 *3* **3** 3 *3* **3** +4 generated **4** 4 generated **4** +select @a,@b,@c; +@a @b @c +4 4 4 +select * from t1; +a b c blob_a blob_b blob_c +1 generated NULL 1 generated NULL +2 *2* NULL 2 *2* NULL +3 *3* **3** 3 *3* **3** +4 generated **4** 4 generated **4** +select @a,@b,@c; +@a @b @c +4 4 4 +select a,b,c from t1; +a b c +1 generated NULL +2 *2* NULL +3 *3* **3** +4 generated **4** +select @a,@b,@c; +@a @b @c +4 4 4 +select a,b,c,blob_a from t1; +a b c blob_a +1 generated NULL 1 +2 *2* NULL 2 +3 *3* **3** 3 +4 generated **4** 4 +select @a,@b,@c; +@a @b @c +4 4 4 +# updates +update t1 set a=a+100 where a=1; +update t1 set a=a+100, b="*102*" where a=2; +update t1 set a=a+100, b=NULL where a=3; +update t1 set a=a+100, b="invisible", c=NULL where a=4; +select * from t1; +a b c blob_a blob_b blob_c +101 generated generated before update 101 generated generated before update +102 generated generated before update 102 generated generated before update +103 generated generated before update 103 generated generated before update +104 generated generated before update 104 generated generated before update +drop trigger t1_ins; +drop trigger t1_update; +drop table t1; diff --git a/mysql-test/suite/vcol/t/not_supported.test b/mysql-test/suite/vcol/t/not_supported.test index b7544cb33a463..1ea7970523ac4 100644 --- a/mysql-test/suite/vcol/t/not_supported.test +++ b/mysql-test/suite/vcol/t/not_supported.test @@ -2,7 +2,7 @@ # MDEV-7113 difference between check_vcol_func_processor and check_partition_func_processor # -# the following functions must not be supported in virtual columns. +# the following functions must not be supported in persistent columns. # but for compatibility reasons it won't be done in a GA version, # we'll only fix most critical issues (inconsistent results, crashes) @@ -13,15 +13,13 @@ set time_zone='+10:00'; set div_precision_increment=20; create table t1 (a int, b int, v decimal(20,19) as (a/3)); ---error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED -create table t2 (a int, b int, v int as (a+@a)); +create table t2 (a int, b int, v int as (a+@a)); drop table t2; --error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED create table t2 (a int, b int, v int as (a+@a) PERSISTENT); create table t3_ok (a int, b int, v int as (a+@@error_count)); --error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED create table t3 (a int, b int, v int as (a+@@error_count) PERSISTENT); ---error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED -create table t4 (a int, b int, v int as (@a:=a)); +create table t4 (a int, b int, v int as (@a:=a)); drop table t4; --error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED create table t4 (a int, b int, v int as (@a:=a) PERSISTENT); create table t8 (a int, b int, v varchar(100) as (from_unixtime(a))); diff --git a/mysql-test/t/default.test b/mysql-test/t/default.test index 41ed161344821..9ae088405facd 100644 --- a/mysql-test/t/default.test +++ b/mysql-test/t/default.test @@ -359,6 +359,8 @@ insert into t1 (b) values(2); insert into t1 (a,b) values(3,4); select * from t1; drop table t1; +CREATE OR REPLACE TABLE t1 (a INT DEFAULT @v); drop table t1; +CREATE TABLE t1 (a INT DEFAULT @v:=1); drop table t1; --echo # --echo # Error handling @@ -407,12 +409,6 @@ CREATE TABLE t1 (a INT DEFAULT(?)); --error ER_EXPRESSION_REFERS_TO_UNINIT_FIELD CREATE TABLE t1 (a INT DEFAULT (b), b INT DEFAULT(a)); ---error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED -CREATE TABLE t1 (a INT DEFAULT @v); - ---error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED -CREATE TABLE t1 (a INT DEFAULT @v:=1); - --error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED CREATE TABLE t1 (a INT DEFAULT(NAME_CONST('xxx', 'yyy')); diff --git a/sql/field.h b/sql/field.h index 0d278968eb5a6..400d2ef0e5e04 100644 --- a/sql/field.h +++ b/sql/field.h @@ -3176,6 +3176,12 @@ class Field_blob :public Field_longstr { The 'value'-object is a cache fronting the storage engine. */ String value; + /** + Cache for blob values when reading a row with a virtual blob + field. This is needed to not destroy the old cached value when + updating the blob with a new value when creating the new row. + */ + String read_value; static void do_copy_blob(Copy_field *copy); static void do_conv_blob(Copy_field *copy); @@ -3279,7 +3285,7 @@ class Field_blob :public Field_longstr { return (uint32) (((ulonglong) 1 << (packlength*8)) -1); } int reset(void) { bzero(ptr, packlength+sizeof(uchar*)); return 0; } - void reset_fields() { bzero((uchar*) &value,sizeof(value)); } + void reset_fields() { bzero((uchar*) &value,sizeof(value)); bzero((uchar*) &read_value,sizeof(read_value)); } uint32 get_field_buffer_size(void) { return value.alloced_length(); } void store_length(uchar *i_ptr, uint i_packlength, uint32 i_number); inline void store_length(uint32 number) @@ -3332,13 +3338,41 @@ class Field_blob :public Field_longstr { memcpy(ptr+packlength, &tmp, sizeof(char*)); return 0; } + /* store value for the duration of the current read record */ + inline void swap_value_and_read_value() + { + read_value.swap(value); + } + inline void set_value(uchar *data) + { + /* Set value pointer. Lengths are not important */ + value.reset((char*) data, 1, 1, &my_charset_bin); + } virtual uchar *pack(uchar *to, const uchar *from, uint max_length); virtual const uchar *unpack(uchar *to, const uchar *from, const uchar *from_end, uint param_data); uint packed_col_length(const uchar *col_ptr, uint length); uint max_packed_col_length(uint max_length); - void free() { value.free(); } - inline void clear_temporary() { bzero((uchar*) &value, sizeof(value)); } + void free() + { + value.free(); + read_value.free(); + } + inline void clear_temporary() + { + uchar *tmp= get_ptr(); + if (likely(value.ptr() == (char*) tmp)) + bzero((uchar*) &value, sizeof(value)); + else + { + /* + Currently read_value should never point to tmp, the following code + is mainly here to make things future proof. + */ + if (unlikely(read_value.ptr() == (char*) tmp)) + bzero((uchar*) &read_value, sizeof(read_value)); + } + } uint size_of() const { return sizeof(*this); } bool has_charset(void) const { return charset() == &my_charset_bin ? FALSE : TRUE; } diff --git a/sql/item_func.cc b/sql/item_func.cc index 8e912fe83c8fd..754ba9d261c1d 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -4580,7 +4580,7 @@ longlong Item_func_sleep::val_int() bool Item_func_user_var::check_vcol_func_processor(void *arg) { - return mark_unsupported_function("@", name.str, arg, VCOL_IMPOSSIBLE); + return mark_unsupported_function("@", name.str, arg, VCOL_NON_DETERMINISTIC); } #define extra_size sizeof(double) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index b6cfd2f2cc1ea..46d7d07dbdf49 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -8043,17 +8043,14 @@ fill_record_n_invoke_before_triggers(THD *thd, TABLE *table, Re-calculate virtual fields to cater for cases when base columns are updated by the triggers. */ - List_iterator_fast f(fields); - Item *fld; - Item_field *item_field; - if (fields.elements) + if (table->vfield && fields.elements) { - fld= (Item_field*)f++; - item_field= fld->field_for_view_update(); - if (item_field && table->vfield) + Item *fld= (Item_field*) fields.head(); + Item_field *item_field= fld->field_for_view_update(); + if (item_field) { DBUG_ASSERT(table == item_field->field->table); - result= table->update_virtual_fields(VCOL_UPDATE_FOR_WRITE); + result|= table->update_virtual_fields(VCOL_UPDATE_FOR_WRITE); } } } diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 7b583e1ddec2e..cea42667c48e9 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1691,8 +1691,12 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) } if (table->vfield) { + /* + We have not yet called update_virtual_fields(VOL_UPDATE_FOR_READ) + in handler methods for the just read row in record[1]. + */ table->move_fields(table->field, table->record[1], table->record[0]); - table->update_virtual_fields(VCOL_UPDATE_INDEXED); + table->update_virtual_fields(VCOL_UPDATE_FOR_REPLACE); table->move_fields(table->field, table->record[0], table->record[1]); } if (info->handle_duplicates == DUP_UPDATE) @@ -2912,6 +2916,8 @@ pthread_handler_t handle_delayed_insert(void *arg) thd->mdl_context.set_needs_thr_lock_abort(TRUE); di->table->mark_columns_needed_for_insert(); + /* Mark all columns for write as we don't know which columns we get from user */ + bitmap_set_all(di->table->write_set); /* Now wait until we get an insert or lock to handle */ /* We will not abort as long as a client thread uses this thread */ @@ -3079,7 +3085,7 @@ pthread_handler_t handle_delayed_insert(void *arg) } -/* Remove pointers from temporary fields to allocated values */ +/* Remove all pointers to data for blob fields so that original table doesn't try to free them */ static void unlink_blobs(register TABLE *table) { @@ -3093,13 +3099,27 @@ static void unlink_blobs(register TABLE *table) /* Free blobs stored in current row */ static void free_delayed_insert_blobs(register TABLE *table) +{ + for (Field **ptr=table->field ; *ptr ; ptr++) + { + if ((*ptr)->flags & BLOB_FLAG) + ((Field_blob *) *ptr)->free(); + } +} + + +/* set value field for blobs to point to data in record */ + +static void set_delayed_insert_blobs(register TABLE *table) { for (Field **ptr=table->field ; *ptr ; ptr++) { if ((*ptr)->flags & BLOB_FLAG) { - my_free(((Field_blob *) (*ptr))->get_ptr()); - ((Field_blob *) (*ptr))->reset(); + Field_blob *blob= ((Field_blob *) *ptr); + uchar *data= blob->get_ptr(); + if (data) + blob->set_value(data); // Set value.ptr() to point to data } } } @@ -3157,6 +3177,8 @@ bool Delayed_insert::handle_inserts(void) stacked_inserts--; mysql_mutex_unlock(&mutex); memcpy(table->record[0],row->record,table->s->reclength); + if (table->s->blob_fields) + set_delayed_insert_blobs(table); thd.start_time=row->start_time; thd.query_start_used=row->query_start_used; @@ -3227,6 +3249,16 @@ bool Delayed_insert::handle_inserts(void) if (info.handle_duplicates == DUP_UPDATE) table->file->extra(HA_EXTRA_INSERT_WITH_UPDATE); thd.clear_error(); // reset error for binlog + + if (table->vfield) + { + /* + Virtual fields where not calculated by caller as the temporary TABLE object used + had vcol_set empty. Better to calculate them here to make the caller faster. + */ + table->update_virtual_fields(VCOL_UPDATE_FOR_WRITE); + } + if (write_record(&thd, table, &info)) { info.error_count++; // Ignore errors @@ -3348,6 +3380,7 @@ bool Delayed_insert::handle_inserts(void) if (table->s->blob_fields) { memcpy(table->record[0],row->record,table->s->reclength); + set_delayed_insert_blobs(table); free_delayed_insert_blobs(table); } delete row; diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index 14812d2f73b2b..2d1619018b2dc 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -2266,6 +2266,7 @@ void Table_triggers_list::mark_fields_used(trg_event_type event) { int action_time; Item_trigger_field *trg_field; + DBUG_ENTER("Table_triggers_list::mark_fields_used"); for (action_time= 0; action_time < (int)TRG_ACTION_MAX; action_time++) { @@ -2280,14 +2281,19 @@ void Table_triggers_list::mark_fields_used(trg_event_type event) /* We cannot mark fields which does not present in table. */ if (trg_field->field_idx != (uint)-1) { + DBUG_PRINT("info", ("marking field: %d", trg_field->field_idx)); bitmap_set_bit(trigger_table->read_set, trg_field->field_idx); if (trg_field->get_settable_routine_parameter()) bitmap_set_bit(trigger_table->write_set, trg_field->field_idx); + if (trigger_table->field[trg_field->field_idx]->vcol_info) + trigger_table->mark_virtual_col(trigger_table-> + field[trg_field->field_idx]); } } } } trigger_table->file->column_bitmaps_signal(); + DBUG_VOID_RETURN; } diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 05f7080609ad5..476d1a4e104b0 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -618,8 +618,6 @@ int mysql_update(THD *thd, while (!(error=info.read_record(&info)) && !thd->killed) { explain->buf_tracker.on_record_read(); - if (table->vfield) - table->update_virtual_fields(VCOL_UPDATE_FOR_READ_WRITE); thd->inc_examined_row_count(1); if (!select || (error= select->skip_record(thd)) > 0) { @@ -735,8 +733,6 @@ int mysql_update(THD *thd, while (!(error=info.read_record(&info)) && !thd->killed) { explain->tracker.on_record_read(); - if (table->vfield) - table->update_virtual_fields(VCOL_UPDATE_FOR_READ_WRITE); thd->inc_examined_row_count(1); if (!select || select->skip_record(thd) > 0) { @@ -2327,6 +2323,17 @@ int multi_update::do_updates() goto err; } table->file->extra(HA_EXTRA_NO_CACHE); + /* + We have to clear the base record, if we have virtual indexed + blob fields, as some storage engines will access the blob fields + to calculate the keys to see if they have changed. Without + clearing the blob pointers will contain random values which can + cause a crash. + This is a workaround for engines that access columns not present in + either read or write set. + */ + if (table->vfield) + empty_record(table); check_opt_it.rewind(); while(TABLE *tbl= check_opt_it++) diff --git a/sql/table.cc b/sql/table.cc index 5c9d480594314..b7f83a502b71f 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -6572,6 +6572,7 @@ bool TABLE::mark_virtual_columns_for_write(bool insert_fl) { Field **vfield_ptr, *tmp_vfield; bool bitmap_updated= false; + DBUG_ENTER("mark_virtual_columns_for_write"); for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) { @@ -6610,7 +6611,7 @@ bool TABLE::mark_virtual_columns_for_write(bool insert_fl) } if (bitmap_updated) file->column_bitmaps_signal(); - return bitmap_updated; + DBUG_RETURN(bitmap_updated); } /* @@ -7324,6 +7325,7 @@ bool is_simple_order(ORDER *order) int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) { DBUG_ENTER("TABLE::update_virtual_fields"); + DBUG_PRINT("enter", ("update_mode: %d", update_mode)); Field **vfield_ptr, *vf; DBUG_ASSERT(vfield); @@ -7336,25 +7338,39 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) DBUG_ASSERT(vcol_info); DBUG_ASSERT(vcol_info->expr); - bool update; + bool update, swap_values= 0; switch (update_mode) { - case VCOL_UPDATE_FOR_READ_WRITE: - if (triggers) - { - update= true; - break; - } case VCOL_UPDATE_FOR_READ: update= !vcol_info->stored_in_db && !(key_read && vf->part_of_key.is_set(file->active_index)) && bitmap_is_set(vcol_set, vf->field_index); + swap_values= 1; break; case VCOL_UPDATE_FOR_WRITE: - update= triggers || bitmap_is_set(vcol_set, vf->field_index); + update= bitmap_is_set(vcol_set, vf->field_index); break; - case VCOL_UPDATE_INDEXED: + case VCOL_UPDATE_FOR_REPLACE: update= !vcol_info->stored_in_db && (vf->flags & PART_KEY_FLAG) && bitmap_is_set(vcol_set, vf->field_index); + if (update && (vf->flags & BLOB_FLAG)) + { + /* + The row has been read into record[1] and Field_blob::value + contains the value for record[0]. Swap value and read_value + to ensure that the virtual column data for the read row will + be in read_value at the end of this function + */ + ((Field_blob*) vf)->swap_value_and_read_value(); + /* Ensure we call swap_value_and_read_value() after update */ + swap_values= 1; + } + break; + case VCOL_UPDATE_INDEXED: + /* Read indexed fields that was not updated in VCOL_UPDATE_FOR_READ */ + update= (!vcol_info->stored_in_db && (vf->flags & PART_KEY_FLAG) && + bitmap_is_set(vcol_set, vf->field_index) && + (key_read && vf->part_of_key.is_set(file->active_index))); + swap_values= 1; break; } @@ -7363,6 +7379,16 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) /* Compute the actual value of the virtual fields */ vcol_info->expr->save_in_field(vf, 0); DBUG_PRINT("info", ("field '%s' - updated", vf->field_name)); + if (swap_values && (vf->flags & BLOB_FLAG)) + { + /* + Remember the read value to allow other update_virtual_field() calls + for the same blob field for the row to be updated. + Field_blob->read_value always contains the virtual column data for + any read row. + */ + ((Field_blob*) vf)->swap_value_and_read_value(); + } } else { diff --git a/sql/table.h b/sql/table.h index b2d5599b74095..facba06a3cc43 100644 --- a/sql/table.h +++ b/sql/table.h @@ -327,9 +327,9 @@ enum release_type { RELEASE_NORMAL, RELEASE_WAIT_FOR_DROP }; enum enum_vcol_update_mode { VCOL_UPDATE_FOR_READ= 0, - VCOL_UPDATE_FOR_READ_WRITE, VCOL_UPDATE_FOR_WRITE, - VCOL_UPDATE_INDEXED + VCOL_UPDATE_INDEXED, + VCOL_UPDATE_FOR_REPLACE }; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 53157258d2e53..ca14542c9b54a 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1913,7 +1913,7 @@ innobase_row_to_mysql( } if (table->vfield) { my_bitmap_map* old_vcol_set = tmp_use_all_columns(table, table->vcol_set); - table->update_virtual_fields(VCOL_UPDATE_FOR_READ_WRITE); + table->update_virtual_fields(VCOL_UPDATE_FOR_READ); tmp_restore_column_map(table->vcol_set, old_vcol_set); } } diff --git a/storage/myisam/ha_myisam.cc b/storage/myisam/ha_myisam.cc index 9e09853871de5..881e90d95c117 100644 --- a/storage/myisam/ha_myisam.cc +++ b/storage/myisam/ha_myisam.cc @@ -664,8 +664,12 @@ static int compute_vcols(MI_INFO *info, uchar *record, int keynum) TABLE *table= (TABLE*)(info->external_ref); table->move_fields(table->field, record, table->field[0]->record_ptr()); if (keynum == -1) // update all vcols - return table->update_virtual_fields(VCOL_UPDATE_INDEXED); - + { + int error= table->update_virtual_fields(VCOL_UPDATE_FOR_READ); + if (table->update_virtual_fields(VCOL_UPDATE_INDEXED)) + error= 1; + return error; + } // update only one key KEY *key= table->key_info + keynum; KEY_PART_INFO *kp= key->key_part, *end= kp + key->ext_key_parts; From 7567cf5aef9c21b2c8dc404cfe10b4a189851163 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 30 Dec 2016 11:23:45 +0200 Subject: [PATCH 58/74] Fixes for using ssl with BUILD scripts. Needed as GNUTLS can't be used by MariaDB on many systems, like OpenSuse 13.2 --- BUILD/SETUP.sh | 7 +++---- cmake/configure.pl | 13 +++++++++---- mysql-test/valgrind.supp | 10 ++++++++++ 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/BUILD/SETUP.sh b/BUILD/SETUP.sh index 5f5b02500881c..d25b9d9d8917f 100755 --- a/BUILD/SETUP.sh +++ b/BUILD/SETUP.sh @@ -120,10 +120,9 @@ path=`dirname $0` get_make_parallel_flag # SSL library to use.--with-ssl will select our bundled yaSSL -# implementation of SSL. To use OpenSSL you will need to specify -# the location of OpenSSL headers and libs on your system. -# Ex --with-ssl=/usr -SSL_LIBRARY=--with-ssl +# implementation of SSL. --with-ssl=yes will first try system library +# then the boundled one --with-ssl=system will use the system library. +SSL_LIBRARY=--with-ssl=system if [ "x$warning_mode" = "xpedantic" ]; then warnings="-W -Wall -ansi -pedantic -Wno-long-long -Wno-unused -D_POSIX_SOURCE" diff --git a/cmake/configure.pl b/cmake/configure.pl index 68baf436c1caa..c502a172a22d2 100644 --- a/cmake/configure.pl +++ b/cmake/configure.pl @@ -165,21 +165,26 @@ sub check_compiler $cmakeargs = $cmakeargs." -DWITH_LIBEVENT=bundled"; next; } - if($option =~ /with-ssl=/) + if($option =~ /with-ssl=yes/) { $cmakeargs = $cmakeargs." -DWITH_SSL=yes"; next; } - if($option =~ /with-debug/) + if($option =~ /with-ssl=system/) { - $cmakeargs = $cmakeargs." -DCMAKE_BUILD_TYPE=Debug -DSECURITY_HARDENED=OFF"; + $cmakeargs = $cmakeargs." -DWITH_SSL=system"; next; } - if($option =~ /with-ssl/) + if($option =~ /with-ssl$/) { $cmakeargs = $cmakeargs." -DWITH_SSL=bundled"; next; } + if($option =~ /with-debug/) + { + $cmakeargs = $cmakeargs." -DCMAKE_BUILD_TYPE=Debug -DSECURITY_HARDENED=OFF"; + next; + } if($option =~ /prefix=/) { $cmake_install_prefix= substr($option, 7); diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 31b214d75f492..76c848dddadf8 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -1077,6 +1077,16 @@ fun:SSL_library_init } +{ + OpenSSL still reachable. + Memcheck:Leak + fun:*alloc + fun:CRYPTO_malloc + fun:sk_new + fun:SSL_COMP_get_compression_methods + fun:SSL_library_init +} + { libcrypto 2.2.1 leak Memcheck:Leak From 00f462cf1be56d4cbac420b88c12dc95bc99579d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 2 Jan 2017 11:03:25 +0100 Subject: [PATCH 59/74] dgcov: import, rewrite to work with cmake and git --- .gitignore | 1 + mysql-test/dgcov.pl | 198 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100755 mysql-test/dgcov.pl diff --git a/.gitignore b/.gitignore index ee52f228ea2e6..737528ffdd453 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.reject *.spec *.bak +*.dgcov *.rpm .*.swp *.ninja diff --git a/mysql-test/dgcov.pl b/mysql-test/dgcov.pl new file mode 100755 index 0000000000000..27647eed0569d --- /dev/null +++ b/mysql-test/dgcov.pl @@ -0,0 +1,198 @@ +#! /usr/bin/perl + +# Copyright (C) 2003,2008 MySQL AB +# Copyright (C) 2010,2017 Sergei Golubchik and MariaDB Corporation +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA + +# Run gcov and report test coverage on only those code lines touched by +# a given list of commits. + +use strict; +use warnings; + +use Getopt::Long; +use File::Find; +use Cwd qw/realpath/; + +my $opt_verbose=0; +my $opt_generate; +my $opt_help; +my $opt_purge; +my $opt_only_gcov; +my $opt_skip_gcov; + +my %cov; +my $file_no=0; + +GetOptions + ("v|verbose+" => \$opt_verbose, + "h|help" => \$opt_help, + "p|purge" => \$opt_purge, + "g|generate" => \$opt_generate, + "o|only-gcov" => \$opt_only_gcov, + "s|skip-gcov" => \$opt_skip_gcov, + ) or usage(); + +usage() if $opt_help; + +sub logv(@) { print STDERR @_,"\n" if $opt_verbose; } +sub gcov_prefix($) { defined($_[0]) ? $_[0] || '#####' : '-' } + +my $root= `git rev-parse --show-toplevel`; +chomp $root; + +die "Failed to find tree root" unless $root; +$root=realpath($root).'/'; +logv "Chdir $root"; +chdir $root or die "chdir($root): $!"; + +my $res; +my $cmd; +if ($opt_purge) +{ + $cmd= "find . -name '*.da' -o -name '*.gcda' -o -name '*.gcov' -o ". + "-name '*.dgcov' | grep -v 'README\.gcov' | xargs rm -f ''"; + logv "Running: $cmd"; + system($cmd)==0 or die "system($cmd): $? $!"; + exit 0; +} + +find(\&gcov_one_file, $root); +find(\&write_coverage, $root) if $opt_generate; +exit 0 if $opt_only_gcov; + +if (@ARGV) { + print_gcov_for_diff(@ARGV); +} else { + print_gcov_for_diff('HEAD') or print_gcov_for_diff('HEAD^'); +} +exit 0; + +sub print_gcov_for_diff { + $cmd="git diff --no-prefix --ignore-space-change @_"; + logv "Running: $cmd"; + open PIPE, '-|', $cmd or die "Failed to popen '$cmd': $!: $?"; + my ($lnum, $cnt, $fcov, $acc, $printme, $fname); + while () { + if (/^diff --git (.*) \1\n/) { + print $acc if $printme; + $fname=$1; + $acc="dgcov $fname"; + $acc=('*' x length($acc)) . "\n$acc\n" . ('*' x length($acc)); + $lnum=undef; + $fcov=$cov{realpath($fname)}; + $printme=0; + logv "File: $fname"; + next; + } + if (/^@@ -\d+,\d+ \+(\d+),(\d+) @@/ and $fcov) { + $lnum=$1; + $cnt=$2; + $acc.="\n@@ +$lnum,$cnt @\@$'"; + logv " lines: $lnum,",$lnum+$cnt; + next; + } + next unless $lnum and $cnt; + $acc.=sprintf '%9s:%5s:%s', '', $lnum, $' if /^ /; + ++$printme, $acc.=sprintf '%9s:%5s:%s', gcov_prefix($fcov->{$lnum}), $lnum, $' if /^\+/; + die "$_^^^ dying", unless /^[- +]/; + ++$lnum; + --$cnt; + } + print $acc if $printme; + close PIPE or die "command '$cmd' failed: $!: $?"; + return defined($fname); +} + +sub usage { + print <^ + +If no arguments are specified, it prints the coverage for all uncommitted +changes, if any, otherwise for the last commit. + +Options: + + -h --help This help. + -v --verbose Show commands run. + -p --purge Delete all test coverage information, to prepare for a + new coverage test. + -o --only-gcov Stop after running gcov, don't run git + -s --skip-gcov Do not run gcov, assume .gcov files are already in place + -g --generate Create .dgcov files for all source files + +Prior to running this tool, MariaDB should be built with + + cmake -DENABLE_GCOV=ON + +and the testsuite should be run. dgcov will report the coverage +for all lines modified in the specified commits. +END + + exit 1; +} + +sub gcov_one_file { + return unless /\.gcda$/; + unless ($opt_skip_gcov) { + $cmd= "gcov -i '$_' 2>/dev/null >/dev/null"; + print STDERR ++$file_no,"\r" if not $opt_verbose and -t STDERR; + logv "Running: $cmd"; + system($cmd)==0 or die "system($cmd): $? $!"; + } + + # now, read the generated file + open FH, '<', "$_.gcov" or die "open(<$_.gcov): $!"; + my $fname; + while () { + chomp; + if (/^function:/) { + next; + } + if (/^file:/) { + $fname=realpath($'); + next; + } + next if /^lcount:\d+,-\d+/; # whatever that means + unless (/^lcount:(\d+),(\d+)/ and $fname) { + warn "unknown line '$_' after running '$cmd'"; + next; + } + $cov{$fname}->{$1}+=$2; + } + close(FH); +} + +sub write_coverage { + my $fn=$File::Find::name; + my $h=$cov{$fn}; + return unless $h and $root eq substr $fn, 0, length($root); + open I, '<', $fn or die "open(<$fn): $!"; + open O, '>', "$fn.dgcov" or die "open(>$fn.dgcov): $!"; + logv "Annotating: ", substr $fn, length($root); + while () { + printf O '%9s:%5s:%s', gcov_prefix($h->{$.}), $., $_; + } + close I; + close O; +} From 53ae72e2ee657e3bdd348365db3b08844d24c3e3 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Mon, 2 Jan 2017 21:28:15 +0100 Subject: [PATCH 60/74] mtr uses dgcov for --gcov option --- mysql-test/lib/mtr_gcov.pl | 71 ------------------------------------ mysql-test/mysql-test-run.pl | 29 +++++---------- 2 files changed, 9 insertions(+), 91 deletions(-) delete mode 100644 mysql-test/lib/mtr_gcov.pl diff --git a/mysql-test/lib/mtr_gcov.pl b/mysql-test/lib/mtr_gcov.pl deleted file mode 100644 index 4c260d089b2d2..0000000000000 --- a/mysql-test/lib/mtr_gcov.pl +++ /dev/null @@ -1,71 +0,0 @@ -# -*- cperl -*- -# Copyright (c) 2004, 2011, Oracle and/or its affiliates. All rights reserved. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -# This is a library file used by the Perl version of mysql-test-run, -# and is part of the translation of the Bourne shell script with the -# same name. - -use strict; - -our $basedir; - -sub gcov_prepare ($) { - my ($dir)= @_; - print "Purging gcov information from '$dir'...\n"; - - system("find $dir -name \*.gcov -o -name \*.da" - . " -o -name \*.gcda | xargs rm"); -} - -# -# Collect gcov statistics. -# Arguments: -# $dir basedir, normally build directory -# $gcov gcov utility program [path] name -# $gcov_msg message file name -# $gcov_err error file name -# -sub gcov_collect ($$$) { - my ($dir, $gcov, $gcov_msg, $gcov_err)= @_; - - # Get current directory to return to later. - my $start_dir= cwd(); - - print "Collecting source coverage info using '$gcov'...$basedir\n"; - -f "$dir/$gcov_msg" and unlink("$dir/$gcov_msg"); - -f "$dir/$gcov_err" and unlink("$dir/$gcov_err"); - - my @dirs= `find "$dir" -type d -print | sort`; - #print "List of directories:\n@dirs\n"; - - foreach my $d ( @dirs ) { - chomp($d); - chdir($d) or next; - - my @flist= glob("*.*.gcno"); - print ("Collecting in '$d'...\n") if @flist; - - foreach my $f (@flist) { - system("$gcov $f 2>>$dir/$gcov_err >>$dir/$gcov_msg"); - system("perl", "$basedir/mysql-test/lib/process-purecov-annotations.pl", "$f.gcov"); - } - chdir($start_dir); - } - print "gcov info in $dir/$gcov_msg, errors in $dir/$gcov_err\n"; -} - - -1; diff --git a/mysql-test/mysql-test-run.pl b/mysql-test/mysql-test-run.pl index 9efc9035555de..b3fe2d2f20ab9 100755 --- a/mysql-test/mysql-test-run.pl +++ b/mysql-test/mysql-test-run.pl @@ -104,7 +104,6 @@ BEGIN require "mtr_process.pl"; require "mtr_io.pl"; -require "mtr_gcov.pl"; require "mtr_gprof.pl"; require "mtr_misc.pl"; @@ -248,11 +247,6 @@ END our $opt_clean_vardir= $ENV{'MTR_CLEAN_VARDIR'}; our $opt_gcov; -our $opt_gcov_src_dir="."; -our $opt_gcov_exe= "gcov"; -our $opt_gcov_err= "mysql-test-gcov.err"; -our $opt_gcov_msg= "mysql-test-gcov.msg"; - our $opt_gprof; our %gprof_dirs; @@ -383,11 +377,6 @@ sub main { # --help will not reach here, so now it's safe to assume we have binaries My::SafeProcess::find_bin(); - if ( $opt_gcov ) { - gcov_prepare($basedir . "/" . $opt_gcov_src_dir); - } - - print "vardir: $opt_vardir\n"; initialize_servers(); init_timers(); @@ -431,6 +420,10 @@ sub main { exit 0; } + if ($opt_gcov) { + system './dgcov.pl --purge'; + } + ####################################################################### my $num_tests= @$tests; if ( $opt_parallel eq "auto" ) { @@ -555,15 +548,15 @@ sub main { mtr_print_line(); - if ( $opt_gcov ) { - gcov_collect($basedir . "/" . $opt_gcov_src_dir, $opt_gcov_exe, - $opt_gcov_msg, $opt_gcov_err); - } - print_total_times($opt_parallel) if $opt_report_times; mtr_report_stats($prefix, $fail, $completed, $extra_warnings); + if ($opt_gcov) { + mtr_report("Running dgcov"); + system "./dgcov.pl --generate > $opt_vardir/last_changes.dgcov"; + } + if ( @$completed != $num_tests) { mtr_error("Not all tests completed (only ". scalar(@$completed) . @@ -1148,7 +1141,6 @@ sub command_line_setup { # Coverage, profiling etc 'gcov' => \$opt_gcov, - 'gcov-src-dir=s' => \$opt_gcov_src_dir, 'gprof' => \$opt_gprof, 'valgrind|valgrind-all' => \$opt_valgrind, 'valgrind-mysqltest' => \$opt_valgrind_mysqltest, @@ -6180,9 +6172,6 @@ ($) actions. Disable facility with NUM=0. gcov Collect coverage information after the test. The result is a gcov file per source and header file. - gcov-src-dir=subdir Collect coverage only within the given subdirectory. - For example, if you're only developing the SQL layer, - it makes sense to use --gcov-src-dir=sql gprof Collect profiling information using gprof. experimental= Refer to list of tests considered experimental; failures will be marked exp-fail instead of fail. From bf95970ec03e353f297c3b809219cee91ac40724 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 3 Jan 2017 15:47:17 +0200 Subject: [PATCH 61/74] Cleanups * Ensure that BUILD scripts cleans up things properly before new BUILD * MySQL -> MariaDB * Ignore wrong valgrind memleak --- BUILD/FINISH.sh | 4 ++-- BUILD/SETUP.sh | 2 +- mysql-test/valgrind.supp | 2 -- 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/BUILD/FINISH.sh b/BUILD/FINISH.sh index 86085fcc59373..b41c5c5139da2 100644 --- a/BUILD/FINISH.sh +++ b/BUILD/FINISH.sh @@ -32,9 +32,9 @@ then configure="$configure --verbose" fi +# git clean -fdX removes all ignored (build) files commands="\ -/bin/rm -rf configure; -/bin/rm -rf CMakeCache.txt CMakeFiles/ +git clean -fdX path=`dirname $0` . \"$path/autorun.sh\"" diff --git a/BUILD/SETUP.sh b/BUILD/SETUP.sh index d25b9d9d8917f..22357ce7fc8b2 100755 --- a/BUILD/SETUP.sh +++ b/BUILD/SETUP.sh @@ -42,7 +42,7 @@ Usage: $0 [-h|-n] [configure-options] Influences the debug flags. Old is default. --prefix=path Build with prefix 'path'. -Note: this script is intended for internal use by MySQL developers. +Note: this script is intended for internal use by MariaDB developers. EOF } diff --git a/mysql-test/valgrind.supp b/mysql-test/valgrind.supp index 76c848dddadf8..20d4b124dd72d 100644 --- a/mysql-test/valgrind.supp +++ b/mysql-test/valgrind.supp @@ -368,8 +368,6 @@ fun:_dl_init } - - # # dlclose can allocate memory for error message, the memory will be # freed by dlerror or other dl* function. From de22cd3fe5caa1db8839701e45f379b3b5be7328 Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 4 Jan 2017 18:45:23 +0200 Subject: [PATCH 62/74] Fixes for failures in test scripts and removal of some compiler warnings - rpl_gtid_stop_start: Random failure - sysvars_innodb,32bit: Fixes for 32 bit --- .../suite/rpl/r/rpl_gtid_stop_start.result | 1 + .../suite/rpl/t/rpl_gtid_stop_start.test | 1 + .../sys_vars/r/sysvars_innodb,32bit.rdiff | 117 ++++++++++-------- sql/table.cc | 5 +- storage/innobase/lock/lock0lock.cc | 2 +- 5 files changed, 69 insertions(+), 57 deletions(-) diff --git a/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result b/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result index 3ba178223192d..3f3b5e4344a6a 100644 --- a/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result +++ b/mysql-test/suite/rpl/r/rpl_gtid_stop_start.result @@ -8,6 +8,7 @@ include/stop_slave.inc Master_Log_File = 'master-bin.000001' Using_Gtid = 'No' CHANGE MASTER TO master_use_gtid=current_pos; +FLUSH LOGS; connection server_1; FLUSH LOGS; include/wait_for_purge.inc "master-bin.000002" diff --git a/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test b/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test index 65b13b57f2d60..09b35011f1ff5 100644 --- a/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test +++ b/mysql-test/suite/rpl/t/rpl_gtid_stop_start.test @@ -28,6 +28,7 @@ CHANGE MASTER TO master_use_gtid=current_pos; --write_file $MYSQLTEST_VARDIR/tmp/mysqld.2.expect wait EOF +FLUSH LOGS; --shutdown_server 30 --source include/wait_until_disconnected.inc diff --git a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff index b2556feb1ad3e..41ec54bb4facc 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff +++ b/mysql-test/suite/sys_vars/r/sysvars_innodb,32bit.rdiff @@ -1,5 +1,5 @@ ---- r/sysvars_innodb.result -+++ r/sysvars_innodb,32bit.result~ +--- ./suite/sys_vars/r/sysvars_innodb.result 2017-01-03 12:06:25.401683053 +0200 ++++ ./suite/sys_vars/r/sysvars_innodb,32bit.reject 2017-01-04 18:26:46.741752657 +0200 @@ -53,7 +53,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8 @@ -103,7 +103,7 @@ VARIABLE_COMMENT Helps in performance tuning in heavily concurrent environments. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000 -@@ -543,7 +543,7 @@ +@@ -557,7 +557,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 5 VARIABLE_SCOPE GLOBAL @@ -112,7 +112,7 @@ VARIABLE_COMMENT If the compression failure rate of a table is greater than this number more padding is added to the pages to reduce the failures. A value of zero implies no padding NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -571,7 +571,7 @@ +@@ -585,7 +585,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 50 VARIABLE_SCOPE GLOBAL @@ -121,7 +121,7 @@ VARIABLE_COMMENT Percentage of empty space on a data page that can be reserved to make the page compressible. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 75 -@@ -599,10 +599,10 @@ +@@ -613,10 +613,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 5000 VARIABLE_SCOPE GLOBAL @@ -134,7 +134,16 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -837,7 +837,7 @@ +@@ -641,7 +641,7 @@ + GLOBAL_VALUE_ORIGIN COMPILE-TIME + DEFAULT_VALUE 0 + VARIABLE_SCOPE GLOBAL +-VARIABLE_TYPE BIGINT UNSIGNED ++VARIABLE_TYPE INT UNSIGNED + VARIABLE_COMMENT InnoDB system tablespace size to be set in recovery. + NUMERIC_MIN_VALUE 0 + NUMERIC_MAX_VALUE 4294967295 +@@ -865,7 +865,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 120 VARIABLE_SCOPE GLOBAL @@ -143,7 +152,7 @@ VARIABLE_COMMENT Number of pages reserved in doublewrite buffer for batch flushing NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 127 -@@ -921,7 +921,7 @@ +@@ -949,7 +949,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -152,7 +161,7 @@ VARIABLE_COMMENT Speeds up the shutdown process of the InnoDB storage engine. Possible values are 0, 1 (faster) or 2 (fastest - crash-like). NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -935,7 +935,7 @@ +@@ -963,7 +963,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 600 VARIABLE_SCOPE GLOBAL @@ -161,7 +170,7 @@ VARIABLE_COMMENT Maximum number of seconds that semaphore times out in InnoDB. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 4294967295 -@@ -1005,7 +1005,7 @@ +@@ -1033,7 +1033,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 100 VARIABLE_SCOPE GLOBAL @@ -170,7 +179,7 @@ VARIABLE_COMMENT Percentage of B-tree page filled during bulk insert NUMERIC_MIN_VALUE 10 NUMERIC_MAX_VALUE 100 -@@ -1019,7 +1019,7 @@ +@@ -1047,7 +1047,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -179,7 +188,7 @@ VARIABLE_COMMENT Make the first page of the given tablespace dirty. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -1033,7 +1033,7 @@ +@@ -1061,7 +1061,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 30 VARIABLE_SCOPE GLOBAL @@ -188,7 +197,7 @@ VARIABLE_COMMENT Number of iterations over which the background flushing is averaged. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1000 -@@ -1061,7 +1061,7 @@ +@@ -1089,7 +1089,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -197,7 +206,7 @@ VARIABLE_COMMENT Controls the durability/speed trade-off for commits. Set to 0 (write and flush redo log to disk only once per second), 1 (flush to disk at each commit), 2 (write to log at commit but flush to disk only once per second) or 3 (flush to disk at prepare and at commit, slower and usually redundant). 1 and 3 guarantees that after a crash, committed transactions will not be lost and will be consistent with the binlog and other transactional engines. 2 can get inconsistent and lose transactions if there is a power failure or kernel crash but not if mysqld crashes. 0 has no guarantees in case of crash. 0 and 2 can be faster than 1 or 3. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 3 -@@ -1089,7 +1089,7 @@ +@@ -1117,7 +1117,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -206,7 +215,7 @@ VARIABLE_COMMENT Set to 0 (don't flush neighbors from buffer pool), 1 (flush contiguous neighbors from buffer pool) or 2 (flush neighbors from buffer pool), when flushing a block NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 2 -@@ -1145,7 +1145,7 @@ +@@ -1173,7 +1173,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -215,7 +224,7 @@ VARIABLE_COMMENT Helps to save your data in case the disk image of the database becomes corrupt. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 6 -@@ -1159,7 +1159,7 @@ +@@ -1187,7 +1187,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -224,7 +233,7 @@ VARIABLE_COMMENT Kills the server during crash recovery. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -1187,7 +1187,7 @@ +@@ -1215,7 +1215,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8000000 VARIABLE_SCOPE GLOBAL @@ -233,7 +242,7 @@ VARIABLE_COMMENT InnoDB Fulltext search cache size in bytes NUMERIC_MIN_VALUE 1600000 NUMERIC_MAX_VALUE 80000000 -@@ -1229,7 +1229,7 @@ +@@ -1257,7 +1257,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 84 VARIABLE_SCOPE GLOBAL @@ -242,7 +251,7 @@ VARIABLE_COMMENT InnoDB Fulltext search maximum token size in characters NUMERIC_MIN_VALUE 10 NUMERIC_MAX_VALUE 84 -@@ -1243,7 +1243,7 @@ +@@ -1271,7 +1271,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 3 VARIABLE_SCOPE GLOBAL @@ -251,7 +260,7 @@ VARIABLE_COMMENT InnoDB Fulltext search minimum token size in characters NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 16 -@@ -1257,7 +1257,7 @@ +@@ -1285,7 +1285,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2000 VARIABLE_SCOPE GLOBAL @@ -260,7 +269,7 @@ VARIABLE_COMMENT InnoDB Fulltext search number of words to optimize for each optimize table call NUMERIC_MIN_VALUE 1000 NUMERIC_MAX_VALUE 10000 -@@ -1271,7 +1271,7 @@ +@@ -1299,7 +1299,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2000000000 VARIABLE_SCOPE GLOBAL @@ -269,7 +278,7 @@ VARIABLE_COMMENT InnoDB Fulltext search query result cache limit in bytes NUMERIC_MIN_VALUE 1000000 NUMERIC_MAX_VALUE 4294967295 -@@ -1299,7 +1299,7 @@ +@@ -1327,7 +1327,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 2 VARIABLE_SCOPE GLOBAL @@ -278,7 +287,7 @@ VARIABLE_COMMENT InnoDB Fulltext search parallel sort degree, will round up to nearest power of 2 number NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 16 -@@ -1313,7 +1313,7 @@ +@@ -1341,7 +1341,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 640000000 VARIABLE_SCOPE GLOBAL @@ -287,7 +296,7 @@ VARIABLE_COMMENT Total memory allocated for InnoDB Fulltext Search cache NUMERIC_MIN_VALUE 32000000 NUMERIC_MAX_VALUE 1600000000 -@@ -1341,7 +1341,7 @@ +@@ -1369,7 +1369,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 100 VARIABLE_SCOPE GLOBAL @@ -296,7 +305,7 @@ VARIABLE_COMMENT Up to what percentage of dirty pages should be flushed when innodb finds it has spare resources to do so. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 100 -@@ -1383,10 +1383,10 @@ +@@ -1411,10 +1411,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 200 VARIABLE_SCOPE GLOBAL @@ -309,7 +318,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1395,12 +1395,12 @@ +@@ -1423,12 +1423,12 @@ SESSION_VALUE NULL GLOBAL_VALUE 2000 GLOBAL_VALUE_ORIGIN COMPILE-TIME @@ -325,7 +334,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1453,7 +1453,7 @@ +@@ -1495,7 +1495,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 50 VARIABLE_SCOPE SESSION @@ -334,7 +343,7 @@ VARIABLE_COMMENT Timeout in seconds an InnoDB transaction may wait for a lock before being rolled back. Values above 100000000 disable the timeout. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1073741824 -@@ -1467,10 +1467,10 @@ +@@ -1509,10 +1509,10 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 16777216 VARIABLE_SCOPE GLOBAL @@ -347,7 +356,7 @@ NUMERIC_BLOCK_SIZE 1024 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -1523,7 +1523,7 @@ +@@ -1565,7 +1565,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 2 VARIABLE_SCOPE GLOBAL @@ -356,7 +365,7 @@ VARIABLE_COMMENT Number of log files in the log group. InnoDB writes to the files in a circular fashion. NUMERIC_MIN_VALUE 2 NUMERIC_MAX_VALUE 100 -@@ -1565,7 +1565,7 @@ +@@ -1607,7 +1607,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8192 VARIABLE_SCOPE GLOBAL @@ -365,7 +374,7 @@ VARIABLE_COMMENT Redo log write ahead unit size to avoid read-on-write, it should match the OS cache block IO size NUMERIC_MIN_VALUE 512 NUMERIC_MAX_VALUE 16384 -@@ -1579,10 +1579,10 @@ +@@ -1621,10 +1621,10 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 1024 VARIABLE_SCOPE GLOBAL @@ -378,7 +387,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1635,10 +1635,10 @@ +@@ -1677,10 +1677,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -391,7 +400,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -1649,7 +1649,7 @@ +@@ -1691,7 +1691,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -400,7 +409,7 @@ VARIABLE_COMMENT Maximum delay of user threads in micro-seconds NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 10000000 -@@ -1747,7 +1747,7 @@ +@@ -1789,7 +1789,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 8 VARIABLE_SCOPE GLOBAL @@ -409,7 +418,7 @@ VARIABLE_COMMENT Number of multi-threaded flush threads NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 64 -@@ -1803,10 +1803,10 @@ +@@ -1845,10 +1845,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -422,7 +431,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY YES -@@ -1831,7 +1831,7 @@ +@@ -1873,7 +1873,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL @@ -431,7 +440,7 @@ VARIABLE_COMMENT Page cleaner threads can be from 1 to 64. Default is 4. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 64 -@@ -1859,7 +1859,7 @@ +@@ -1901,7 +1901,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 16 VARIABLE_SCOPE GLOBAL @@ -440,7 +449,7 @@ VARIABLE_COMMENT Number of rw_locks protecting buffer pool page_hash. Rounded up to the next power of 2 NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1024 -@@ -1873,7 +1873,7 @@ +@@ -1915,7 +1915,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 16384 VARIABLE_SCOPE GLOBAL @@ -449,7 +458,7 @@ VARIABLE_COMMENT Page size to use for all InnoDB tablespaces. NUMERIC_MIN_VALUE 4096 NUMERIC_MAX_VALUE 65536 -@@ -1915,7 +1915,7 @@ +@@ -1957,7 +1957,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 300 VARIABLE_SCOPE GLOBAL @@ -458,7 +467,7 @@ VARIABLE_COMMENT Number of UNDO log pages to purge in one batch from the history list. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 5000 -@@ -1929,7 +1929,7 @@ +@@ -1971,7 +1971,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -467,7 +476,7 @@ VARIABLE_COMMENT Dictates rate at which UNDO records are purged. Value N means purge rollback segment(s) on every Nth iteration of purge invocation NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -1971,7 +1971,7 @@ +@@ -2013,7 +2013,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL @@ -476,7 +485,7 @@ VARIABLE_COMMENT Purge threads can be from 1 to 32. Default is 4. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 32 -@@ -1999,7 +1999,7 @@ +@@ -2041,7 +2041,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 56 VARIABLE_SCOPE GLOBAL @@ -485,7 +494,7 @@ VARIABLE_COMMENT Number of pages that must be accessed sequentially for InnoDB to trigger a readahead. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 64 -@@ -2013,7 +2013,7 @@ +@@ -2055,7 +2055,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL @@ -494,7 +503,7 @@ VARIABLE_COMMENT Number of background read I/O threads in InnoDB. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 64 -@@ -2041,10 +2041,10 @@ +@@ -2083,10 +2083,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -507,7 +516,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2069,7 +2069,7 @@ +@@ -2111,7 +2111,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -516,7 +525,7 @@ VARIABLE_COMMENT Number of undo logs to use (deprecated). NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -2083,7 +2083,7 @@ +@@ -2125,7 +2125,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -525,7 +534,7 @@ VARIABLE_COMMENT An InnoDB page number. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 4294967295 -@@ -2139,7 +2139,7 @@ +@@ -2181,7 +2181,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1048576 VARIABLE_SCOPE GLOBAL @@ -534,7 +543,7 @@ VARIABLE_COMMENT Memory buffer size for index creation NUMERIC_MIN_VALUE 65536 NUMERIC_MAX_VALUE 67108864 -@@ -2153,10 +2153,10 @@ +@@ -2195,10 +2195,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 6 VARIABLE_SCOPE GLOBAL @@ -547,7 +556,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2349,7 +2349,7 @@ +@@ -2391,7 +2391,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 1 VARIABLE_SCOPE GLOBAL @@ -556,7 +565,7 @@ VARIABLE_COMMENT Size of the mutex/lock wait array. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 1024 -@@ -2377,10 +2377,10 @@ +@@ -2419,10 +2419,10 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 30 VARIABLE_SCOPE GLOBAL @@ -569,7 +578,7 @@ NUMERIC_BLOCK_SIZE 0 ENUM_VALUE_LIST NULL READ_ONLY NO -@@ -2419,7 +2419,7 @@ +@@ -2461,7 +2461,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -578,7 +587,7 @@ VARIABLE_COMMENT Helps in performance tuning in heavily concurrent environments. Sets the maximum number of threads allowed inside InnoDB. Value 0 will disable the thread throttling. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000 -@@ -2433,7 +2433,7 @@ +@@ -2475,7 +2475,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 10000 VARIABLE_SCOPE GLOBAL @@ -587,7 +596,7 @@ VARIABLE_COMMENT Time of innodb thread sleeping before joining InnoDB queue (usec). Value 0 disable a sleep NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 1000000 -@@ -2503,7 +2503,7 @@ +@@ -2545,7 +2545,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 128 VARIABLE_SCOPE GLOBAL @@ -596,7 +605,7 @@ VARIABLE_COMMENT Number of undo logs to use. NUMERIC_MIN_VALUE 1 NUMERIC_MAX_VALUE 128 -@@ -2531,7 +2531,7 @@ +@@ -2573,7 +2573,7 @@ GLOBAL_VALUE_ORIGIN COMPILE-TIME DEFAULT_VALUE 0 VARIABLE_SCOPE GLOBAL @@ -605,7 +614,7 @@ VARIABLE_COMMENT Number of undo tablespaces to use. NUMERIC_MIN_VALUE 0 NUMERIC_MAX_VALUE 95 -@@ -2615,7 +2615,7 @@ +@@ -2657,7 +2657,7 @@ GLOBAL_VALUE_ORIGIN CONFIG DEFAULT_VALUE 4 VARIABLE_SCOPE GLOBAL diff --git a/sql/table.cc b/sql/table.cc index b7f83a502b71f..41f764d1e59b5 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -2410,8 +2410,9 @@ int TABLE_SHARE::init_from_binary_frm_image(THD *thd, bool write, } } } - DBUG_ASSERT((table_check_constraints - share->check_constraints) == - share->table_check_constraints - share->field_check_constraints); + DBUG_ASSERT((uint) (table_check_constraints - share->check_constraints) == + (uint) (share->table_check_constraints - + share->field_check_constraints)); if (options) { diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 0a178abde37a2..5711a681e633a 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -4790,7 +4790,7 @@ lock_table( lock_mutex_enter(); DBUG_EXECUTE_IF("fatal-semaphore-timeout", - { os_thread_sleep(3600000000); }); + { os_thread_sleep(3600000000LL); }); /* We have to check if the new lock is compatible with any locks other transactions have in the table lock queue. */ From 135e144479c70d8e470e67fd95e4b17051127952 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 5 Jan 2017 01:07:03 +0200 Subject: [PATCH 63/74] MDEV-11598 Assertion `!table || (!table->read_set || bitmap_is_set(table->read_set, field_index))' failed Found and fixed 2 problems: - Filesort addon fields didn't mark virtual columns properly - multi-range-read calculated vcol bitmap but was not using it. This caused wrong vcol field to be calculated on read, which caused the assert. --- mysql-test/suite/vcol/inc/vcol_keys.inc | 51 ++++++++++++++++++ .../suite/vcol/r/vcol_keys_innodb.result | 47 ++++++++++++++++ .../suite/vcol/r/vcol_keys_myisam.result | 47 ++++++++++++++++ sql/field.cc | 11 ++++ sql/field.h | 3 ++ sql/filesort.cc | 32 +++++------ sql/opt_range.cc | 53 +++++++++++++------ sql/opt_range.h | 12 ++--- 8 files changed, 213 insertions(+), 43 deletions(-) diff --git a/mysql-test/suite/vcol/inc/vcol_keys.inc b/mysql-test/suite/vcol/inc/vcol_keys.inc index 4d4773ec3a6d8..7c9f60c0fb055 100644 --- a/mysql-test/suite/vcol/inc/vcol_keys.inc +++ b/mysql-test/suite/vcol/inc/vcol_keys.inc @@ -181,3 +181,54 @@ create table t1 (a int, b double as (rand())); --error ER_GENERATED_COLUMN_FUNCTION_IS_NOT_ALLOWED alter table t1 add index (b); drop table t1; + +# +# MDEV-11598 Assertion `!table || (!table->read_set... failed +# + +CREATE OR REPLACE TABLE t1 ( + f2 DOUBLE NOT NULL DEFAULT '0', + f3 DOUBLE NOT NULL DEFAULT '0', + f4 DOUBLE, + f5 DOUBLE DEFAULT '0', + v4 DOUBLE AS (IF(f4,f3,f2)) VIRTUAL, + KEY (f5), + KEY (v4) +); + +INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,4,1,0),(5,7,NULL,0); +INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f3, f5, f3 FROM t1; +INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,0,NULL,1); +INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f5, f5, f3 FROM t1; +DELETE FROM t1 WHERE f5 = 1 OR v4 = 4 ORDER BY f5,v4 LIMIT 9; +SELECT * from t1; +DROP TABLE t1; + +# Another similar failure + +CREATE TABLE t1 ( + d DECIMAL(63,0) NOT NULL DEFAULT 0, + c VARCHAR(64) NOT NULL DEFAULT '', + vd DECIMAL(63,0) AS (d) VIRTUAL, + vc VARCHAR(2048) AS (c) VIRTUAL, + pk BIGINT AUTO_INCREMENT, + PRIMARY KEY(pk)); + +INSERT INTO t1 (d,c) VALUES (0.5,'foo'); +SELECT * FROM t1 WHERE vc != 'bar' ORDER BY vd; +DROP TABLE t1; + +# +# MDEV-11729: Crash when using partial indexed virtual fields +# + +CREATE TABLE t1 ( + pk BIGINT, + c CHAR(64) NOT NULL DEFAULT '', + vc CHAR(64) AS (c) VIRTUAL, + PRIMARY KEY(pk), + INDEX(vc(32)) +); +DELETE FROM t1 WHERE vc IS NULL ORDER BY pk; +DROP TABLE t1; + diff --git a/mysql-test/suite/vcol/r/vcol_keys_innodb.result b/mysql-test/suite/vcol/r/vcol_keys_innodb.result index 43c911118c2e1..19e0db0033672 100644 --- a/mysql-test/suite/vcol/r/vcol_keys_innodb.result +++ b/mysql-test/suite/vcol/r/vcol_keys_innodb.result @@ -168,3 +168,50 @@ create table t1 (a int, b double as (rand())); alter table t1 add index (b); ERROR HY000: Function or expression 'rand()' cannot be used in the GENERATED ALWAYS AS clause of `b` drop table t1; +CREATE OR REPLACE TABLE t1 ( +f2 DOUBLE NOT NULL DEFAULT '0', +f3 DOUBLE NOT NULL DEFAULT '0', +f4 DOUBLE, +f5 DOUBLE DEFAULT '0', +v4 DOUBLE AS (IF(f4,f3,f2)) VIRTUAL, +KEY (f5), +KEY (v4) +); +INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,4,1,0),(5,7,NULL,0); +INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f3, f5, f3 FROM t1; +INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,0,NULL,1); +INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f5, f5, f3 FROM t1; +DELETE FROM t1 WHERE f5 = 1 OR v4 = 4 ORDER BY f5,v4 LIMIT 9; +SELECT * from t1; +f2 f3 f4 f5 v4 +5 7 NULL 0 5 +5 4 0 4 5 +5 7 0 7 5 +5 0 0 4 5 +5 0 0 7 5 +5 7 7 7 7 +5 1 1 0 1 +DROP TABLE t1; +CREATE TABLE t1 ( +d DECIMAL(63,0) NOT NULL DEFAULT 0, +c VARCHAR(64) NOT NULL DEFAULT '', +vd DECIMAL(63,0) AS (d) VIRTUAL, +vc VARCHAR(2048) AS (c) VIRTUAL, +pk BIGINT AUTO_INCREMENT, +PRIMARY KEY(pk)); +INSERT INTO t1 (d,c) VALUES (0.5,'foo'); +Warnings: +Note 1265 Data truncated for column 'd' at row 1 +SELECT * FROM t1 WHERE vc != 'bar' ORDER BY vd; +d c vd vc pk +1 foo 1 foo 1 +DROP TABLE t1; +CREATE TABLE t1 ( +pk BIGINT, +c CHAR(64) NOT NULL DEFAULT '', +vc CHAR(64) AS (c) VIRTUAL, +PRIMARY KEY(pk), +INDEX(vc(32)) +); +DELETE FROM t1 WHERE vc IS NULL ORDER BY pk; +DROP TABLE t1; diff --git a/mysql-test/suite/vcol/r/vcol_keys_myisam.result b/mysql-test/suite/vcol/r/vcol_keys_myisam.result index efca19db5bb69..eebf0eacafb11 100644 --- a/mysql-test/suite/vcol/r/vcol_keys_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_keys_myisam.result @@ -278,3 +278,50 @@ create table t1 (a int, b double as (rand())); alter table t1 add index (b); ERROR HY000: Function or expression 'rand()' cannot be used in the GENERATED ALWAYS AS clause of `b` drop table t1; +CREATE OR REPLACE TABLE t1 ( +f2 DOUBLE NOT NULL DEFAULT '0', +f3 DOUBLE NOT NULL DEFAULT '0', +f4 DOUBLE, +f5 DOUBLE DEFAULT '0', +v4 DOUBLE AS (IF(f4,f3,f2)) VIRTUAL, +KEY (f5), +KEY (v4) +); +INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,4,1,0),(5,7,NULL,0); +INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f3, f5, f3 FROM t1; +INSERT INTO t1 (f2,f3,f4,f5) VALUES (5,0,NULL,1); +INSERT INTO t1 (f2,f3,f4,f5) SELECT f2, f5, f5, f3 FROM t1; +DELETE FROM t1 WHERE f5 = 1 OR v4 = 4 ORDER BY f5,v4 LIMIT 9; +SELECT * from t1; +f2 f3 f4 f5 v4 +5 7 NULL 0 5 +5 4 0 4 5 +5 7 0 7 5 +5 0 0 4 5 +5 0 0 7 5 +5 7 7 7 7 +5 1 1 0 1 +DROP TABLE t1; +CREATE TABLE t1 ( +d DECIMAL(63,0) NOT NULL DEFAULT 0, +c VARCHAR(64) NOT NULL DEFAULT '', +vd DECIMAL(63,0) AS (d) VIRTUAL, +vc VARCHAR(2048) AS (c) VIRTUAL, +pk BIGINT AUTO_INCREMENT, +PRIMARY KEY(pk)); +INSERT INTO t1 (d,c) VALUES (0.5,'foo'); +Warnings: +Note 1265 Data truncated for column 'd' at row 1 +SELECT * FROM t1 WHERE vc != 'bar' ORDER BY vd; +d c vd vc pk +1 foo 1 foo 1 +DROP TABLE t1; +CREATE TABLE t1 ( +pk BIGINT, +c CHAR(64) NOT NULL DEFAULT '', +vc CHAR(64) AS (c) VIRTUAL, +PRIMARY KEY(pk), +INDEX(vc(32)) +); +DELETE FROM t1 WHERE vc IS NULL ORDER BY pk; +DROP TABLE t1; diff --git a/sql/field.cc b/sql/field.cc index 8d53fca27d79b..ff8b948ef62f9 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -10864,3 +10864,14 @@ bool Field::save_in_field_ignore_value(bool view_error_processing) return save_in_field_default_value(view_error_processing); return 0; // ignore } + + +void Field::register_field_in_read_map() +{ + if (vcol_info) + { + Item *vcol_item= vcol_info->expr; + vcol_item->walk(&Item::register_field_in_read_map, 1, 0); + } + bitmap_set_bit(table->read_set, field_index); +} diff --git a/sql/field.h b/sql/field.h index 400d2ef0e5e04..fd68ade116501 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1475,6 +1475,9 @@ class Field: public Value_source bool save_in_field_default_value(bool view_eror_processing); bool save_in_field_ignore_value(bool view_error_processing); + /* Mark field in read map. Updates also virtual fields */ + void register_field_in_read_map(); + friend int cre_myisam(char * name, register TABLE *form, uint options, ulonglong auto_increment_value); friend class Copy_field; diff --git a/sql/filesort.cc b/sql/filesort.cc index 2210dc569dfe1..6046693cba157 100644 --- a/sql/filesort.cc +++ b/sql/filesort.cc @@ -716,7 +716,7 @@ static ha_rows find_all_keys(THD *thd, Sort_param *param, SQL_SELECT *select, TABLE *sort_form; handler *file; MY_BITMAP *save_read_set, *save_write_set, *save_vcol_set; - + Item *sort_cond; DBUG_ENTER("find_all_keys"); DBUG_PRINT("info",("using: %s", (select ? select->quick ? "ranges" : "where": @@ -754,22 +754,22 @@ static ha_rows find_all_keys(THD *thd, Sort_param *param, SQL_SELECT *select, /* Remember original bitmaps */ save_read_set= sort_form->read_set; save_write_set= sort_form->write_set; - save_vcol_set= sort_form->vcol_set; + save_vcol_set= sort_form->vcol_set; + /* Set up temporary column read map for columns used by sort */ bitmap_clear_all(&sort_form->tmp_set); - /* Temporary set for register_used_fields and register_field_in_read_map */ - sort_form->read_set= &sort_form->tmp_set; + sort_form->column_bitmaps_set(&sort_form->tmp_set, &sort_form->tmp_set, + &sort_form->tmp_set); register_used_fields(param); if (quick_select) - select->quick->add_used_key_part_to_set(sort_form->read_set); + select->quick->add_used_key_part_to_set(); - Item *sort_cond= !select ? - 0 : !select->pre_idx_push_select_cond ? - select->cond : select->pre_idx_push_select_cond; + sort_cond= (!select ? 0 : + (!select->pre_idx_push_select_cond ? + select->cond : select->pre_idx_push_select_cond)); if (sort_cond) sort_cond->walk(&Item::register_field_in_read_map, 1, sort_form); - sort_form->column_bitmaps_set(&sort_form->tmp_set, &sort_form->tmp_set, - &sort_form->tmp_set); + sort_form->file->column_bitmaps_signal(); if (quick_select) { @@ -1259,7 +1259,6 @@ static void register_used_fields(Sort_param *param) { reg1 SORT_FIELD *sort_field; TABLE *table=param->sort_form; - MY_BITMAP *bitmap= table->read_set; for (sort_field= param->local_sortorder ; sort_field != param->end ; @@ -1269,14 +1268,7 @@ static void register_used_fields(Sort_param *param) if ((field= sort_field->field)) { if (field->table == table) - { - if (field->vcol_info) - { - Item *vcol_item= field->vcol_info->expr; - vcol_item->walk(&Item::register_field_in_read_map, 1, 0); - } - bitmap_set_bit(bitmap, field->field_index); - } + field->register_field_in_read_map(); } else { // Item @@ -1289,7 +1281,7 @@ static void register_used_fields(Sort_param *param) SORT_ADDON_FIELD *addonf= param->addon_field; Field *field; for ( ; (field= addonf->field) ; addonf++) - bitmap_set_bit(bitmap, field->field_index); + field->register_field_in_read_map(); } else { diff --git a/sql/opt_range.cc b/sql/opt_range.cc index 01b836fddf782..8446da519266e 100644 --- a/sql/opt_range.cc +++ b/sql/opt_range.cc @@ -11101,6 +11101,7 @@ int QUICK_RANGE_SELECT::reset() HANDLER_BUFFER empty_buf; MY_BITMAP * const save_read_set= head->read_set; MY_BITMAP * const save_write_set= head->write_set; + MY_BITMAP * const save_vcol_set= head->vcol_set; DBUG_ENTER("QUICK_RANGE_SELECT::reset"); last_range= NULL; cur_range= (QUICK_RANGE**) ranges.buffer; @@ -11114,7 +11115,8 @@ int QUICK_RANGE_SELECT::reset() } if (in_ror_merged_scan) - head->column_bitmaps_set_no_signal(&column_bitmap, &column_bitmap); + head->column_bitmaps_set_no_signal(&column_bitmap, &column_bitmap, + &column_bitmap); if (file->inited == handler::NONE) { @@ -11157,8 +11159,8 @@ int QUICK_RANGE_SELECT::reset() err: /* Restore bitmaps set on entry */ if (in_ror_merged_scan) - head->column_bitmaps_set_no_signal(save_read_set, save_write_set); - + head->column_bitmaps_set_no_signal(save_read_set, save_write_set, + save_vcol_set); DBUG_RETURN(error); } @@ -11189,13 +11191,16 @@ int QUICK_RANGE_SELECT::get_next() MY_BITMAP * const save_read_set= head->read_set; MY_BITMAP * const save_write_set= head->write_set; + MY_BITMAP * const save_vcol_set= head->vcol_set; /* We don't need to signal the bitmap change as the bitmap is always the same for this head->file */ - head->column_bitmaps_set_no_signal(&column_bitmap, &column_bitmap); + head->column_bitmaps_set_no_signal(&column_bitmap, &column_bitmap, + &column_bitmap); result= file->multi_range_read_next(&dummy); - head->column_bitmaps_set_no_signal(save_read_set, save_write_set); + head->column_bitmaps_set_no_signal(save_read_set, save_write_set, + save_vcol_set); DBUG_RETURN(result); } @@ -11372,7 +11377,7 @@ QUICK_SELECT_DESC::QUICK_SELECT_DESC(QUICK_RANGE_SELECT *q, used_key_parts (used_key_parts_arg) { QUICK_RANGE *r; - /* + /* Use default MRR implementation for reverse scans. No table engine currently can do an MRR scan with output in reverse index order. */ @@ -11847,62 +11852,76 @@ void QUICK_ROR_UNION_SELECT::add_keys_and_lengths(String *key_names, } -void QUICK_RANGE_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set) +void QUICK_RANGE_SELECT::add_used_key_part_to_set() { uint key_len; KEY_PART *part= key_parts; for (key_len=0; key_len < max_used_key_length; key_len += (part++)->store_length) { - bitmap_set_bit(col_set, part->field->field_index); + /* + We have to use field_index instead of part->field + as for partial fields, part->field points to + a temporary field that is only part of the original + field. field_index always points to the original field + */ + Field *field= head->field[part->field->field_index]; + field->register_field_in_read_map(); } } -void QUICK_GROUP_MIN_MAX_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set) +void QUICK_GROUP_MIN_MAX_SELECT::add_used_key_part_to_set() { uint key_len; KEY_PART_INFO *part= index_info->key_part; for (key_len=0; key_len < max_used_key_length; key_len += (part++)->store_length) { - bitmap_set_bit(col_set, part->field->field_index); + /* + We have to use field_index instead of part->field + as for partial fields, part->field points to + a temporary field that is only part of the original + field. field_index always points to the original field + */ + Field *field= head->field[part->field->field_index]; + field->register_field_in_read_map(); } } -void QUICK_ROR_INTERSECT_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set) +void QUICK_ROR_INTERSECT_SELECT::add_used_key_part_to_set() { List_iterator_fast it(quick_selects); QUICK_SELECT_WITH_RECORD *quick; while ((quick= it++)) { - quick->quick->add_used_key_part_to_set(col_set); + quick->quick->add_used_key_part_to_set(); } } -void QUICK_INDEX_SORT_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set) +void QUICK_INDEX_SORT_SELECT::add_used_key_part_to_set() { QUICK_RANGE_SELECT *quick; List_iterator_fast it(quick_selects); while ((quick= it++)) { - quick->add_used_key_part_to_set(col_set); + quick->add_used_key_part_to_set(); } if (pk_quick_select) - pk_quick_select->add_used_key_part_to_set(col_set); + pk_quick_select->add_used_key_part_to_set(); } -void QUICK_ROR_UNION_SELECT::add_used_key_part_to_set(MY_BITMAP *col_set) +void QUICK_ROR_UNION_SELECT::add_used_key_part_to_set() { QUICK_SELECT_I *quick; List_iterator_fast it(quick_selects); while ((quick= it++)) { - quick->add_used_key_part_to_set(col_set); + quick->add_used_key_part_to_set(); } } diff --git a/sql/opt_range.h b/sql/opt_range.h index 6970b87f6d811..9e4521a9437c1 100644 --- a/sql/opt_range.h +++ b/sql/opt_range.h @@ -1006,7 +1006,7 @@ class QUICK_SELECT_I This is used by an optimization in filesort. */ - virtual void add_used_key_part_to_set(MY_BITMAP *col_set)=0; + virtual void add_used_key_part_to_set()=0; }; @@ -1097,7 +1097,7 @@ class QUICK_RANGE_SELECT : public QUICK_SELECT_I virtual void replace_handler(handler *new_file) { file= new_file; } QUICK_SELECT_I *make_reverse(uint used_key_parts_arg); - virtual void add_used_key_part_to_set(MY_BITMAP *col_set); + virtual void add_used_key_part_to_set(); private: /* Default copy ctor used by QUICK_SELECT_DESC */ @@ -1261,7 +1261,7 @@ class QUICK_INDEX_SORT_SELECT : public QUICK_SELECT_I /* used to get rows collected in Unique */ READ_RECORD read_record; - virtual void add_used_key_part_to_set(MY_BITMAP *col_set); + virtual void add_used_key_part_to_set(); }; @@ -1336,7 +1336,7 @@ class QUICK_ROR_INTERSECT_SELECT : public QUICK_SELECT_I void add_keys_and_lengths(String *key_names, String *used_lengths); Explain_quick_select *get_explain(MEM_ROOT *alloc); bool is_keys_used(const MY_BITMAP *fields); - void add_used_key_part_to_set(MY_BITMAP *col_set); + void add_used_key_part_to_set(); #ifndef DBUG_OFF void dbug_dump(int indent, bool verbose); #endif @@ -1416,7 +1416,7 @@ class QUICK_ROR_UNION_SELECT : public QUICK_SELECT_I void add_keys_and_lengths(String *key_names, String *used_lengths); Explain_quick_select *get_explain(MEM_ROOT *alloc); bool is_keys_used(const MY_BITMAP *fields); - void add_used_key_part_to_set(MY_BITMAP *col_set); + void add_used_key_part_to_set(); #ifndef DBUG_OFF void dbug_dump(int indent, bool verbose); #endif @@ -1560,7 +1560,7 @@ class QUICK_GROUP_MIN_MAX_SELECT : public QUICK_SELECT_I bool unique_key_range() { return false; } int get_type() { return QS_TYPE_GROUP_MIN_MAX; } void add_keys_and_lengths(String *key_names, String *used_lengths); - void add_used_key_part_to_set(MY_BITMAP *col_set); + void add_used_key_part_to_set(); #ifndef DBUG_OFF void dbug_dump(int indent, bool verbose); #endif From 6c5c98316f6c5e4ad84ce5b4cb657459828e805d Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 5 Jan 2017 14:36:44 +0200 Subject: [PATCH 64/74] Updated mysql-test/README with information about KB --- mysql-test/README | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mysql-test/README b/mysql-test/README index 0fba1cc07e396..28b8203b2dce2 100644 --- a/mysql-test/README +++ b/mysql-test/README @@ -76,3 +76,9 @@ then put your .test file and .result file(s) into a tar.gz archive, add a README that explains the problem, ftp the archive to ftp://ftp.askmonty.org/private and submit a report to http://mariadb.org/jira about it. + +The latest information about mysql-test-run can be found at: +https://mariadb.com/kb/en/mariadb/mysqltest/ + +If you want to create .rdiff files, check +https://mariadb.com/kb/en/mariadb/mysql-test-auxiliary-files/ From c9b3e4535bb4b6d2aa0f7bc1ce71730e6aceca8b Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 9 Jan 2017 18:46:20 +0200 Subject: [PATCH 65/74] MDEV-11737 Failing assertion: block->magic_n == MEM_BLOCK_MAGIC_N Issue was that the m_prebuilt array was reused without resetting a counter, which caused a memory overrun. Adjusted test case to 79 characters --- .../suite/vcol/r/vcol_keys_innodb.result | 10 ++++ mysql-test/suite/vcol/t/vcol_keys_innodb.test | 59 +++++++++++-------- storage/innobase/handler/ha_innodb.cc | 3 + 3 files changed, 48 insertions(+), 24 deletions(-) diff --git a/mysql-test/suite/vcol/r/vcol_keys_innodb.result b/mysql-test/suite/vcol/r/vcol_keys_innodb.result index 19e0db0033672..5053b5105cf36 100644 --- a/mysql-test/suite/vcol/r/vcol_keys_innodb.result +++ b/mysql-test/suite/vcol/r/vcol_keys_innodb.result @@ -215,3 +215,13 @@ INDEX(vc(32)) ); DELETE FROM t1 WHERE vc IS NULL ORDER BY pk; DROP TABLE t1; +# +# MDEV-11737 Failing assertion: block->magic_n == MEM_BLOCK_MAGIC_N +# +CREATE TABLE t1 (i INT PRIMARY KEY, vi INT AS (i*2) VIRTUAL UNIQUE) +ENGINE=InnoDB; +CREATE TABLE t2 (i INT) ENGINE=InnoDB; +ALTER TABLE t1 ADD COLUMN col INT; +SELECT * FROM t1 WHERE vi < 2; +i vi col +DROP TABLE t1, t2; diff --git a/mysql-test/suite/vcol/t/vcol_keys_innodb.test b/mysql-test/suite/vcol/t/vcol_keys_innodb.test index 8eeef96b43fea..c872a9a6fe0d8 100644 --- a/mysql-test/suite/vcol/t/vcol_keys_innodb.test +++ b/mysql-test/suite/vcol/t/vcol_keys_innodb.test @@ -1,34 +1,34 @@ -################################################################################ -# t/vcol_keys_innodb.test # -# # -# Purpose: # -# Testing keys, indexes defined upon virtual columns. # -# # -# InnoDB branch # -# # -#------------------------------------------------------------------------------# -# Original Author: Andrey Zhakov # -# Original Date: 2008-09-04 # -# Change Author: # -# Change Date: # -# Change: # -################################################################################ +############################################################################### +# t/vcol_keys_innodb.test # +# # +# Purpose: # +# Testing keys, indexes defined upon virtual columns. # +# # +# InnoDB branch # +# # +#-----------------------------------------------------------------------------# +# Original Author: Andrey Zhakov # +# Original Date: 2008-09-04 # +# Change Author: # +# Change Date: # +# Change: # +############################################################################### # -# NOTE: PLEASE DO NOT ADD NOT MYISAM SPECIFIC TESTCASES HERE ! +# NOTE: PLEASE DO NOT ADD NOT INNODB SPECIFIC TESTCASES HERE ! # TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN # THE SOURCED FILES ONLY. # -#------------------------------------------------------------------------------# +#-----------------------------------------------------------------------------# # General not engine specific settings and requirements --source suite/vcol/inc/vcol_init_vars.pre -#------------------------------------------------------------------------------# +#-----------------------------------------------------------------------------# # Cleanup --source suite/vcol/inc/vcol_cleanup.inc -#------------------------------------------------------------------------------# +#-----------------------------------------------------------------------------# # Engine specific settings and requirements ##### Storage engine to be tested @@ -39,14 +39,25 @@ SET @@session.storage_engine = 'InnoDB'; ##### Workarounds for known open engine specific bugs # none -#------------------------------------------------------------------------------# +#-----------------------------------------------------------------------------# # Execute the tests to be applied to all storage engines let $with_foreign_keys = 1; --source suite/vcol/inc/vcol_keys.inc -#------------------------------------------------------------------------------# -# Execute storage engine specific tests - -#------------------------------------------------------------------------------# # Cleanup --source suite/vcol/inc/vcol_cleanup.inc + +#-----------------------------------------------------------------------------# +# Execute storage engine specific tests +#-----------------------------------------------------------------------------# + +--echo # +--echo # MDEV-11737 Failing assertion: block->magic_n == MEM_BLOCK_MAGIC_N +--echo # + +CREATE TABLE t1 (i INT PRIMARY KEY, vi INT AS (i*2) VIRTUAL UNIQUE) +ENGINE=InnoDB; +CREATE TABLE t2 (i INT) ENGINE=InnoDB; +ALTER TABLE t1 ADD COLUMN col INT; +SELECT * FROM t1 WHERE vi < 2; +DROP TABLE t1, t2; diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index ff80d9670922b..d9c32f286e6e4 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -8497,7 +8497,10 @@ ha_innobase::build_template( index_contains = dict_index_contains_col_or_prefix( index, num_v, true); if (index_contains) + { + m_prebuilt->n_template = 0; goto no_icp; + } } else { index_contains = dict_index_contains_col_or_prefix( index, i - num_v, false); From 177c191ff4b9c0033665a292fdb5907095ddb823 Mon Sep 17 00:00:00 2001 From: Monty Date: Mon, 9 Jan 2017 19:31:21 +0200 Subject: [PATCH 66/74] MDEV-11606 Server crashes in mi_make_key / sort_key_read Problem was that one internal record buffer in MYISAM was not big enough to handle virtual fields. Fixed by extending buffer. Fixed test case to 79 characters --- .../suite/vcol/r/vcol_keys_myisam.result | 270 ++++++++++-------- mysql-test/suite/vcol/t/vcol_keys_myisam.test | 245 ++++++++++++---- storage/myisam/mi_check.c | 2 +- storage/myisam/mi_open.c | 1 + 4 files changed, 351 insertions(+), 167 deletions(-) diff --git a/mysql-test/suite/vcol/r/vcol_keys_myisam.result b/mysql-test/suite/vcol/r/vcol_keys_myisam.result index eebf0eacafb11..078d6cfff1c0f 100644 --- a/mysql-test/suite/vcol/r/vcol_keys_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_keys_myisam.result @@ -1,119 +1,3 @@ -create table t1 (a int, b int as (a+1), c int, index(b)); -insert t1 (a,c) values (0x7890abcd, 0x76543210); -insert t1 (a,c) select seq, sin(seq)*10000 from seq_1_to_1000; -explain select * from t1 where b=10; -id select_type table type possible_keys key key_len ref rows Extra -1 SIMPLE t1 ref b b 5 const 1 -select * from t1 where b=10; -a b c -9 10 4121 - -MyISAM file: datadir/test/t1 -Record format: Fixed length -Character set: latin1_swedish_ci (8) -Data records: 1001 Deleted blocks: 0 -Recordlength: 9 - -table description: -Key Start Len Index Type -1 10 4 multip. long NULL -update t1 set a=20 where b=10; -select * from t1 where b=10; -a b c -select * from t1 where b=21; -a b c -20 21 4121 -20 21 9129 -delete from t1 where b=21; -select * from t1 where b=21; -a b c -alter table t1 add column d char(20) as (concat(a,c)); -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -create index i on t1 (d); -check table t1; -Table Op Msg_type Msg_text -test.t1 check status OK -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -check table t1 quick; -Table Op Msg_type Msg_text -test.t1 check status OK -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -check table t1 medium; -Table Op Msg_type Msg_text -test.t1 check status OK -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -check table t1 extended; -Table Op Msg_type Msg_text -test.t1 check status OK -show keys from t1; -Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -t1 1 b 1 b A 999 NULL NULL YES BTREE -t1 1 i 1 d A 999 NULL NULL YES BTREE -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -delete from t1 where b=12; -analyze table t1; -Table Op Msg_type Msg_text -test.t1 analyze status OK -show keys from t1; -Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -t1 1 b 1 b A 998 NULL NULL YES BTREE -t1 1 i 1 d A 998 NULL NULL YES BTREE -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -optimize table t1; -Table Op Msg_type Msg_text -test.t1 optimize status OK -show keys from t1; -Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment -t1 1 b 1 b A 998 NULL NULL YES BTREE -t1 1 i 1 d A 998 NULL NULL YES BTREE -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -repair table t1; -Table Op Msg_type Msg_text -test.t1 repair status OK -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -repair table t1 quick; -Table Op Msg_type Msg_text -test.t1 repair status OK -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -repair table t1 extended; -Table Op Msg_type Msg_text -test.t1 repair status OK -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -repair table t1 use_frm; -Table Op Msg_type Msg_text -test.t1 repair warning Number of rows changed from 0 to 998 -test.t1 repair status OK -select * from t1 where b=11; -a b c d -10 11 -5440 10-5440 -update t1 set a=30 where b=11; -select * from t1 where b=11; -a b c d -select * from t1 where b=31; -a b c d -30 31 -5440 30-5440 -30 31 -9880 30-9880 -drop table t1; SET @@session.storage_engine = 'MyISAM'; # - UNIQUE KEY # - INDEX @@ -325,3 +209,157 @@ INDEX(vc(32)) ); DELETE FROM t1 WHERE vc IS NULL ORDER BY pk; DROP TABLE t1; +# +# Original test +# +create table t1 (a int, b int as (a+1), c int, index(b)); +insert t1 (a,c) values (0x7890abcd, 0x76543210); +insert t1 (a,c) select seq, sin(seq)*10000 from seq_1_to_1000; +explain select * from t1 where b=10; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t1 ref b b 5 const 1 +select * from t1 where b=10; +a b c +9 10 4121 + +MyISAM file: datadir/test/t1 +Record format: Fixed length +Character set: latin1_swedish_ci (8) +Data records: 1001 Deleted blocks: 0 +Recordlength: 9 + +table description: +Key Start Len Index Type +1 10 4 multip. long NULL +update t1 set a=20 where b=10; +select * from t1 where b=10; +a b c +select * from t1 where b=21; +a b c +20 21 4121 +20 21 9129 +delete from t1 where b=21; +select * from t1 where b=21; +a b c +alter table t1 add column d char(20) as (concat(a,c)); +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +create index i on t1 (d); +check table t1; +Table Op Msg_type Msg_text +test.t1 check status OK +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +check table t1 quick; +Table Op Msg_type Msg_text +test.t1 check status OK +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +check table t1 medium; +Table Op Msg_type Msg_text +test.t1 check status OK +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +check table t1 extended; +Table Op Msg_type Msg_text +test.t1 check status OK +show keys from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 1 b 1 b A 999 NULL NULL YES BTREE +t1 1 i 1 d A 999 NULL NULL YES BTREE +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +delete from t1 where b=12; +analyze table t1; +Table Op Msg_type Msg_text +test.t1 analyze status OK +show keys from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 1 b 1 b A 998 NULL NULL YES BTREE +t1 1 i 1 d A 998 NULL NULL YES BTREE +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +optimize table t1; +Table Op Msg_type Msg_text +test.t1 optimize status OK +show keys from t1; +Table Non_unique Key_name Seq_in_index Column_name Collation Cardinality Sub_part Packed Null Index_type Comment Index_comment +t1 1 b 1 b A 998 NULL NULL YES BTREE +t1 1 i 1 d A 998 NULL NULL YES BTREE +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +repair table t1; +Table Op Msg_type Msg_text +test.t1 repair status OK +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +repair table t1 quick; +Table Op Msg_type Msg_text +test.t1 repair status OK +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +repair table t1 extended; +Table Op Msg_type Msg_text +test.t1 repair status OK +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +repair table t1 use_frm; +Table Op Msg_type Msg_text +test.t1 repair warning Number of rows changed from 0 to 998 +test.t1 repair status OK +select * from t1 where b=11; +a b c d +10 11 -5440 10-5440 +update t1 set a=30 where b=11; +select * from t1 where b=11; +a b c d +select * from t1 where b=31; +a b c d +30 31 -5440 30-5440 +30 31 -9880 30-9880 +drop table t1; +# +# MDEV-11606 Server crashes in mi_make_key / sort_key_read +# +CREATE TABLE t1 ( +pk BIGINT AUTO_INCREMENT, +col_date DATE NULL, +col_datetime DATETIME(1) NULL, +col_int TINYINT(13) UNSIGNED ZEROFILL NULL, +col_varchar VARBINARY(2222) NULL, +col_timestamp TIMESTAMP(2) NULL, +col_bit BIT(64) NOT NULL DEFAULT 0, +col_blob MEDIUMBLOB NULL, +col_dec DECIMAL(10,9) ZEROFILL NOT NULL DEFAULT 0, +col_time TIME(4) NULL, +col_year YEAR NOT NULL DEFAULT '1970', +col_char CHAR(129) NULL, +col_enum SET('','a','b','c','d','e','f','foo','bar') NULL, +vcol_dec DECIMAL(50,18) ZEROFILL AS (col_dec) VIRTUAL, +vcol_bit BIT(48) AS (col_bit) VIRTUAL, +vcol_char CHAR(224) AS (col_char) VIRTUAL, +vcol_datetime DATETIME(4) AS (col_datetime) VIRTUAL, +vcol_year YEAR AS (col_year) VIRTUAL, +vcol_varchar VARBINARY(356) AS (col_varchar) VIRTUAL, +vcol_blob MEDIUMBLOB AS (col_blob) VIRTUAL, +vcol_timestamp TIMESTAMP(5) AS (col_timestamp) VIRTUAL, +vcol_int BIGINT(46) AS (col_int) VIRTUAL, +vcol_time TIME(1) AS (col_time) VIRTUAL, +vcol_date DATE AS (col_date) VIRTUAL, +vcol_enum SET('','a','b','c','d','e','f','foo','bar') AS (col_enum) VIRTUAL, +UNIQUE(pk), +PRIMARY KEY(pk) +) ENGINE=MyISAM; +ALTER TABLE t1 ADD INDEX(col_enum,vcol_int); +ALTER TABLE t1 ADD INDEX(col_year); +DROP TABLE t1; diff --git a/mysql-test/suite/vcol/t/vcol_keys_myisam.test b/mysql-test/suite/vcol/t/vcol_keys_myisam.test index 68fd7e1731be3..049037216dcc5 100644 --- a/mysql-test/suite/vcol/t/vcol_keys_myisam.test +++ b/mysql-test/suite/vcol/t/vcol_keys_myisam.test @@ -1,6 +1,59 @@ - --source include/have_sequence.inc --let $datadir= `select @@datadir` +############################################################################### +# t/vcol_keys_myisam.test # +# # +# Purpose: # +# Testing keys, indexes defined upon virtual columns. # +# # +# MyISAM branch # +# # +#-----------------------------------------------------------------------------# +# Original Author: Andrey Zhakov # +# Original Date: 2008-09-04 # +# Change Author: # +# Change Date: # +# Change: # +############################################################################### + +# +# NOTE: PLEASE DO NOT ADD NOT MYISAM SPECIFIC TESTCASES HERE ! +# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN +# THE SOURCED FILES ONLY. +# + +#------------------------------------------------------------------------------ +# General not engine specific settings and requirements +--source suite/vcol/inc/vcol_init_vars.pre + +#------------------------------------------------------------------------------ +# Cleanup +--source suite/vcol/inc/vcol_cleanup.inc + +#------------------------------------------------------------------------------ +# Engine specific settings and requirements + +##### Storage engine to be tested +# Set the session storage engine +SET @@session.storage_engine = 'MyISAM'; + +##### Workarounds for known open engine specific bugs +# none + +#------------------------------------------------------------------------------ +# Execute the tests to be applied to all storage engines +--source suite/vcol/inc/vcol_keys.inc + +# Cleanup +--source suite/vcol/inc/vcol_cleanup.inc + +#-----------------------------------------------------------------------------# +# Execute storage engine specific test +#-----------------------------------------------------------------------------# + +--echo # +--echo # Original test +--echo # create table t1 (a int, b int as (a+1), c int, index(b)); insert t1 (a,c) values (0x7890abcd, 0x76543210); @@ -49,56 +102,148 @@ select * from t1 where b=31; --exec $MYISAMCHK $datadir/test/t1 --error 1 --exec $MYISAMCHK -r $datadir/test/t1 - drop table t1; -################################################################################ -# t/vcol_keys_myisam.test # -# # -# Purpose: # -# Testing keys, indexes defined upon virtual columns. # -# # -# MyISAM branch # -# # -#------------------------------------------------------------------------------# -# Original Author: Andrey Zhakov # -# Original Date: 2008-09-04 # -# Change Author: # -# Change Date: # -# Change: # -################################################################################ - -# -# NOTE: PLEASE DO NOT ADD NOT MYISAM SPECIFIC TESTCASES HERE ! -# TESTCASES WHICH MUST BE APPLIED TO ALL STORAGE ENGINES MUST BE ADDED IN -# THE SOURCED FILES ONLY. -# - -#------------------------------------------------------------------------------# -# General not engine specific settings and requirements ---source suite/vcol/inc/vcol_init_vars.pre - -#------------------------------------------------------------------------------# -# Cleanup ---source suite/vcol/inc/vcol_cleanup.inc - -#------------------------------------------------------------------------------# -# Engine specific settings and requirements - -##### Storage engine to be tested -# Set the session storage engine -SET @@session.storage_engine = 'MyISAM'; - -##### Workarounds for known open engine specific bugs -# none +--echo # +--echo # MDEV-11606 Server crashes in mi_make_key / sort_key_read +--echo # -#------------------------------------------------------------------------------# -# Execute the tests to be applied to all storage engines ---source suite/vcol/inc/vcol_keys.inc +CREATE TABLE t1 ( + pk BIGINT AUTO_INCREMENT, + col_date DATE NULL, + col_datetime DATETIME(1) NULL, + col_int TINYINT(13) UNSIGNED ZEROFILL NULL, + col_varchar VARBINARY(2222) NULL, + col_timestamp TIMESTAMP(2) NULL, + col_bit BIT(64) NOT NULL DEFAULT 0, + col_blob MEDIUMBLOB NULL, + col_dec DECIMAL(10,9) ZEROFILL NOT NULL DEFAULT 0, + col_time TIME(4) NULL, + col_year YEAR NOT NULL DEFAULT '1970', + col_char CHAR(129) NULL, + col_enum SET('','a','b','c','d','e','f','foo','bar') NULL, + vcol_dec DECIMAL(50,18) ZEROFILL AS (col_dec) VIRTUAL, + vcol_bit BIT(48) AS (col_bit) VIRTUAL, + vcol_char CHAR(224) AS (col_char) VIRTUAL, + vcol_datetime DATETIME(4) AS (col_datetime) VIRTUAL, + vcol_year YEAR AS (col_year) VIRTUAL, + vcol_varchar VARBINARY(356) AS (col_varchar) VIRTUAL, + vcol_blob MEDIUMBLOB AS (col_blob) VIRTUAL, + vcol_timestamp TIMESTAMP(5) AS (col_timestamp) VIRTUAL, + vcol_int BIGINT(46) AS (col_int) VIRTUAL, + vcol_time TIME(1) AS (col_time) VIRTUAL, + vcol_date DATE AS (col_date) VIRTUAL, + vcol_enum SET('','a','b','c','d','e','f','foo','bar') AS (col_enum) VIRTUAL, + UNIQUE(pk), + PRIMARY KEY(pk) + ) ENGINE=MyISAM; -#------------------------------------------------------------------------------# -# Execute storage engine specific tests - -#------------------------------------------------------------------------------# -# Cleanup ---source suite/vcol/inc/vcol_cleanup.inc +--disable_query_log +INSERT INTO t1 (col_date,col_datetime,col_int,col_varchar,col_timestamp,col_bit,col_blob,col_dec,col_time,col_year,col_char,col_enum) VALUES + (NULL,'2011-04-17 15:46:11.056462',6,'rsprn','1980-01-01 00:00:00',b'0001011111000111001110000110100110010101101','spr',0.0,'00:00:00',1988,'p',''), + ('2007-05-18',NULL,5,'rnwg','2009-07-07 23:46:32.052699',b'01010','n',0.6,'04:35:56.018027',1995,'wgrpq',''), + ('1994-03-16','2006-03-15 22:48:25.013225',7,'grpquarw','2034-05-17 10:51:23.048265',b'0001011101001011000111101','rp',0.3,'15:13:22.043368',2004,'pq',''), + ('1980-01-01','1987-06-01 04:14:04.027480',1,'qu','1989-07-05 09:46:16.038513',b'001111011000100011100111010100101010000100010100101','uar',0.6,'11:56:35.031314',1986,'',''), + ('2014-08-16','2021-01-08 20:59:16.041252',0,'arw','2022-12-20 22:48:53.014627',b'110110101100001011001110110100','m',0.6,'17:26:26.039855',2008,'rw',''), + ('1981-12-02','1992-02-10 09:29:41.028674',6,'wk','2017-09-25 10:37:25.043516',b'010101001110111010101001101000101010',NULL,0.7,'00:00:00',2035,'kaz',''), + ('2022-11-11','2029-03-07 17:24:19.043489',122,'a','1980-01-01 00:00:00',b'0001100111011','z',0.0,'10:23:45.050733',2018,'a',''), + ('2004-03-12','1979-02-18 00:00:00',0,'zjeiwvd','1998-07-03 16:09:05.053954',b'0000110101111001001110100100111001111111100001110','',0.4,'07:43:46.015324',2028,'je',''), + ('2016-08-08','1986-03-10 00:00:00',3,'eiwv','2025-08-07 12:24:53.040869',b'010001101110100111111','iw',0.3,'02:48:45.058781',2020,'',''), + ('1979-02-25',NULL,2,'wvd','1980-01-01 00:00:00',b'010','vdm',0.1,'13:35:24.021296',1995,'dmd',''), + ('2012-04-19','2034-10-06 23:29:21.057367',9,'mdiv','2022-05-16 05:28:40.005808',b'110111111101000010011011001','divj',0.3,'11:24:50.017885',1997,'iv',''), + ('2009-10-13','1997-01-21 13:04:34.011524',1,'vjqs','1980-01-01 00:00:00',b'11110011100101011100001110110000101001100010000011110110011','j',0.7,'00:00:00',2008,'qsxmh',''), + ('2012-01-01','2011-06-23 06:11:33.014723',9,'r','2022-04-27 05:29:32.023395',b'1001011010101100100111','',0.5,'03:34:01.048002',1976,'sx',''), + ('2007-08-24','2012-01-24 22:16:53.014811',2,'fpp','2005-10-11 08:58:32.021273',b'011011111011',NULL,0.4,'18:11:17.036115',2026,'m',''), + (NULL,'1985-05-28 00:00:00',7,'hjw','2030-06-10 10:15:54.061818',b'0011110101001100011011000101010','',0.5,'02:03:21.020237',1994,'z',''), + ('2006-09-09','2002-11-10 06:16:27.008631',0,'jw','2030-07-16 00:00:00',b'11101110111101000010101110000010001110110001001101110100','wa',0.8,'00:00:00',1974,'',''), + ('1993-04-22','1980-01-01 00:00:00',3,'a','2020-02-06 08:11:00.051515',b'001110110010','gew',0.4,'11:59:05.013933',1998,NULL,''), + ('1985-11-05','2019-12-24 04:13:43.062741',7,'foo','1986-07-02 00:00:00',b'10100110001010100011111111101011011101001110010110101100110000','',0.5,'17:32:55.060182',1978,'w',''), + ('2007-02-20','2033-10-16 18:47:42.006517',1,'clc','1971-10-14 00:00:00',b'100001010','',0.1,'20:38:42.062598',2004,'lcfy',''), + ('1979-12-10','1972-08-04 00:00:00',8,'','2021-12-24 04:51:09.011443',b'011010010010010010011101000010110011110110110010100010','e',0.1,'23:08:10.014396',2026,'cfykywl',''), + ('2002-07-04',NULL,NULL,'fyky','1980-11-02 00:00:00',b'10000010111010000110','f',0.6,'13:07:14.014203',1979,'ykywl',''), + ('2019-02-15','2004-08-28 10:16:46.053753',8,'kywl','2004-12-01 00:00:00',b'010111010110001110110011000010110111011','ywlcne',0.2,'09:10:39.028897',2031,'wlcnemiuaab',''), + ('1989-05-15','1973-09-04 00:00:00',2,'lcne',NULL,b'100001100101110110000011110','',0.9,'19:22:16.015548',2013,'cnemiuaa',''), + ('2023-04-20','2018-12-04 04:19:46.040664',8,NULL,'1985-04-18 05:32:12.013509',b'111011011111100100000001','nem',0.1,'23:18:05.007794',2026,'emiua',''), + (NULL,'2024-08-10 03:52:31.047182',2,'miua','2035-09-03 06:07:22.008308',b'000111','',0.1,'03:58:02.003946',1988,'iua',''), + ('1977-09-01','1987-02-01 03:44:00.061840',5,'u','1997-11-27 04:02:13.014551',b'111110001001100101101000100010001011101000011',NULL,0.3,'00:00:00',1985,'aab',''), + (NULL,'1980-01-01 00:00:00',6,'a','1989-03-02 09:07:11.058643',b'10101001011110110010111010010100001010000110000110010',NULL,0.8,'17:41:15.057101',1975,'br',''), + ('1980-01-01','2019-02-10 20:56:51.063985',NULL,'rr','2000-02-28 01:38:27.004468',b'0101001011110001010001','rifnhu',0.5,'20:55:57.029281',1973,'if',''), + (NULL,'2019-04-15 02:13:03.019581',7,'fnhu','2000-03-25 18:48:46.063113',b'011000110','nhuu',0.0,'00:00:00',2030,'h',''), + ('1997-04-01',NULL,3,NULL,'1976-05-22 04:48:42.013754',b'101110101010000111101',NULL,0.1,'12:09:48.030076',2031,'u',''), + (NULL,'1993-10-10 15:11:03.021823',NULL,'ufzasunkrcpvasdqkxbwptigbpnesqigwegcnfeuvrgnecpthm','1986-06-12 04:29:01.017855',b'11110','fza',0.9,NULL,1994,'zasun',''), + ('1997-02-13','2001-04-08 02:01:53.018388',0,'as','1975-09-18 00:00:00',b'00111000101001001101001000100100010101110011010111000001011011','su',0.3,'15:15:31.011102',1972,'unk',''), + ('1975-07-26','2022-12-24 00:00:00',0,'foo','1986-01-22 21:27:38.024505',b'1111110000000001000010111','krc',0.9,'05:46:08.055789',1996,'rcp',''), + ('2000-08-04','1980-01-01 00:00:00',48,'foo','2015-12-21 17:04:06.008790',b'0100001110011001011101011101110110100010000101101100011010','g',0.7,'18:31:50.046211',2025,'pvasdq',''), + ('2001-03-26','2032-03-19 13:15:13.063368',NULL,'foo','2026-09-05 02:46:10.026919',b'11','a',0.3,'03:12:20.039599',2019,'sdqkx',''), + ('2035-01-14','1987-09-15 05:46:00.041527',4,'p','1980-01-01 00:00:00',b'1111100111','dqkxbwpt',0.0,'09:57:41.059145',1995,'q',''), + ('2035-07-13','2011-04-13 04:05:14.025091',212,'kxbw','1985-02-14 11:58:32.002055',b'1010100011110101011111111011010','x',0.2,'19:35:02.024655',1975,'bwptig',''), + ('2026-02-08','2015-05-04 09:31:22.017074',4,'wpt','2024-01-26 11:06:03.057899',b'011000010000100000011000011011000100101111001100000111011010','ptig',0.0,'00:00:00',1977,'tig',''), + ('1981-05-11',NULL,NULL,'foo','1981-12-09 10:10:34.008107',b'01000100100100110011111','gbp',0.0,'13:05:42.035253',2019,'bpn',''), + ('1977-05-16',NULL,9,'pne',NULL,b'001101100111001110110010111001110100','s',0.0,'15:09:37.063819',1998,'ne',''), + ('1980-01-01','2018-12-02 00:27:35.056455',2,'','1981-07-07 23:39:32.028644',b'0000101001010111010001101000','es',0.3,'15:43:30.016638',2013,NULL,''), + ('2027-09-05','1998-05-14 04:15:42.009728',1,'s','2015-07-16 00:00:00',b'01011101101010000110011010000111001000001000011','',0.6,'08:39:24.041879',2035,'qigweg',''), + ('2005-02-04',NULL,2,'i','1974-01-11 11:02:16.024653',b'01001101110001001101101010011001001101010010000','gw',0.6,'03:28:30.012172',1978,'weg',''), + (NULL,'1976-06-21 00:00:00',5,NULL,'2023-11-25 15:49:52.021725',b'101011010001100','e',0.1,'00:00:00',1977,'gcn',''), + ('1989-07-13',NULL,1,'c','1978-02-22 02:55:14.047104',b'01101010100001100110111011101000111011101101110011','f',0.8,NULL,1987,'nfeu',''), + ('2004-04-27','2019-06-28 08:04:35.039213',0,'f','1990-01-09 14:22:27.065127',b'00101001011','eu',0.0,'13:33:09.039791',2007,'uvrgne',''), + ('2008-09-08','1990-11-05 00:00:00',1,'w','2026-12-23 00:00:00',b'0001101','vrgnec',0.3,'19:13:14.037732',1983,'d',''), + ('2026-08-12','2026-11-23 11:18:35.012315',4,'rgnec','1988-09-06 07:11:55.057710',b'11010111001001101100100010110011100001000100001011000000000010','',0.9,NULL,1982,'gnecpth',''), + ('1992-12-03','2033-08-18 04:47:11.033829',65,'n','1989-11-21 17:42:13.012747',b'11011011110000000',NULL,0.9,'10:08:34.006377',1971,'ecpth',''), + ('1976-10-11','1975-05-18 00:00:00',3,'c','2017-11-06 03:33:38.002741',b'0111100010000111000111111100111100111000101100111111100','p',0.7,'03:28:07.039921',2014,'thmhf',''), + ('2014-04-19','2023-08-07 16:18:59.024013',0,'','2006-05-04 23:01:46.019351',b'0101101011011101101011101110000001001000110100101000011001110','h',0.5,'00:00:00',1995,'m',''), + ('1990-07-16','1980-01-01 00:00:00',8,'hffqbythjwpukqubzpomntrddrwhzjtqvb','1985-08-04 05:33:20.030471',b'001101111111100110101111000011100','ff',0.0,'00:00:00',2002,'fqbythj',''), + (NULL,'2019-01-12 00:00:00',0,NULL,'2009-01-25 00:00:00',b'100111111010000110010011100100000011101001010101111','qb',0.4,'20:35:33.059895',1981,'byt',''), + ('1991-10-07',NULL,2,'yt','2027-04-19 06:38:46.020191',b'001','t',0.4,'10:02:06.014126',2004,'h',''), + (NULL,'2009-07-05 00:00:00',241,'j','1981-06-26 12:35:20.061910',b'10101110001101001000011010010111000','wpukqu',0.5,'00:00:00',1973,'pukqu',''), + ('2001-05-26','2007-01-06 00:57:02.048605',0,'u',NULL,b'111100','k',0.7,'03:19:10.052988',2026,'q',NULL), + ('2008-03-15','1990-09-11 00:00:00',5,'ubz','1980-01-01 00:00:00',b'11010111011110001101111000000011000111101100111','b',0.5,'00:34:27.006616',2013,NULL,''), + ('1984-08-01','2000-09-20 09:35:47.025609',3,'zp','2016-11-22 19:38:52.053299',b'00000010','po',0.7,'19:47:19.014687',1996,'o',''), + ('1978-02-05','1978-05-08 04:30:57.023271',7,'foo','2000-04-06 08:42:13.019650',b'11000110111100101010001110111101111000001101','n',0.8,NULL,1980,'trdd',''), + ('2017-04-11','2002-09-26 12:59:43.051659',8,'rd','1972-03-27 13:09:07.017459',b'00011110001001001000000100110100101010','ddrwh',0.7,'00:00:00',2021,'drwhzj',''), + ('1980-01-01','1986-05-04 05:15:19.008418',0,'r','2005-10-04 09:21:09.020131',b'1101100010101001010011010001011101001111110010101111011','wh',0.0,'00:00:00',1975,'hzjtqv',''), + ('2035-12-12','1980-01-01 00:00:00',0,'x',NULL,b'0010000101010110111100000110000010001000100001000110111000010110','zjt',0.4,'15:51:12.040679',1984,'jtqvbji',''), + ('1993-05-12','2000-11-11 20:54:49.053753',0,'tqvb','2022-02-26 14:26:36.004981',b'110000101110000111011','qv',0.7,'00:00:00',1972,'v',''), + ('1971-08-22','2029-02-15 16:39:35.007278',2,NULL,'2033-09-22 08:28:19.057517',b'11111101011101110111100011011111001','bji',0.8,'08:34:37.000701',2000,'o',''), + (NULL,NULL,5,'foo','1982-02-24 00:00:00',b'00111111000111111111010111010111011101','iklce',0.8,'01:23:11.014485',2021,'klcek',''), + (NULL,NULL,8,'lce','1988-07-28 11:48:23.011427',b'101101000101010000000100000001011','',0.3,'17:15:34.034697',1991,'cekxqy',''), + ('2029-12-07','1993-12-24 00:45:29.060155',3,'ekx','1980-01-01 00:00:00',b'01001010110110000100100100111010110000000101001011111110001100','q',0.7,'10:39:47.004022',2006,'foo',''), + ('2015-10-20','1980-01-01 00:00:00',189,'xqy','2028-12-19 00:00:00',b'101001011011100101110010101000101110100110','qy',0.5,'15:16:59.059052',1993,'foo',''), + ('1998-08-07','2017-08-07 01:53:34.056737',5,'oxsolbx',NULL,b'1000111010110010110','xsolbxth',0.3,'22:56:09.003450',2014,'s',''), + ('2016-01-25','2000-09-14 22:35:41.048328',6,'foo','2004-10-11 00:00:00',b'001','olbxt',0.0,'14:15:54.033066',1983,'lbxt',''), + ('1979-09-02','2027-01-19 09:34:15.034597',4,'bxth','1989-10-23 09:11:09.055445',b'011011001110000011011011',NULL,0.8,'05:31:31.006489',1978,'xthdc',''), + ('1980-01-01',NULL,8,'th','2012-02-07 00:00:00',b'00101011001100111001101011010110','hdc',0.0,'22:09:17.054381',2013,'dcprs',''), + (NULL,'2018-08-03 17:37:14.049040',2,'cprswpj','1990-07-28 07:56:50.026324',b'0000010111011110100100010010011011010010001111011010000010011101','',0.1,NULL,1971,'prswpjx',''), + ('1984-05-07','2012-05-07 00:00:00',1,'rswp','2030-05-09 07:42:25.003848',b'1001','swp',0.4,'13:27:32.040813',1997,'wpj',''), + (NULL,'2030-03-22 14:03:46.000742',7,'pjxixm','2022-05-11 00:00:00',b'00111110011001010010001111010001111110010010000111','j',0.8,'00:00:00',1996,'x',NULL), + ('2000-12-03','2020-08-13 16:03:09.041436',8,'ix','1985-05-19 11:28:09.002728',b'11011010000101000110111111010111','xmvfwm',0.9,'19:06:00.002417',1976,'m',''), + ('1990-03-13','2035-09-08 21:29:04.011731',5,'vfwmsys','2029-11-03 04:28:54.058532',b'0110001011001010100','fw',0.6,'20:30:32.032224',1994,'wmsys',''), + ('2035-06-04','2027-06-07 11:27:21.038934',8,'ms','1987-09-02 00:00:00',b'001101101101111110010110110011','syse',0.1,'01:10:53.060943',2027,'yse',''), + (NULL,'1993-06-06 07:29:56.029103',NULL,'','1971-06-08 23:51:55.054403',b'11001110001111111001001010101110111011000100111010','se',0.2,'10:24:53.013058',1995,'eb',''), + ('2018-06-12','2020-08-06 23:47:35.060301',5,NULL,NULL,b'110011110111010111','blwc',0.6,NULL,1971,'lwc',''), + ('2005-02-03','2016-10-11 00:00:00',0,'w','2005-03-25 00:00:00',b'0101001001001','',0.2,'21:56:26.025743',1971,'c',''), + ('2025-10-10',NULL,1,'vumvyv','2034-05-06 18:17:26.004829',b'10110101101100100001000011001111100100111101100','um',0.3,'19:42:29.005509',1992,'m',''), + ('2014-01-27','1980-01-01 00:00:00',2,'vyvb','1975-04-08 10:13:06.052060',b'111011110111111010011111011011101111','yv',0.4,'15:13:32.059509',2011,NULL,''), + ('2016-08-04','2008-12-03 01:55:41.030042',6,'vb','1993-09-08 00:01:40.016566',b'000101110101100111001101010110','b',0.5,'08:39:05.055786',1993,'it',''), + ('1980-01-01','1980-01-01 00:00:00',9,'tx','1991-07-19 05:37:43.056696',b'110110111010111101010001100010111100110011111010100100100','k',0.9,'00:00:00',1999,'h',''), + (NULL,'2007-06-14 15:47:29.017306',9,'foo',NULL,b'0100111000001011111000111010000011011000011000101010','qjxdzd',0.3,'13:52:31.035851',2001,'jxdzd',''), + (NULL,'2022-04-28 18:27:19.060240',4,'foo','2018-02-28 02:49:16.013066',b'10000100000000110011011110101110100001100011101110011011100','d',0.0,'15:01:48.022368',1987,'zdytun',''), + ('1987-10-10','1975-06-02 02:47:57.012240',9,'dy','1980-06-10 00:00:00',b'110000111','',0.7,'08:13:44.003967',1981,'',''), + ('2015-09-28',NULL,4,'d','2005-08-09 08:35:04.039832',b'00','b',0.1,'00:47:43.048164',1973,'yt',''), + ('2030-01-18','2011-12-09 00:00:00',6,NULL,'1982-05-13 00:00:00',b'111110100001000010001110110010111100001011010','v',0.2,'00:39:24.001557',2027,'tun',''), + ('1980-06-28','2016-04-07 00:00:00',8,'y','1980-01-01 00:00:00',b'01011100100001010110101110111110110','unq',0.7,'00:00:00',2016,'foo',''), + ('2018-09-16','1975-01-03 00:00:00',NULL,'qv','2028-05-10 00:00:00',b'0001100101001011100110','vv',0.0,'23:54:42.064230',2000,'vr',''), + ('2033-12-11','2021-07-23 12:20:17.025201',3,'rm','1996-07-16 00:00:00',b'1011001110','mpyx',0.7,'04:04:01.055956',2009,'pyx',''), + ('1982-12-18','1996-09-16 00:00:00',9,'yx','2003-10-14 03:54:44.012072',b'11001011011000001111011000101111101110100101','x',0.9,NULL,1981,'rencqh',''), + ('1980-01-01','2009-01-17 12:11:34.030449',7,'encqh','1980-01-01 00:00:00',b'1101001101001000101010001100100100','ncq',0.4,'23:44:22.012217',2030,'b',''), + ('2015-05-03','1987-03-19 17:37:53.053429',6,'','2012-08-03 00:00:00',b'0','cq',0.0,'17:16:43.030750',2035,'qh',NULL), + (NULL,'1980-01-01 00:00:00',1,'huyr','2012-02-11 14:15:13.004778',b'110011001100010100001101011001011110010000011001110101','',0.7,'09:33:00.034425',2024,'uyr',''), + ('2019-06-05','2020-08-05 23:53:07.028129',1,'yr',NULL,b'1001011110101010001111101000011001011111100','rflu',0.3,NULL,2016,'fluezqe',''), + ('1980-01-10','2025-05-12 08:22:39.039097',1,'lu','1975-07-24 00:00:00',b'10100111001111101001110000110011','',0.1,'23:58:28.031575',2005,NULL,''), + ('2008-03-17','1982-05-27 11:44:53.038339',NULL,'uezqe','2024-10-12 02:16:04.063095',b'1001010110101101000101011011000011','e',0.9,'19:37:29.063243',1987,'zqekmq','') +; +--enable_query_log +--disable_warnings +ALTER TABLE t1 ADD INDEX(col_enum,vcol_int); +ALTER TABLE t1 ADD INDEX(col_year); +--enable_warnings +DROP TABLE t1; diff --git a/storage/myisam/mi_check.c b/storage/myisam/mi_check.c index 8de200f5e7235..8711b9ff9aa4f 100644 --- a/storage/myisam/mi_check.c +++ b/storage/myisam/mi_check.c @@ -2781,7 +2781,7 @@ int mi_repair_parallel(HA_CHECK *param, register MI_INFO *info, del=info->state->del; param->glob_crc=0; /* for compressed tables */ - max_pack_reclength= share->base.pack_reclength; + max_pack_reclength= MY_MAX(share->base.pack_reclength, share->vreclength); if (share->options & HA_OPTION_COMPRESS_RECORD) set_if_bigger(max_pack_reclength, share->max_pack_length); if (!(sort_param=(MI_SORT_PARAM *) diff --git a/storage/myisam/mi_open.c b/storage/myisam/mi_open.c index 01a13a6225cc5..604dd1674ae34 100644 --- a/storage/myisam/mi_open.c +++ b/storage/myisam/mi_open.c @@ -740,6 +740,7 @@ uchar *mi_alloc_rec_buff(MI_INFO *info, ulong length, uchar **buf) else length= info->s->base.pack_reclength; length= MY_MAX(length, info->s->base.max_key_length); + length= MY_MAX(length, info->s->vreclength); /* Avoid unnecessary realloc */ if (newptr && length == old_length) return newptr; From 5f0c31f928338e8a6ffde098b7ffd3d1a8b02903 Mon Sep 17 00:00:00 2001 From: Monty Date: Tue, 10 Jan 2017 18:28:24 +0200 Subject: [PATCH 67/74] MDEV-11597 Assertion when doing select from virtual column with impossible value - Changed error handlers interface so that they can change error level in the handler - Give warnings and errors when calculating virtual columns - On insert/update error is fatal in strict mode. - SELECT and DELETE will only give a warning if a virtual field generates an error - Added VCOL_UPDATE_FOR_DELETE and VCOL_UPDATE_INDEX_FOR_REPLACE to be able to easily detect in update_virtual_fields() if we should use an error handler to mask errors or not. --- mysql-test/suite/vcol/inc/vcol_ins_upd.inc | 65 +++++ .../suite/vcol/r/vcol_ins_upd_innodb.result | 220 ++++++++++++++++ .../suite/vcol/r/vcol_ins_upd_myisam.result | 237 ++++++++++++++++++ sql/handler.cc | 4 +- sql/item_func.cc | 4 +- sql/log.cc | 4 +- sql/sp.cc | 13 +- sql/sql_acl.cc | 6 +- sql/sql_base.cc | 16 +- sql/sql_base.h | 2 +- sql/sql_class.cc | 22 +- sql/sql_class.h | 11 +- sql/sql_delete.cc | 5 +- sql/sql_insert.cc | 11 +- sql/sql_show.cc | 6 +- sql/sql_trigger.cc | 2 +- sql/sql_update.cc | 3 +- sql/table.cc | 49 +++- sql/table.h | 2 + storage/federated/ha_federated.cc | 2 +- storage/federatedx/ha_federatedx.cc | 2 +- 21 files changed, 628 insertions(+), 58 deletions(-) diff --git a/mysql-test/suite/vcol/inc/vcol_ins_upd.inc b/mysql-test/suite/vcol/inc/vcol_ins_upd.inc index d9a1e0628705c..a7a43aae5296b 100644 --- a/mysql-test/suite/vcol/inc/vcol_ins_upd.inc +++ b/mysql-test/suite/vcol/inc/vcol_ins_upd.inc @@ -314,3 +314,68 @@ select * from t1; drop table t1,t2; +--echo # +--echo # Test error handling with virtual columns +--echo # + +CREATE TABLE IF NOT EXISTS t1 ( + f1 DOUBLE, + f2 DOUBLE NOT NULL DEFAULT '0', + f3 DOUBLE, + f4 DOUBLE NOT NULL DEFAULT '0', + v1 DOUBLE AS ( ( f1 DIV ( f1 ) ) <= f2 ) VIRTUAL, + v2 DOUBLE AS ( ( f2 DIV ( f2 ) ) <= f2 ) VIRTUAL, + KEY (v2) +); + +set sql_mode='strict_all_tables,error_for_division_by_zero'; +--error ER_DIVISION_BY_ZERO +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 0, 0, 0); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 1, 1, 1); +--error ER_DIVISION_BY_ZERO +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +INSERT IGNORE INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 1, 1, 1); +select v1 from t1; + +--error ER_DIVISION_BY_ZERO +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0,0,0,0), (2,2,2,2); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (3,3,3,3), (4,4,4,4); +--error ER_DIVISION_BY_ZERO +INSERT INTO t1 (f1, f2, f3, f4) VALUES (5,5,5,5), (1,0,0,0); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (6,6,0,0); + +--error ER_DIVISION_BY_ZERO +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT f3, f4, f3, f4 FROM t1; +select count(*) from t1; +DELETE FROM t1 WHERE v2 != f1 and f1 < 5; +select count(*) from t1; +select * from t1; +--error ER_BAD_NULL_ERROR +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1; +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1 where f2 !=0; +UPDATE t1 SET f3 = v1 WHERE f2 = 2 AND v2 is null; +SELECT * FROM t1; +TRUNCATE TABLE t1; + +set sql_mode='error_for_division_by_zero'; +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 0, 0, 0); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 1, 1, 1); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 1, 1, 1); +select v1 from t1; + +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0,0,0,0), (2,2,2,2); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (3,3,3,3), (4,4,4,4); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (5,5,5,5), (1,0,0,0); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (6,6,0,0); + +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT f3, f4, f3, f4 FROM t1; +select count(*) from t1; +DELETE FROM t1 WHERE v2 != f1 and f1 < 5; +select count(*) from t1; +select * from t1; +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1; +UPDATE t1 SET f3 = v1 WHERE f2 = 2 AND v2 is null; +drop table t1; +set sql_mode=@@global.sql_mode; diff --git a/mysql-test/suite/vcol/r/vcol_ins_upd_innodb.result b/mysql-test/suite/vcol/r/vcol_ins_upd_innodb.result index d6793e226685c..e3853156b4dff 100644 --- a/mysql-test/suite/vcol/r/vcol_ins_upd_innodb.result +++ b/mysql-test/suite/vcol/r/vcol_ins_upd_innodb.result @@ -451,3 +451,223 @@ id name name_hash 2050 name1 9b46b0dd3a8083c070c3b9953bb5f3f95c5ab4da 2051 name2+1 fd4f236320db3956a5ec073c5ec39707d7f05708 drop table t1,t2; +# +# Test error handling with virtual columns +# +CREATE TABLE IF NOT EXISTS t1 ( +f1 DOUBLE, +f2 DOUBLE NOT NULL DEFAULT '0', +f3 DOUBLE, +f4 DOUBLE NOT NULL DEFAULT '0', +v1 DOUBLE AS ( ( f1 DIV ( f1 ) ) <= f2 ) VIRTUAL, +v2 DOUBLE AS ( ( f2 DIV ( f2 ) ) <= f2 ) VIRTUAL, +KEY (v2) +); +set sql_mode='strict_all_tables,error_for_division_by_zero'; +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 0, 0, 0); +ERROR 22012: Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 1, 1, 1); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +ERROR 22012: Division by 0 +INSERT IGNORE INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 1, 1, 1); +select v1 from t1; +v1 +1 +0 +NULL +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0,0,0,0), (2,2,2,2); +ERROR 22012: Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (3,3,3,3), (4,4,4,4); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (5,5,5,5), (1,0,0,0); +ERROR 22012: Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (6,6,0,0); +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT f3, f4, f3, f4 FROM t1; +ERROR 22012: Division by 0 +select count(*) from t1; +count(*) +6 +DELETE FROM t1 WHERE v2 != f1 and f1 < 5; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +select count(*) from t1; +count(*) +3 +select * from t1; +f1 f2 f3 f4 v1 v2 +1 1 1 1 1 1 +1 0 1 1 0 NULL +6 6 0 0 1 1 +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1; +ERROR 23000: Column 'f2' cannot be null +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1 where f2 !=0; +Warnings: +Warning 1365 Division by 0 +UPDATE t1 SET f3 = v1 WHERE f2 = 2 AND v2 is null; +Warnings: +Warning 1365 Division by 0 +SELECT * FROM t1; +f1 f2 f3 f4 v1 v2 +1 1 1 1 1 1 +1 0 1 1 0 NULL +6 6 0 0 1 1 +1 1 10 10 1 1 +1 1 10 10 1 1 +Warnings: +Warning 1365 Division by 0 +TRUNCATE TABLE t1; +set sql_mode='error_for_division_by_zero'; +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 0, 0, 0); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 1, 1, 1); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 1, 1, 1); +select v1 from t1; +v1 +NULL +1 +0 +NULL +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0,0,0,0), (2,2,2,2); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (3,3,3,3), (4,4,4,4); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (5,5,5,5), (1,0,0,0); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (6,6,0,0); +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT f3, f4, f3, f4 FROM t1; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +select count(*) from t1; +count(*) +22 +DELETE FROM t1 WHERE v2 != f1 and f1 < 5; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +select count(*) from t1; +count(*) +15 +select * from t1; +f1 f2 f3 f4 v1 v2 +0 0 0 0 NULL NULL +1 1 1 1 1 1 +1 0 1 1 0 NULL +0 0 0 0 NULL NULL +5 5 5 5 1 1 +1 0 0 0 0 NULL +6 6 0 0 1 1 +0 0 0 0 NULL NULL +1 1 1 1 1 1 +1 1 1 1 1 1 +1 1 1 1 1 1 +0 0 0 0 NULL NULL +5 5 5 5 1 1 +0 0 0 0 NULL NULL +0 0 0 0 NULL NULL +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +UPDATE t1 SET f3 = v1 WHERE f2 = 2 AND v2 is null; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +drop table t1; +set sql_mode=@@global.sql_mode; diff --git a/mysql-test/suite/vcol/r/vcol_ins_upd_myisam.result b/mysql-test/suite/vcol/r/vcol_ins_upd_myisam.result index 97f6b9be563ce..dcf957becaed2 100644 --- a/mysql-test/suite/vcol/r/vcol_ins_upd_myisam.result +++ b/mysql-test/suite/vcol/r/vcol_ins_upd_myisam.result @@ -389,3 +389,240 @@ id name name_hash 2051 name2+1 fd4f236320db3956a5ec073c5ec39707d7f05708 2041 name3+1 93c9096df48221428de46e146abc9f4f94bf7d2e drop table t1,t2; +# +# Test error handling with virtual columns +# +CREATE TABLE IF NOT EXISTS t1 ( +f1 DOUBLE, +f2 DOUBLE NOT NULL DEFAULT '0', +f3 DOUBLE, +f4 DOUBLE NOT NULL DEFAULT '0', +v1 DOUBLE AS ( ( f1 DIV ( f1 ) ) <= f2 ) VIRTUAL, +v2 DOUBLE AS ( ( f2 DIV ( f2 ) ) <= f2 ) VIRTUAL, +KEY (v2) +); +set sql_mode='strict_all_tables,error_for_division_by_zero'; +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 0, 0, 0); +ERROR 22012: Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 1, 1, 1); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +ERROR 22012: Division by 0 +INSERT IGNORE INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 1, 1, 1); +select v1 from t1; +v1 +1 +0 +NULL +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0,0,0,0), (2,2,2,2); +ERROR 22012: Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (3,3,3,3), (4,4,4,4); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (5,5,5,5), (1,0,0,0); +ERROR 22012: Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (6,6,0,0); +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT f3, f4, f3, f4 FROM t1; +ERROR 22012: Division by 0 +select count(*) from t1; +count(*) +13 +DELETE FROM t1 WHERE v2 != f1 and f1 < 5; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +select count(*) from t1; +count(*) +8 +select * from t1; +f1 f2 f3 f4 v1 v2 +1 1 1 1 1 1 +1 0 1 1 0 NULL +5 5 5 5 1 1 +6 6 0 0 1 1 +1 1 1 1 1 1 +1 1 1 1 1 1 +1 1 1 1 1 1 +5 5 5 5 1 1 +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1; +ERROR 23000: Column 'f2' cannot be null +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1 where f2 !=0; +Warnings: +Warning 1365 Division by 0 +UPDATE t1 SET f3 = v1 WHERE f2 = 2 AND v2 is null; +Warnings: +Warning 1365 Division by 0 +SELECT * FROM t1; +f1 f2 f3 f4 v1 v2 +1 1 1 1 1 1 +1 0 1 1 0 NULL +1 1 10 10 1 1 +1 1 10 10 1 1 +1 1 10 10 1 1 +5 5 5 5 1 1 +6 6 0 0 1 1 +1 1 1 1 1 1 +1 1 1 1 1 1 +1 1 1 1 1 1 +1 1 10 10 1 1 +1 1 10 10 1 1 +5 5 5 5 1 1 +1 1 10 10 1 1 +1 1 10 10 1 1 +1 1 10 10 1 1 +1 1 10 10 1 1 +Warnings: +Warning 1365 Division by 0 +TRUNCATE TABLE t1; +set sql_mode='error_for_division_by_zero'; +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 0, 0, 0); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 1, 1, 1); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (1, 0, 1, 1); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0, 1, 1, 1); +select v1 from t1; +v1 +NULL +1 +0 +NULL +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (0,0,0,0), (2,2,2,2); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (3,3,3,3), (4,4,4,4); +INSERT INTO t1 (f1, f2, f3, f4) VALUES (5,5,5,5), (1,0,0,0); +Warnings: +Warning 1365 Division by 0 +INSERT INTO t1 (f1, f2, f3, f4) VALUES (6,6,0,0); +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT f3, f4, f3, f4 FROM t1; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +select count(*) from t1; +count(*) +22 +DELETE FROM t1 WHERE v2 != f1 and f1 < 5; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +select count(*) from t1; +count(*) +15 +select * from t1; +f1 f2 f3 f4 v1 v2 +0 0 0 0 NULL NULL +1 1 1 1 1 1 +1 0 1 1 0 NULL +0 0 0 0 NULL NULL +5 5 5 5 1 1 +1 0 0 0 0 NULL +6 6 0 0 1 1 +0 0 0 0 NULL NULL +1 1 1 1 1 1 +1 1 1 1 1 1 +1 1 1 1 1 1 +0 0 0 0 NULL NULL +5 5 5 5 1 1 +0 0 0 0 NULL NULL +0 0 0 0 NULL NULL +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +INSERT INTO t1 ( f1, f2, f3, f4 ) SELECT v1, v2, 10,10 FROM t1; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +Warning 1048 Column 'f2' cannot be null +Warning 1365 Division by 0 +UPDATE t1 SET f3 = v1 WHERE f2 = 2 AND v2 is null; +Warnings: +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +Warning 1365 Division by 0 +drop table t1; +set sql_mode=@@global.sql_mode; diff --git a/sql/handler.cc b/sql/handler.cc index a91a97598d665..f06c5d71a5ee2 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -4974,7 +4974,7 @@ class Table_exists_error_handler : public Internal_error_handler bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { @@ -4987,7 +4987,7 @@ class Table_exists_error_handler : public Internal_error_handler return TRUE; } - if (level == Sql_condition::WARN_LEVEL_ERROR) + if (*level == Sql_condition::WARN_LEVEL_ERROR) m_unhandled_errors++; return FALSE; } diff --git a/sql/item_func.cc b/sql/item_func.cc index 754ba9d261c1d..f6162b54806f0 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -4127,7 +4127,7 @@ class Lock_wait_timeout_handler: public Internal_error_handler bool handle_condition(THD * /* thd */, uint sql_errno, const char * /* sqlstate */, - Sql_condition::enum_warning_level /* level */, + Sql_condition::enum_warning_level* /* level */, const char *message, Sql_condition ** /* cond_hdl */); }; @@ -4136,7 +4136,7 @@ bool Lock_wait_timeout_handler:: handle_condition(THD *thd, uint sql_errno, const char * /* sqlstate */, - Sql_condition::enum_warning_level /* level */, + Sql_condition::enum_warning_level* /* level */, const char *message, Sql_condition ** /* cond_hdl */) { diff --git a/sql/log.cc b/sql/log.cc index 57984bd03f4cc..233265323882c 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -195,7 +195,7 @@ class Silence_log_table_errors : public Internal_error_handler virtual bool handle_condition(THD *thd, uint sql_errno, const char* sql_state, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl); const char *message() const { return m_message; } @@ -205,7 +205,7 @@ bool Silence_log_table_errors::handle_condition(THD *, uint, const char*, - Sql_condition::enum_warning_level, + Sql_condition::enum_warning_level*, const char* msg, Sql_condition ** cond_hdl) { diff --git a/sql/sp.cc b/sql/sp.cc index 05f075390c15e..a1662cefb9eb3 100644 --- a/sql/sp.cc +++ b/sql/sp.cc @@ -691,7 +691,7 @@ struct Silence_deprecated_warning : public Internal_error_handler virtual bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl); }; @@ -701,13 +701,13 @@ Silence_deprecated_warning::handle_condition( THD *, uint sql_errno, const char*, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char*, Sql_condition ** cond_hdl) { *cond_hdl= NULL; if (sql_errno == ER_WARN_DEPRECATED_SYNTAX && - level == Sql_condition::WARN_LEVEL_WARN) + *level == Sql_condition::WARN_LEVEL_WARN) return TRUE; return FALSE; @@ -780,7 +780,7 @@ class Bad_db_error_handler : public Internal_error_handler virtual bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* message, Sql_condition ** cond_hdl); @@ -794,7 +794,8 @@ bool Bad_db_error_handler::handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level + *level, const char* message, Sql_condition ** cond_hdl) { @@ -1477,7 +1478,7 @@ class Lock_db_routines_error_handler : public Internal_error_handler bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { diff --git a/sql/sql_acl.cc b/sql/sql_acl.cc index d2840a81e7ef4..4a3c9ae75e7bf 100644 --- a/sql/sql_acl.cc +++ b/sql/sql_acl.cc @@ -10155,7 +10155,7 @@ class Silence_routine_definer_errors : public Internal_error_handler virtual bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl); @@ -10170,12 +10170,12 @@ Silence_routine_definer_errors::handle_condition( THD *thd, uint sql_errno, const char*, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { *cond_hdl= NULL; - if (level == Sql_condition::WARN_LEVEL_ERROR) + if (*level == Sql_condition::WARN_LEVEL_ERROR) { switch (sql_errno) { diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 46d7d07dbdf49..ac4b8a451075e 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -68,7 +68,7 @@ bool No_such_table_error_handler::handle_condition(THD *, uint sql_errno, const char*, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char*, Sql_condition ** cond_hdl) { @@ -79,7 +79,7 @@ No_such_table_error_handler::handle_condition(THD *, return TRUE; } - if (level == Sql_condition::WARN_LEVEL_ERROR) + if (*level == Sql_condition::WARN_LEVEL_ERROR) m_unhandled_errors++; return FALSE; } @@ -112,7 +112,7 @@ class Repair_mrg_table_error_handler : public Internal_error_handler bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl); @@ -142,7 +142,7 @@ bool Repair_mrg_table_error_handler::handle_condition(THD *, uint sql_errno, const char*, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char*, Sql_condition ** cond_hdl) { @@ -1254,7 +1254,7 @@ class MDL_deadlock_handler : public Internal_error_handler virtual bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl); @@ -1273,7 +1273,7 @@ class MDL_deadlock_handler : public Internal_error_handler bool MDL_deadlock_handler::handle_condition(THD *, uint sql_errno, const char*, - Sql_condition::enum_warning_level, + Sql_condition::enum_warning_level*, const char*, Sql_condition ** cond_hdl) { @@ -2834,7 +2834,7 @@ class MDL_deadlock_discovery_repair_handler : public Internal_error_handler virtual bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { @@ -7900,7 +7900,6 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, table_arg->update_default_fields(0, ignore_errors)) goto err; /* Update virtual fields */ - thd->abort_on_warning= FALSE; if (table_arg->vfield && table_arg->update_virtual_fields(VCOL_UPDATE_FOR_WRITE)) goto err; @@ -7908,6 +7907,7 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, thd->no_errors= save_no_errors; DBUG_RETURN(thd->is_error()); err: + DBUG_PRINT("error",("got error")); thd->abort_on_warning= save_abort_on_warning; thd->no_errors= save_no_errors; if (fields.elements) diff --git a/sql/sql_base.h b/sql/sql_base.h index 8f363a738636d..374ac56c3d834 100644 --- a/sql/sql_base.h +++ b/sql/sql_base.h @@ -624,7 +624,7 @@ class No_such_table_error_handler : public Internal_error_handler bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl); diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 1837f2878c7a7..5c7cfdce6573e 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -640,7 +640,7 @@ char *thd_security_context(THD *thd, bool Drop_table_error_handler::handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { @@ -660,7 +660,7 @@ MDL_deadlock_and_lock_abort_error_handler:: handle_condition(THD *thd, uint sql_errno, const char *sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition **cond_hdl) { @@ -945,7 +945,7 @@ void THD::push_internal_handler(Internal_error_handler *handler) bool THD::handle_condition(uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { @@ -1072,6 +1072,7 @@ Sql_condition* THD::raise_condition(uint sql_errno, Diagnostics_area *da= get_stmt_da(); Sql_condition *cond= NULL; DBUG_ENTER("THD::raise_condition"); + DBUG_ASSERT(level < Sql_condition::WARN_LEVEL_END); if (!(variables.option_bits & OPTION_SQL_NOTES) && (level == Sql_condition::WARN_LEVEL_NOTE)) @@ -1099,24 +1100,23 @@ Sql_condition* THD::raise_condition(uint sql_errno, push_warning and strict SQL_MODE case. */ level= Sql_condition::WARN_LEVEL_ERROR; - killed= KILL_BAD_DATA; } - switch (level) - { + if (handle_condition(sql_errno, sqlstate, &level, msg, &cond)) + DBUG_RETURN(cond); + + switch (level) { case Sql_condition::WARN_LEVEL_NOTE: case Sql_condition::WARN_LEVEL_WARN: got_warning= 1; break; case Sql_condition::WARN_LEVEL_ERROR: break; - default: - DBUG_ASSERT(FALSE); + case Sql_condition::WARN_LEVEL_END: + /* Impossible */ + break; } - if (handle_condition(sql_errno, sqlstate, level, msg, &cond)) - DBUG_RETURN(cond); - if (level == Sql_condition::WARN_LEVEL_ERROR) { mysql_audit_general(this, MYSQL_AUDIT_GENERAL_ERROR, sql_errno, msg); diff --git a/sql/sql_class.h b/sql/sql_class.h index a55e17393186d..3a7454e0a89e7 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1558,6 +1558,7 @@ show_system_thread(enum_thread_type thread) Internal error handlers are exception handlers used by the server implementation. */ + class Internal_error_handler { protected: @@ -1595,7 +1596,7 @@ class Internal_error_handler virtual bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) = 0; @@ -1616,7 +1617,7 @@ class Dummy_error_handler : public Internal_error_handler bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { @@ -1643,7 +1644,7 @@ class Drop_table_error_handler : public Internal_error_handler bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl); @@ -1664,7 +1665,7 @@ class MDL_deadlock_and_lock_abort_error_handler: public Internal_error_handler bool handle_condition(THD *thd, uint sql_errno, const char *sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition **cond_hdl); @@ -3818,7 +3819,7 @@ class THD :public Statement, */ bool handle_condition(uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl); diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index e9a3be3006034..2f4c2d9a6f8ff 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -560,10 +560,9 @@ bool mysql_delete(THD *thd, TABLE_LIST *table_list, COND *conds, ! thd->is_error()) { explain->tracker.on_record_read(); - if (table->vfield) - table->update_virtual_fields(VCOL_UPDATE_FOR_WRITE); thd->inc_examined_row_count(1); - // thd->is_error() is tested to disallow delete row on error + if (table->vfield) + (void) table->update_virtual_fields(VCOL_UPDATE_FOR_DELETE); if (!select || select->skip_record(thd) > 0) { explain->tracker.on_record_after_where(); diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index cea42667c48e9..b9c1dcebee927 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -3174,6 +3174,7 @@ bool Delayed_insert::handle_inserts(void) while ((row=rows.get())) { + int tmp_error; stacked_inserts--; mysql_mutex_unlock(&mutex); memcpy(table->record[0],row->record,table->s->reclength); @@ -3250,16 +3251,18 @@ bool Delayed_insert::handle_inserts(void) table->file->extra(HA_EXTRA_INSERT_WITH_UPDATE); thd.clear_error(); // reset error for binlog + tmp_error= 0; if (table->vfield) { /* - Virtual fields where not calculated by caller as the temporary TABLE object used - had vcol_set empty. Better to calculate them here to make the caller faster. + Virtual fields where not calculated by caller as the temporary + TABLE object used had vcol_set empty. Better to calculate them + here to make the caller faster. */ - table->update_virtual_fields(VCOL_UPDATE_FOR_WRITE); + tmp_error= table->update_virtual_fields(VCOL_UPDATE_FOR_WRITE); } - if (write_record(&thd, table, &info)) + if (tmp_error || write_record(&thd, table, &info)) { info.error_count++; // Ignore errors thread_safe_increment(delayed_insert_errors,&LOCK_delayed_status); diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 0a2dab91c1473..58fc2edc6f209 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -1069,7 +1069,7 @@ class Show_create_error_handler : public Internal_error_handler { } bool handle_condition(THD *thd, uint sql_errno, const char * /* sqlstate */, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char *message, Sql_condition ** /* cond_hdl */) { /* @@ -4675,7 +4675,7 @@ class Warnings_only_error_handler : public Internal_error_handler bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { @@ -4684,7 +4684,7 @@ class Warnings_only_error_handler : public Internal_error_handler sql_errno == ER_TRG_NO_CREATION_CTX) return true; - if (level != Sql_condition::WARN_LEVEL_ERROR) + if (*level != Sql_condition::WARN_LEVEL_ERROR) return false; if (!thd->get_stmt_da()->is_error()) diff --git a/sql/sql_trigger.cc b/sql/sql_trigger.cc index 2d1619018b2dc..bd02289b135ac 100644 --- a/sql/sql_trigger.cc +++ b/sql/sql_trigger.cc @@ -308,7 +308,7 @@ class Deprecated_trigger_syntax_handler : public Internal_error_handler virtual bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* message, Sql_condition ** cond_hdl) { diff --git a/sql/sql_update.cc b/sql/sql_update.cc index 476d1a4e104b0..c46b80f798b23 100644 --- a/sql/sql_update.cc +++ b/sql/sql_update.cc @@ -2403,7 +2403,8 @@ int multi_update::do_updates() field_num++; } while ((tbl= check_opt_it++)); - if (table->vfield && table->update_virtual_fields(VCOL_UPDATE_INDEXED)) + if (table->vfield && + table->update_virtual_fields(VCOL_UPDATE_INDEXED_FOR_UPDATE)) goto err2; table->status|= STATUS_UPDATED; diff --git a/sql/table.cc b/sql/table.cc index 41f764d1e59b5..4242b870d559b 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -4932,7 +4932,7 @@ bool TABLE_LIST::prep_check_option(THD *thd, uint8 check_opt_type) void TABLE_LIST::hide_view_error(THD *thd) { - if (thd->killed || thd->get_internal_handler()) + if ((thd->killed && !thd->is_error())|| thd->get_internal_handler()) return; /* Hide "Unknown column" or "Unknown function" error */ DBUG_ASSERT(thd->is_error()); @@ -7308,6 +7308,24 @@ bool is_simple_order(ORDER *order) return TRUE; } +class Turn_errors_to_warnings_handler : public Internal_error_handler +{ +public: + Turn_errors_to_warnings_handler() {} + bool handle_condition(THD *thd, + uint sql_errno, + const char* sqlstate, + Sql_condition::enum_warning_level *level, + const char* msg, + Sql_condition ** cond_hdl) + { + *cond_hdl= NULL; + if (*level == Sql_condition::WARN_LEVEL_ERROR) + *level= Sql_condition::WARN_LEVEL_WARN; + return(0); + } +}; + /* @brief Compute values for virtual columns used in query @@ -7328,9 +7346,22 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) DBUG_ENTER("TABLE::update_virtual_fields"); DBUG_PRINT("enter", ("update_mode: %d", update_mode)); Field **vfield_ptr, *vf; + Turn_errors_to_warnings_handler Suppress_errors; + int error; + bool handler_pushed= 0; DBUG_ASSERT(vfield); + error= 0; in_use->reset_arena_for_cached_items(expr_arena); + + /* When reading or deleting row, ignore errors from virtual columns */ + if (update_mode == VCOL_UPDATE_FOR_READ || + update_mode == VCOL_UPDATE_FOR_DELETE || + update_mode == VCOL_UPDATE_INDEXED) + { + in_use->push_internal_handler(&Suppress_errors); + handler_pushed= 1; + } /* Iterate over virtual fields in the table */ for (vfield_ptr= vfield; *vfield_ptr; vfield_ptr++) { @@ -7347,6 +7378,8 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) && bitmap_is_set(vcol_set, vf->field_index); swap_values= 1; break; + case VCOL_UPDATE_FOR_DELETE: + /* Fall trough */ case VCOL_UPDATE_FOR_WRITE: update= bitmap_is_set(vcol_set, vf->field_index); break; @@ -7367,6 +7400,7 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) } break; case VCOL_UPDATE_INDEXED: + case VCOL_UPDATE_INDEXED_FOR_UPDATE: /* Read indexed fields that was not updated in VCOL_UPDATE_FOR_READ */ update= (!vcol_info->stored_in_db && (vf->flags & PART_KEY_FLAG) && bitmap_is_set(vcol_set, vf->field_index) && @@ -7377,9 +7411,12 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) if (update) { + int field_error __attribute__((unused)) = 0; /* Compute the actual value of the virtual fields */ - vcol_info->expr->save_in_field(vf, 0); - DBUG_PRINT("info", ("field '%s' - updated", vf->field_name)); + if (vcol_info->expr->save_in_field(vf, 0)) + field_error= error= 1; + DBUG_PRINT("info", ("field '%s' - updated error: %d", + vf->field_name, field_error)); if (swap_values && (vf->flags & BLOB_FLAG)) { /* @@ -7396,8 +7433,12 @@ int TABLE::update_virtual_fields(enum_vcol_update_mode update_mode) DBUG_PRINT("info", ("field '%s' - skipped", vf->field_name)); } } + if (handler_pushed) + in_use->pop_internal_handler(); in_use->reset_arena_for_cached_items(0); - DBUG_RETURN(0); + + /* Return 1 only of we got a fatal error, not a warning */ + DBUG_RETURN(in_use->is_error()); } int TABLE::update_virtual_field(Field *vf) diff --git a/sql/table.h b/sql/table.h index facba06a3cc43..3b8086dd09f2c 100644 --- a/sql/table.h +++ b/sql/table.h @@ -328,7 +328,9 @@ enum enum_vcol_update_mode { VCOL_UPDATE_FOR_READ= 0, VCOL_UPDATE_FOR_WRITE, + VCOL_UPDATE_FOR_DELETE, VCOL_UPDATE_INDEXED, + VCOL_UPDATE_INDEXED_FOR_UPDATE, VCOL_UPDATE_FOR_REPLACE }; diff --git a/storage/federated/ha_federated.cc b/storage/federated/ha_federated.cc index 038415cc1884a..879a17782aa15 100644 --- a/storage/federated/ha_federated.cc +++ b/storage/federated/ha_federated.cc @@ -1653,7 +1653,7 @@ class Net_error_handler : public Internal_error_handler public: bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { return sql_errno >= ER_ABORTING_CONNECTION && diff --git a/storage/federatedx/ha_federatedx.cc b/storage/federatedx/ha_federatedx.cc index a4da84ddaf3b5..e3506e1a4df00 100644 --- a/storage/federatedx/ha_federatedx.cc +++ b/storage/federatedx/ha_federatedx.cc @@ -1786,7 +1786,7 @@ class Net_error_handler : public Internal_error_handler public: bool handle_condition(THD *thd, uint sql_errno, const char* sqlstate, - Sql_condition::enum_warning_level level, + Sql_condition::enum_warning_level *level, const char* msg, Sql_condition ** cond_hdl) { return sql_errno >= ER_ABORTING_CONNECTION && From 833fda8f1a0c415ced1f2b8c1851809383bcc639 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 11 Jan 2017 14:13:30 +0200 Subject: [PATCH 68/74] InnoDB: Enable UNIV_DEBUG_VALGRIND for cmake -DWITH_VALGRIND The symbol HAVE_VALGRIND_MEMCHECK_H was never defined. Instead, the symbol HAVE_VALGRIND should have been used. --- storage/innobase/include/univ.i | 3 ++- storage/xtradb/include/univ.i | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index e6ddf45fabaa9..7b867edfb22d9 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. Portions of this file contain modifications contributed and copyrighted by @@ -171,7 +172,7 @@ command. Not tested on Windows. */ #define UNIV_COMPILE_TEST_FUNCS */ -#if defined(HAVE_valgrind)&& defined(HAVE_VALGRIND_MEMCHECK_H) +#if defined HAVE_valgrind && defined HAVE_VALGRIND # define UNIV_DEBUG_VALGRIND #endif /* HAVE_VALGRIND */ #if 0 diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index 397843d309c33..3444cbb1e7ffe 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. Portions of this file contain modifications contributed and copyrighted by @@ -183,7 +184,7 @@ command. Not tested on Windows. */ #define UNIV_COMPILE_TEST_FUNCS */ -#if defined(HAVE_valgrind)&& defined(HAVE_VALGRIND_MEMCHECK_H) +#if defined HAVE_valgrind && defined HAVE_VALGRIND # define UNIV_DEBUG_VALGRIND #endif #if 0 From 4507f1e5d4fbfc2a638417683fc71d1768729cac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 11 Jan 2017 14:26:30 +0200 Subject: [PATCH 69/74] Remove an excessive copyright message. --- storage/innobase/include/univ.i | 1 - storage/xtradb/include/univ.i | 1 - 2 files changed, 2 deletions(-) diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index 1a5004549b985..53be1cf1ffeb4 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -3,7 +3,6 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described diff --git a/storage/xtradb/include/univ.i b/storage/xtradb/include/univ.i index e3c79e9b8261e..ad0565a0290e8 100644 --- a/storage/xtradb/include/univ.i +++ b/storage/xtradb/include/univ.i @@ -3,7 +3,6 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2013, 2017, MariaDB Corporation. Copyright (c) 2008, Google Inc. -Copyright (c) 2013, 2015, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described From 2dc5d8bb7e087a340edb989c045070b808e0da57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 12 Jan 2017 12:33:46 +0200 Subject: [PATCH 70/74] Improve an MDEV-9011 test of innodb_encrypt_log. Test crash recovery from an encrypted redo log with innodb_encrypt_log=0. Previously, we did a clean shutdown, so only the log checkpoint information would have been read from the redo log. With this change, we will be reading and applying encrypted redo log records. include/start_mysqld.inc: Observe $restart_parameters. encryption.innodb-log-encrypt: Remove some unnecessary statements, and instead of restarting the server and concurrently accessing the files while the server is running, kill the server, check the files, and finally start up the server. innodb.log_data_file_size: Use start_mysqld.inc with $restart_parameters. --- mysql-test/include/start_mysqld.inc | 9 ++++++- .../encryption/r/innodb-log-encrypt.result | 4 +-- .../encryption/t/innodb-log-encrypt.test | 25 +++---------------- .../suite/innodb/t/log_data_file_size.test | 11 +++----- 4 files changed, 18 insertions(+), 31 deletions(-) diff --git a/mysql-test/include/start_mysqld.inc b/mysql-test/include/start_mysqld.inc index 983c566821e85..e31f26aad8c85 100644 --- a/mysql-test/include/start_mysqld.inc +++ b/mysql-test/include/start_mysqld.inc @@ -1,7 +1,14 @@ # Include this script only after using shutdown_mysqld.inc # where $_expect_file_name was initialized. # Write file to make mysql-test-run.pl start up the server again ---exec echo "restart" > $_expect_file_name +if ($restart_parameters) +{ + --exec echo "restart: $restart_parameters" > $_expect_file_name +} +if (!$restart_parameters) +{ + --exec echo "restart" > $_expect_file_name +} # Turn on reconnect --enable_reconnect diff --git a/mysql-test/suite/encryption/r/innodb-log-encrypt.result b/mysql-test/suite/encryption/r/innodb-log-encrypt.result index 655e3023f7a88..98fc277e513fe 100644 --- a/mysql-test/suite/encryption/r/innodb-log-encrypt.result +++ b/mysql-test/suite/encryption/r/innodb-log-encrypt.result @@ -13,14 +13,14 @@ set current_num = current_num + 1; end while; end// commit; -set autocommit=0; +begin; call innodb_insert_proc(2000); commit; -set autocommit=1; update t1 set c1 = c1 +1; select count(*) from t1; count(*) 2000 +# Kill the server # ibdata1 yes on expecting NOT FOUND NOT FOUND /privatejanprivate/ in ibdata1 # t1 yes on expecting NOT FOUND diff --git a/mysql-test/suite/encryption/t/innodb-log-encrypt.test b/mysql-test/suite/encryption/t/innodb-log-encrypt.test index 7c2e6f847b1de..0555f9d49aa6b 100644 --- a/mysql-test/suite/encryption/t/innodb-log-encrypt.test +++ b/mysql-test/suite/encryption/t/innodb-log-encrypt.test @@ -6,16 +6,6 @@ # MDEV-9011: Redo log encryption does not work # ---disable_query_log -let $innodb_file_format_orig = `SELECT @@innodb_file_format`; -let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; ---enable_query_log - ---disable_query_log -let $innodb_file_format_orig = `SELECT @@innodb_file_format`; -let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; ---enable_query_log - SET GLOBAL innodb_file_format = `Barracuda`; SET GLOBAL innodb_file_per_table = ON; @@ -35,16 +25,13 @@ end// delimiter ;// commit; -set autocommit=0; +begin; call innodb_insert_proc(2000); commit; -set autocommit=1; update t1 set c1 = c1 +1; select count(*) from t1; --- source include/restart_mysqld.inc - --let $MYSQLD_DATADIR=`select @@datadir` --let ib1_IBD = $MYSQLD_DATADIR/ibdata1 --let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd @@ -53,6 +40,8 @@ select count(*) from t1; --let SEARCH_RANGE = 10000000 --let SEARCH_PATTERN=privatejanprivate +-- source include/kill_mysqld.inc + --echo # ibdata1 yes on expecting NOT FOUND -- let SEARCH_FILE=$ib1_IBD -- source include/search_pattern_in_file.inc @@ -68,7 +57,7 @@ select count(*) from t1; --echo # Restart mysqld --innodb_encrypt_log=0 -- let $restart_parameters=--innodb_encrypt_log=0 --- source include/restart_mysqld.inc +-- source include/start_mysqld.inc insert into t1 values(5000, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); insert into t1 values(5001, substring(MD5(RAND()), -64), REPEAT('publicmessage',10)); @@ -106,9 +95,3 @@ insert into t1 values(5004, substring(MD5(RAND()), -64), REPEAT('publicmessage', drop procedure innodb_insert_proc; drop table t1; - -# reset system ---disable_query_log -EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; -EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; ---enable_query_log diff --git a/mysql-test/suite/innodb/t/log_data_file_size.test b/mysql-test/suite/innodb/t/log_data_file_size.test index 23f1ede483f21..9328f3bbe6d20 100644 --- a/mysql-test/suite/innodb/t/log_data_file_size.test +++ b/mysql-test/suite/innodb/t/log_data_file_size.test @@ -23,7 +23,7 @@ perl; use Fcntl 'SEEK_CUR', 'SEEK_END'; my $page_size = $ENV{'INNODB_PAGE_SIZE'}; -my $restart = 'restart'; +my $restart; if ($ENV{'MYSQLD_IS_DEBUG'}) { # It is impractical to ensure that CREATE TABLE t will extend ibdata1. @@ -37,17 +37,14 @@ if ($ENV{'MYSQLD_IS_DEBUG'}) while() { unless (/\0*/gso) { $empty_tail= 0; last } } if ($empty_tail) { - $restart = 'restart: --innodb-data-file-size-debug=' . $size; + $restart = "--innodb-data-file-size-debug=$size"; truncate(FILE, $page_size * $root); } close FILE; } open(FILE, ">$ENV{MYSQLTEST_VARDIR}/log/start_mysqld.txt") || die; -print FILE '--exec echo "', $restart, '" > $_expect_file_name ---enable_reconnect ---source include/wait_until_connected_again.inc ---disable_reconnect -'; +print FILE "--let \$restart_parameters=$restart\n" if $restart; +print FILE "--source include/start_mysqld.inc\n"; close FILE; open(FILE, "+<", "$ENV{'MYSQLD_DATADIR'}test/ibd4.ibd") or die; truncate(FILE, $page_size * 4); From 1ba7234b211b4deec1e71f9b2d7db16543ece0e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 12 Jan 2017 13:30:10 +0200 Subject: [PATCH 71/74] Follow-up to MDEV-11713: Make more use of DBUG_LOG --- storage/innobase/buf/buf0flu.cc | 18 ++++------------ storage/innobase/fil/fil0fil.cc | 19 +++++++++-------- storage/innobase/fil/fil0pagecompress.cc | 6 ++---- storage/innobase/handler/ha_innodb.cc | 10 ++++----- storage/innobase/lock/lock0lock.cc | 21 +++++++++---------- storage/innobase/row/row0purge.cc | 24 +++++++++++----------- storage/innobase/trx/trx0purge.cc | 7 +++---- storage/innobase/trx/trx0trx.cc | 26 ++++++++++-------------- 8 files changed, 58 insertions(+), 73 deletions(-) diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index 0f901363596fe..2738fbd0ec719 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -3761,9 +3761,7 @@ FlushObserver::FlushObserver( m_removed->at(i) = 0; } -#ifdef FLUSH_LIST_OBSERVER_DEBUG - ib::info() << "FlushObserver constructor: " << m_trx->id; -#endif /* FLUSH_LIST_OBSERVER_DEBUG */ + DBUG_LOG("flush", "FlushObserver(): trx->id=" << m_trx->id); } /** FlushObserver deconstructor */ @@ -3774,9 +3772,7 @@ FlushObserver::~FlushObserver() UT_DELETE(m_flushed); UT_DELETE(m_removed); -#ifdef FLUSH_LIST_OBSERVER_DEBUG - ib::info() << "FlushObserver deconstructor: " << m_trx->id; -#endif /* FLUSH_LIST_OBSERVER_DEBUG */ + DBUG_LOG("flush", "~FlushObserver(): trx->id=" << m_trx->id); } /** Check whether trx is interrupted @@ -3809,10 +3805,7 @@ FlushObserver::notify_flush( m_stage->inc(); } -#ifdef FLUSH_LIST_OBSERVER_DEBUG - ib::info() << "Flush <" << bpage->id.space() - << ", " << bpage->id.page_no() << ">"; -#endif /* FLUSH_LIST_OBSERVER_DEBUG */ + DBUG_LOG("flush", "Flush " << bpage->id); } /** Notify observer of a remove @@ -3827,10 +3820,7 @@ FlushObserver::notify_remove( m_removed->at(buf_pool->instance_no)++; -#ifdef FLUSH_LIST_OBSERVER_DEBUG - ib::info() << "Remove <" << bpage->id.space() - << ", " << bpage->id.page_no() << ">"; -#endif /* FLUSH_LIST_OBSERVER_DEBUG */ + DBUG_LOG("flush", "Remove " << bpage->id); } /** Flush dirty pages and wait. */ diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index f104112eb7ac7..773eb0406aefe 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -7376,14 +7376,17 @@ fil_space_get_crypt_data( space->crypt_data = fil_space_read_crypt_data(space_id, page, offset); ut_free(buf); -#ifdef UNIV_DEBUG - ib::info() << "Read page 0 from tablespace for" - << "space " << space_id - << " name " << space->name - << " key_id " << (space->crypt_data ? space->crypt_data->key_id : 0) - << " encryption " << (space->crypt_data ? space->crypt_data->encryption : 0) - << " handle " << node->handle; -#endif + DBUG_LOG("crypt", + "Read page 0 from" + << " tablespace " << space_id + << " name " << space->name + << " key_id " << (space->crypt_data + ? space->crypt_data->key_id + : 0) + << " encryption " + << (space->crypt_data + ? space->crypt_data->encryption : 0) + << " handle " << node->handle); ut_a(space->id == space_id); diff --git a/storage/innobase/fil/fil0pagecompress.cc b/storage/innobase/fil/fil0pagecompress.cc index 98d732a9f9db3..cd2c62e00c123 100644 --- a/storage/innobase/fil/fil0pagecompress.cc +++ b/storage/innobase/fil/fil0pagecompress.cc @@ -487,10 +487,8 @@ fil_decompress_page( *write_size = actual_size; } -#ifdef UNIV_PAGECOMPRESS_DEBUG - ib::info() << "Preparing for decompress for len " - << actual_size << "."; -#endif /* UNIV_PAGECOMPRESS_DEBUG */ + DBUG_LOG("compress", "Preparing for decompress for len " + << actual_size << "."); switch(compression_alg) { case PAGE_ZLIB_ALGORITHM: diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index d9c32f286e6e4..9e9a6a5994ddd 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -5255,13 +5255,13 @@ innobase_rollback( error = trx_rollback_for_mysql(trx); if (trx->state == TRX_STATE_FORCED_ROLLBACK) { -#ifdef UNIV_DEBUG +#ifndef DBUG_OFF char buffer[1024]; - ib::info() << "Forced rollback : " - << thd_get_error_context_description(thd, - buffer, sizeof(buffer), 512); -#endif /* UNIV_DEBUG */ + DBUG_LOG("trx", "Forced rollback: " + << thd_get_error_context_description( + thd, buffer, sizeof buffer, 512)); +#endif /* !DBUG_OFF */ trx->state = TRX_STATE_NOT_STARTED; } diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 5711a681e633a..379d00b00d785 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -2780,12 +2780,10 @@ RecLock::jump_queue( ut_ad(conflict_lock->trx->lock.que_state == TRX_QUE_LOCK_WAIT); ut_ad(conflict_lock->trx->lock.wait_lock == conflict_lock); -#ifdef UNIV_DEBUG - ib::info() << "Granting High Priority Transaction (ID): " - << lock->trx->id << " the lock jumping over" - << " waiting Transaction (ID): " - << conflict_lock->trx->id; -#endif /* UNIV_DEBUG */ + DBUG_LOG("trx", + "Granting High Priority Transaction " + << lock->trx->id << " a lock jumping over" + << " waiting Transaction " << conflict_lock->trx->id); lock_reset_lock_and_trx_wait(lock); return(true); @@ -2954,11 +2952,12 @@ RecLock::make_trx_hit_list( /* Assert that it is not waiting for current record. */ ut_ad(trx->lock.wait_lock != next); -#ifdef UNIV_DEBUG - ib::info() << "High Priority Transaction (ID): " - << lock->trx->id << " waking up blocking" - << " transaction (ID): " << trx->id; -#endif /* UNIV_DEBUG */ + + DBUG_LOG("trx", "High Priority Transaction " + << lock->trx->id + << " waking up blocking transaction " + << trx->id); + trx->lock.was_chosen_as_deadlock_victim = true; lock_cancel_waiting_and_release(trx->lock.wait_lock); trx_mutex_exit(trx); diff --git a/storage/innobase/row/row0purge.cc b/storage/innobase/row/row0purge.cc index 23e08c38cdd4a..b41e3642cf0e2 100644 --- a/storage/innobase/row/row0purge.cc +++ b/storage/innobase/row/row0purge.cc @@ -515,22 +515,22 @@ row_purge_remove_sec_if_poss_leaf( page = btr_cur_get_page(btr_cur); if (!lock_test_prdt_page_lock( - trx, - page_get_space_id(page), - page_get_page_no(page)) - && page_get_n_recs(page) < 2 - && page_get_page_no(page) != - dict_index_get_page(index)) { + trx, + page_get_space_id(page), + page_get_page_no(page)) + && page_get_n_recs(page) < 2 + && btr_cur_get_block(btr_cur) + ->page.id.page_no() != + dict_index_get_page(index)) { /* this is the last record on page, and it has a "page" lock on it, which mean search is still depending on it, so do not delete */ -#ifdef UNIV_DEBUG - ib::info() << "skip purging last" - " record on page " - << page_get_page_no(page) - << "."; -#endif /* UNIV_DEBUG */ + DBUG_LOG("purge", + "skip purging last" + " record on page " + << btr_cur_get_block(btr_cur) + ->page.id); btr_pcur_close(&pcur); mtr_commit(&mtr); diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 3526db99e7965..acde762a0b72c 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -933,10 +933,9 @@ trx_purge_mark_undo_for_truncate( return; } -#ifdef UNIV_DEBUG - ib::info() << "UNDO tablespace with space identifier " - << undo_trunc->get_marked_space_id() << " marked for truncate"; -#endif /* UNIV_DEBUG */ + DBUG_LOG("undo", + "marking for truncate UNDO tablespace " + << undo_trunc->get_marked_space_id()); /* Step-3: Iterate over all the rsegs of selected UNDO tablespace and mark them temporarily unavailable for allocation.*/ diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index 68a0ad535500c..3d1ac5172b395 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -3379,24 +3379,20 @@ trx_kill_blocking(trx_t* trx) trx_mutex_exit(victim_trx); -#ifdef UNIV_DEBUG +#ifndef DBUG_OFF char buffer[1024]; - char* thr_text; - trx_id_t id; +#endif /* !DBUG_OFF */ - thr_text = thd_get_error_context_description(victim_trx->mysql_thd, - buffer, sizeof(buffer), - 512); - id = victim_trx->id; -#endif /* UNIV_DEBUG */ - trx_rollback_for_mysql(victim_trx); + DBUG_LOG("trx", + "High Priority Transaction " + << trx->id << " killed transaction " + << victim_trx->id << " in hit list" + << " - " + << thd_get_error_context_description( + victim_trx->mysql_thd, + buffer, sizeof(buffer), 512)); -#ifdef UNIV_DEBUG - ib::info() << "High Priority Transaction (ID): " - << trx->id << " killed transaction (ID): " - << id << " in hit list" - << " - " << thr_text; -#endif /* UNIV_DEBUG */ + trx_rollback_for_mysql(victim_trx); trx_mutex_enter(victim_trx); version++; From bb109aeea3e696f89357c946a3c04ca112925d11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 16 Jan 2017 13:57:39 +0200 Subject: [PATCH 72/74] MDEV-6076: Fix a broken assertion. --- storage/innobase/page/page0page.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index 5f330b9e0711f..704ed55f2b34f 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -480,7 +480,7 @@ page_create_zip( /* PAGE_MAX_TRX_ID or PAGE_ROOT_AUTO_INC are always 0 for temporary tables. */ - ut_ad(!dict_table_is_temporary(index->table) || max_trx_id == 0); + ut_ad(max_trx_id == 0 || !dict_table_is_temporary(index->table)); /* In secondary indexes and the change buffer, PAGE_MAX_TRX_ID must be zero on non-leaf pages. max_trx_id can be 0 when the index consists of an empty root (leaf) page. */ From c849b7df616cc8a5378609f243898644c7689f44 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 17 Jan 2017 11:52:31 +0200 Subject: [PATCH 73/74] MDEV-11785 Remove INFORMATION_SCHEMA.INNODB_TEMP_TABLE_INFO The INFORMATION_SCHEMA view INNODB_TEMP_TABLE_INFO was added to MySQL 5.7 as part of the work to implement temporary tables without any redo logging. The only use case of this view was SELECT COUNT(*) in some tests, to see how many temporary tables exist in InnoDB. The columns do not report much useful information. For example, the table name would not be the user-specified table name, but a generated #sql name. Also, the session that created the table is not identified. --- storage/innobase/handler/ha_innodb.cc | 1 - storage/innobase/handler/i_s.cc | 283 +------------------------- storage/innobase/handler/i_s.h | 3 +- 3 files changed, 2 insertions(+), 285 deletions(-) diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 9e9a6a5994ddd..4e1d8e3367bd4 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -23396,7 +23396,6 @@ i_s_innodb_cmp_per_index_reset, i_s_innodb_buffer_page, i_s_innodb_buffer_page_lru, i_s_innodb_buffer_stats, -i_s_innodb_temp_table_info, i_s_innodb_metrics, i_s_innodb_ft_default_stopword, i_s_innodb_ft_deleted, diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 4a13faa480f22..01fadaf8e10e0 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2016, Oracle and/or its affiliates. -Copyrigth (c) 2014, 2016, MariaDB Corporation +Copyrigth (c) 2014, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -4097,287 +4097,6 @@ UNIV_INTERN struct st_maria_plugin i_s_innodb_ft_config = STRUCT_FLD(maturity, MariaDB_PLUGIN_MATURITY_STABLE), }; -/* Fields of the dynamic table INNODB_TEMP_TABLE_INFO. */ -static ST_FIELD_INFO i_s_innodb_temp_table_info_fields_info[] = -{ -#define IDX_TEMP_TABLE_ID 0 - {STRUCT_FLD(field_name, "TABLE_ID"), - STRUCT_FLD(field_length, MY_INT64_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONGLONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_NAME 1 - {STRUCT_FLD(field_name, "NAME"), - STRUCT_FLD(field_length, MAX_TABLE_UTF8_LEN), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_MAYBE_NULL), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_N_COLS 2 - {STRUCT_FLD(field_name, "N_COLS"), - STRUCT_FLD(field_length, MY_INT32_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_SPACE_ID 3 - {STRUCT_FLD(field_name, "SPACE"), - STRUCT_FLD(field_length, MY_INT32_NUM_DECIMAL_DIGITS), - STRUCT_FLD(field_type, MYSQL_TYPE_LONG), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_UNSIGNED), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_PTT 4 - {STRUCT_FLD(field_name, "PER_TABLE_TABLESPACE"), - STRUCT_FLD(field_length, 64), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_MAYBE_NULL), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - -#define IDX_TEMP_TABLE_IS_COMPRESSED 5 - {STRUCT_FLD(field_name, "IS_COMPRESSED"), - STRUCT_FLD(field_length, 64), - STRUCT_FLD(field_type, MYSQL_TYPE_STRING), - STRUCT_FLD(value, 0), - STRUCT_FLD(field_flags, MY_I_S_MAYBE_NULL), - STRUCT_FLD(old_name, ""), - STRUCT_FLD(open_method, SKIP_OPEN_TABLE)}, - END_OF_ST_FIELD_INFO -}; - -struct temp_table_info_t{ - table_id_t m_table_id; - char m_table_name[MAX_TABLE_UTF8_LEN]; - unsigned m_n_cols; - unsigned m_space_id; - char m_per_table_tablespace[64]; - char m_is_compressed[64]; -}; - -typedef std::vector > - temp_table_info_cache_t; - -/*******************************************************************//** -Fill Information Schema table INNODB_TEMP_TABLE_INFO for a particular -temp-table -@return 0 on success, 1 on failure */ -static -int -i_s_innodb_temp_table_info_fill( -/*=============================*/ - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables - to fill */ - const temp_table_info_t* info) /*!< in: temp-table - information */ -{ - TABLE* table; - Field** fields; - - DBUG_ENTER("i_s_innodb_temp_table_info_fill"); - - table = tables->table; - - fields = table->field; - - OK(fields[IDX_TEMP_TABLE_ID]->store(info->m_table_id, true)); - - OK(field_store_string( - fields[IDX_TEMP_TABLE_NAME], info->m_table_name)); - - OK(fields[IDX_TEMP_TABLE_N_COLS]->store(info->m_n_cols)); - - OK(fields[IDX_TEMP_TABLE_SPACE_ID]->store(info->m_space_id)); - - OK(field_store_string( - fields[IDX_TEMP_TABLE_PTT], info->m_per_table_tablespace)); - - OK(field_store_string( - fields[IDX_TEMP_TABLE_IS_COMPRESSED], info->m_is_compressed)); - - DBUG_RETURN(schema_table_store_record(thd, table)); -} - -/*******************************************************************//** -Populate current table information to cache */ -static -void -innodb_temp_table_populate_cache( -/*=============================*/ - const dict_table_t* table, /*! in: table */ - temp_table_info_t* cache) /*! in/out: populate data in this - cache */ -{ - cache->m_table_id = table->id; - - char db_utf8[MAX_DB_UTF8_LEN]; - char table_utf8[MAX_TABLE_UTF8_LEN]; - - dict_fs2utf8(table->name.m_name, - db_utf8, sizeof(db_utf8), - table_utf8, sizeof(table_utf8)); - strcpy(cache->m_table_name, table_utf8); - - cache->m_n_cols = table->n_cols; - - cache->m_space_id = table->space; - - if (fsp_is_system_temporary(table->space)) { - strcpy(cache->m_per_table_tablespace, "FALSE"); - } else { - strcpy(cache->m_per_table_tablespace, "TRUE"); - } - - if (dict_table_page_size(table).is_compressed()) { - strcpy(cache->m_is_compressed, "TRUE"); - } else { - strcpy(cache->m_is_compressed, "FALSE"); - } -} - -/*******************************************************************//** -This function will iterate over all available table and will fill -stats for temp-tables to INNODB_TEMP_TABLE_INFO. -@return 0 on success, 1 on failure */ -static -int -i_s_innodb_temp_table_info_fill_table( -/*===================================*/ - THD* thd, /*!< in: thread */ - TABLE_LIST* tables, /*!< in/out: tables to fill */ - Item* ) /*!< in: condition (ignored) */ -{ - int status = 0; - dict_table_t* table = NULL; - - DBUG_ENTER("i_s_innodb_temp_table_info_fill_table"); - - /* Only allow the PROCESS privilege holder to access the stats */ - if (check_global_access(thd, PROCESS_ACL)) { - DBUG_RETURN(0); - } - - /* First populate all temp-table info by acquiring dict_sys->mutex. - Note: Scan is being done on NON-LRU list which mainly has system - table entries and temp-table entries. This means 2 things: list - is smaller so processing would be faster and most of the data - is relevant */ - temp_table_info_cache_t all_temp_info_cache; - all_temp_info_cache.reserve(UT_LIST_GET_LEN(dict_sys->table_non_LRU)); - - mutex_enter(&dict_sys->mutex); - for (table = UT_LIST_GET_FIRST(dict_sys->table_non_LRU); - table != NULL; - table = UT_LIST_GET_NEXT(table_LRU, table)) { - - if (!dict_table_is_temporary(table)) { - continue; - } - - temp_table_info_t current_temp_table_info; - - innodb_temp_table_populate_cache( - table, ¤t_temp_table_info); - - all_temp_info_cache.push_back(current_temp_table_info); - } - mutex_exit(&dict_sys->mutex); - - /* Now populate the info to MySQL table */ - temp_table_info_cache_t::const_iterator end = all_temp_info_cache.end(); - for (temp_table_info_cache_t::const_iterator it - = all_temp_info_cache.begin(); - it != end; - it++) { - status = i_s_innodb_temp_table_info_fill(thd, tables, &(*it)); - if (status) { - break; - } - } - - DBUG_RETURN(status); -} - -/*******************************************************************//** -Bind the dynamic table INFORMATION_SCHEMA.INNODB_TEMP_TABLE_INFO. -@return 0 on success, 1 on failure */ -static -int -i_s_innodb_temp_table_info_init( -/*=============================*/ - void* p) /*!< in/out: table schema object */ -{ - ST_SCHEMA_TABLE* schema; - - DBUG_ENTER("i_s_innodb_temp_table_info_init"); - - schema = reinterpret_cast(p); - - schema->fields_info = i_s_innodb_temp_table_info_fields_info; - schema->fill_table = i_s_innodb_temp_table_info_fill_table; - - DBUG_RETURN(0); -} - -struct st_maria_plugin i_s_innodb_temp_table_info = -{ - /* the plugin type (a MYSQL_XXX_PLUGIN value) */ - /* int */ - STRUCT_FLD(type, MYSQL_INFORMATION_SCHEMA_PLUGIN), - - /* pointer to type-specific plugin descriptor */ - /* void* */ - STRUCT_FLD(info, &i_s_info), - - /* plugin name */ - /* const char* */ - STRUCT_FLD(name, "INNODB_TEMP_TABLE_INFO"), - - /* plugin author (for SHOW PLUGINS) */ - /* const char* */ - STRUCT_FLD(author, plugin_author), - - /* general descriptive text (for SHOW PLUGINS) */ - /* const char* */ - STRUCT_FLD(descr, "InnoDB Temp Table Stats"), - - /* the plugin license (PLUGIN_LICENSE_XXX) */ - /* int */ - STRUCT_FLD(license, PLUGIN_LICENSE_GPL), - - /* the function to invoke when plugin is loaded */ - /* int (*)(void*); */ - STRUCT_FLD(init, i_s_innodb_temp_table_info_init), - - /* the function to invoke when plugin is unloaded */ - /* int (*)(void*); */ - STRUCT_FLD(deinit, i_s_common_deinit), - - /* plugin version (for SHOW PLUGINS) */ - /* unsigned int */ - STRUCT_FLD(version, INNODB_VERSION_SHORT), - - /* struct st_mysql_show_var* */ - STRUCT_FLD(status_vars, NULL), - - /* struct st_mysql_sys_var** */ - STRUCT_FLD(system_vars, NULL), - - STRUCT_FLD(version_info, INNODB_VERSION_STR), - STRUCT_FLD(maturity, MariaDB_PLUGIN_MATURITY_GAMMA) -}; - /* Fields of the dynamic table INNODB_BUFFER_POOL_STATS. */ static ST_FIELD_INFO i_s_innodb_buffer_stats_fields_info[] = { diff --git a/storage/innobase/handler/i_s.h b/storage/innobase/handler/i_s.h index d272490dd2748..8d34fbf8fbbc5 100644 --- a/storage/innobase/handler/i_s.h +++ b/storage/innobase/handler/i_s.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2007, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyrigth (c) 2014, 2016, MariaDB Corporation +Copyrigth (c) 2014, 2017, MariaDB Corporation This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -50,7 +50,6 @@ extern struct st_maria_plugin i_s_innodb_ft_config; extern struct st_maria_plugin i_s_innodb_buffer_page; extern struct st_maria_plugin i_s_innodb_buffer_page_lru; extern struct st_maria_plugin i_s_innodb_buffer_stats; -extern struct st_maria_plugin i_s_innodb_temp_table_info; extern struct st_maria_plugin i_s_innodb_sys_tables; extern struct st_maria_plugin i_s_innodb_sys_tablestats; extern struct st_maria_plugin i_s_innodb_sys_indexes; From 6a65de6cda17cc955d73b992f91acb76e8ef7512 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 17 Jan 2017 15:28:50 +0200 Subject: [PATCH 74/74] Correct a test broken in the merge 7c81f15ec3bcc --- mysql-test/suite/innodb_zip/t/innochecksum_3.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb_zip/t/innochecksum_3.test b/mysql-test/suite/innodb_zip/t/innochecksum_3.test index 4b386099f865c..60c4b167d9a83 100644 --- a/mysql-test/suite/innodb_zip/t/innochecksum_3.test +++ b/mysql-test/suite/innodb_zip/t/innochecksum_3.test @@ -163,9 +163,9 @@ foreach (glob("$ENV{MYSQLD_DATADIR}/*/*.ibd")) { } EOF -let $restart_parameters = restart: --innodb_checksum_algorithm=strict_none --default_storage_engine=InnoDB; +--let $restart_parameters = --innodb_checksum_algorithm=strict_none --default_storage_engine=InnoDB --source include/start_mysqld.inc - +--let $restart_parameters= # check the table status is GOOD with DML INSERT INTO tab1 (pk, linestring_key, linestring_nokey) VALUES (4, ST_GeomFromText('MULTIPOINT(0 0,5 5,10 10,20 20) '), ST_GeomFromText('MULTIPOINT(0 0,5 5,10 10,20 20) '));