From d537fd3b39ff586d45659092ebd359e37c055e0c Mon Sep 17 00:00:00 2001 From: Varun Nagaraju Date: Fri, 28 Jun 2024 12:40:41 +0530 Subject: [PATCH 1/2] PS-9222 ALTER TABLE ALGORITHM=INSTANT FIX #1 https://perconadev.atlassian.net/browse/PS-9222 Problem ======= When writing to the redo log, an issue of column order change not being recorded with INSTANT DDL was fixed by creating an array with size equal to the number of fields in the index which kept track of whether the original position of the field was changed or not. Later, that array would be used to make a decision on logging the field. But, this solution didn't take into account the fact that there could be column prefixes because of the primary key. This resulted in inaccurate entries being filled in the fields_with_changed_order[] array. Solution ======== It is fixed by using the method, get_col_phy_pos() which takes into account the existence of column prefix instead of get_phy_pos() while generating fields_with_changed_order[] array. --- .../r/instant_alter_index_prefix.result | 38 ++++++++++++++ .../innodb/t/instant_alter_index_prefix.test | 50 +++++++++++++++++++ storage/innobase/mtr/mtr0log.cc | 4 +- 3 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 mysql-test/suite/innodb/r/instant_alter_index_prefix.result create mode 100644 mysql-test/suite/innodb/t/instant_alter_index_prefix.test diff --git a/mysql-test/suite/innodb/r/instant_alter_index_prefix.result b/mysql-test/suite/innodb/r/instant_alter_index_prefix.result new file mode 100644 index 000000000000..c38870b25bae --- /dev/null +++ b/mysql-test/suite/innodb/r/instant_alter_index_prefix.result @@ -0,0 +1,38 @@ +without prefix field in the index +CREATE TABLE t1 (c1 TINYTEXT COLLATE ascii_bin NOT NULL, c2 DATETIME(3) NOT NULL, c3 TEXT, UNIQUE KEY (c1(30))); +INSERT INTO t1 (c1, c2, c3) VALUE ('k1','2021-12-21','something'); +INSERT INTO t1 (c1, c2, c3) VALUE ('k3','2021-12-21','something else'); +ALTER TABLE t1 ADD COLUMN c4 VARCHAR(18) NOT NULL, ALGORITHM=INSTANT; +SET GLOBAL innodb_log_checkpoint_now = ON; +SET GLOBAL innodb_page_cleaner_disabled_debug = 1; +SET GLOBAL innodb_dict_stats_disabled_debug = 1; +SET GLOBAL innodb_master_thread_disabled_debug = 1; +SET GLOBAL innodb_checkpoint_disabled = ON; +UPDATE t1 SET c4 = 'value' WHERE c1 = 'k1'; +# Restart the server and reload the table to see if tables are corrupted. +# Kill and restart +# Run a select to confirm that the database started up successfully +SELECT * FROM t1; +c1 c2 c3 c4 +k1 2021-12-21 00:00:00.000 something value +k3 2021-12-21 00:00:00.000 something else +DROP TABLE t1; +with prefix field in the index +CREATE TABLE t1 (c1 TINYTEXT COLLATE ascii_bin NOT NULL, c2 DATETIME(3) NOT NULL, c3 TEXT, PRIMARY KEY (c1(30))); +INSERT INTO t1 (c1, c2, c3) VALUE ('k1','2021-12-21','something'); +INSERT INTO t1 (c1, c2, c3) VALUE ('k3','2021-12-21','something else'); +ALTER TABLE t1 ADD COLUMN c4 VARCHAR(18) NOT NULL, ALGORITHM=INSTANT; +SET GLOBAL innodb_log_checkpoint_now = ON; +SET GLOBAL innodb_page_cleaner_disabled_debug = 1; +SET GLOBAL innodb_dict_stats_disabled_debug = 1; +SET GLOBAL innodb_master_thread_disabled_debug = 1; +SET GLOBAL innodb_checkpoint_disabled = ON; +UPDATE t1 SET c4 = 'value' WHERE c1 = 'k1'; +# Restart the server and reload the table to see if tables are corrupted. +# Kill and restart +# Run a select to confirm that the database started up successfully +SELECT * FROM t1; +c1 c2 c3 c4 +k1 2021-12-21 00:00:00.000 something value +k3 2021-12-21 00:00:00.000 something else +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/instant_alter_index_prefix.test b/mysql-test/suite/innodb/t/instant_alter_index_prefix.test new file mode 100644 index 000000000000..f01ac58105bc --- /dev/null +++ b/mysql-test/suite/innodb/t/instant_alter_index_prefix.test @@ -0,0 +1,50 @@ +# PS-9222 Testing if ALGORITHM=INSTANT crashes server +source include/have_debug.inc; + +--echo without prefix field in the index +CREATE TABLE t1 (c1 TINYTEXT COLLATE ascii_bin NOT NULL, c2 DATETIME(3) NOT NULL, c3 TEXT, UNIQUE KEY (c1(30))); +INSERT INTO t1 (c1, c2, c3) VALUE ('k1','2021-12-21','something'); +INSERT INTO t1 (c1, c2, c3) VALUE ('k3','2021-12-21','something else'); + +ALTER TABLE t1 ADD COLUMN c4 VARCHAR(18) NOT NULL, ALGORITHM=INSTANT; + +SET GLOBAL innodb_log_checkpoint_now = ON; +SET GLOBAL innodb_page_cleaner_disabled_debug = 1; +SET GLOBAL innodb_dict_stats_disabled_debug = 1; +SET GLOBAL innodb_master_thread_disabled_debug = 1; +SET GLOBAL innodb_checkpoint_disabled = ON; + +UPDATE t1 SET c4 = 'value' WHERE c1 = 'k1'; + +--echo # Restart the server and reload the table to see if tables are corrupted. +--source include/kill_and_restart_mysqld.inc + +-- echo # Run a select to confirm that the database started up successfully +SELECT * FROM t1; + +# cleanup +DROP TABLE t1; + +--echo with prefix field in the index +CREATE TABLE t1 (c1 TINYTEXT COLLATE ascii_bin NOT NULL, c2 DATETIME(3) NOT NULL, c3 TEXT, PRIMARY KEY (c1(30))); +INSERT INTO t1 (c1, c2, c3) VALUE ('k1','2021-12-21','something'); +INSERT INTO t1 (c1, c2, c3) VALUE ('k3','2021-12-21','something else'); + +ALTER TABLE t1 ADD COLUMN c4 VARCHAR(18) NOT NULL, ALGORITHM=INSTANT; + +SET GLOBAL innodb_log_checkpoint_now = ON; +SET GLOBAL innodb_page_cleaner_disabled_debug = 1; +SET GLOBAL innodb_dict_stats_disabled_debug = 1; +SET GLOBAL innodb_master_thread_disabled_debug = 1; +SET GLOBAL innodb_checkpoint_disabled = ON; + +UPDATE t1 SET c4 = 'value' WHERE c1 = 'k1'; + +--echo # Restart the server and reload the table to see if tables are corrupted. +--source include/kill_and_restart_mysqld.inc + +-- echo # Run a select to confirm that the database started up successfully +SELECT * FROM t1; + +# cleanup +DROP TABLE t1; diff --git a/storage/innobase/mtr/mtr0log.cc b/storage/innobase/mtr/mtr0log.cc index 51704f1446d1..6d205c7221a4 100644 --- a/storage/innobase/mtr/mtr0log.cc +++ b/storage/innobase/mtr/mtr0log.cc @@ -862,8 +862,8 @@ bool mlog_open_and_write_index(mtr_t *mtr, const byte *rec, if (col->is_instant_added() || col->is_instant_dropped()) { continue; - } else if (col->get_phy_pos() >= phy_pos) { - phy_pos = col->get_phy_pos(); + } else if (col->get_col_phy_pos() >= phy_pos) { + phy_pos = col->get_col_phy_pos(); } else { fields_with_changed_order[i] = true; } From 39e99576bab18c8e7e8193985d9ed3015719788e Mon Sep 17 00:00:00 2001 From: Varun Nagaraju Date: Fri, 28 Jun 2024 13:17:31 +0530 Subject: [PATCH 2/2] PS-9222 ALTER TABLE ALGORITHM=INSTANT FIX #2 https://perconadev.atlassian.net/browse/PS-9222 Problem ======= When writing to the redo log, an issue of column order change not being recorded with INSTANT DDL was fixed by checking if the fields are also reordered, then adding the columns into the list. However when calculating the size of the buffer this fix doesn't take account the extra fields that may be logged, and causing the assertion on the buffer size failed eventually. Solution ======== To calculate the buffer size correctly, we move the logic of finding reordered fiedls before buffer size calculation, then count the number of fields with the same logic when deciding if a field needs to be logged. --- .../instant_alter_stored_column_order.result | 44 ++++++++++ .../t/instant_alter_stored_column_order.test | 56 ++++++++++++ storage/innobase/mtr/mtr0log.cc | 88 ++++++++++--------- 3 files changed, 148 insertions(+), 40 deletions(-) create mode 100644 mysql-test/suite/innodb/r/instant_alter_stored_column_order.result create mode 100644 mysql-test/suite/innodb/t/instant_alter_stored_column_order.test diff --git a/mysql-test/suite/innodb/r/instant_alter_stored_column_order.result b/mysql-test/suite/innodb/r/instant_alter_stored_column_order.result new file mode 100644 index 000000000000..afc915c1a7c4 --- /dev/null +++ b/mysql-test/suite/innodb/r/instant_alter_stored_column_order.result @@ -0,0 +1,44 @@ +without prefix field in the index +CREATE TABLE t1 (c1 TINYTEXT COLLATE ascii_bin NOT NULL, c2 DATETIME(3) NOT NULL, c3 TEXT, UNIQUE KEY (c1(30))); +INSERT INTO t1 (c1, c2, c3) VALUE ('k1','2021-12-21','something'); +INSERT INTO t1 (c1, c2, c3) VALUE ('k3','2021-12-21','something else'); +ALTER TABLE t1 +CHANGE c2 c2 DATETIME(3) NOT NULL AFTER c3, +ADD COLUMN c4 VARCHAR(18) NOT NULL, +ALGORITHM=INSTANT; +SET GLOBAL innodb_log_checkpoint_now = ON; +SET GLOBAL innodb_page_cleaner_disabled_debug = 1; +SET GLOBAL innodb_dict_stats_disabled_debug = 1; +SET GLOBAL innodb_master_thread_disabled_debug = 1; +SET GLOBAL innodb_checkpoint_disabled = ON; +UPDATE t1 SET c4 = 'value' WHERE c1 = 'k1'; +# Restart the server and reload the table to see if tables are corrupted. +# Kill and restart +# Run a select to confirm that the database started up successfully +SELECT * FROM t1; +c1 c3 c2 c4 +k1 something 2021-12-21 00:00:00.000 value +k3 something else 2021-12-21 00:00:00.000 +DROP TABLE t1; +with prefix field in the index +CREATE TABLE t1 (c1 TINYTEXT COLLATE ascii_bin NOT NULL, c2 DATETIME(3) NOT NULL, c3 TEXT, PRIMARY KEY (c1(30))); +INSERT INTO t1 (c1, c2, c3) VALUE ('k1','2021-12-21','something'); +INSERT INTO t1 (c1, c2, c3) VALUE ('k3','2021-12-21','something else'); +ALTER TABLE t1 +CHANGE c2 c2 DATETIME(3) NOT NULL AFTER c3, +ADD COLUMN c4 VARCHAR(18) NOT NULL, +ALGORITHM=INSTANT; +SET GLOBAL innodb_log_checkpoint_now = ON; +SET GLOBAL innodb_page_cleaner_disabled_debug = 1; +SET GLOBAL innodb_dict_stats_disabled_debug = 1; +SET GLOBAL innodb_master_thread_disabled_debug = 1; +SET GLOBAL innodb_checkpoint_disabled = ON; +UPDATE t1 SET c4 = 'value' WHERE c1 = 'k1'; +# Restart the server and reload the table to see if tables are corrupted. +# Kill and restart +# Run a select to confirm that the database started up successfully +SELECT * FROM t1; +c1 c3 c2 c4 +k1 something 2021-12-21 00:00:00.000 value +k3 something else 2021-12-21 00:00:00.000 +DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/instant_alter_stored_column_order.test b/mysql-test/suite/innodb/t/instant_alter_stored_column_order.test new file mode 100644 index 000000000000..e1086efd27ae --- /dev/null +++ b/mysql-test/suite/innodb/t/instant_alter_stored_column_order.test @@ -0,0 +1,56 @@ +# PS-9222 Testing if ALGORITHM=INSTANT crashes server +source include/have_debug.inc; + +--echo without prefix field in the index +CREATE TABLE t1 (c1 TINYTEXT COLLATE ascii_bin NOT NULL, c2 DATETIME(3) NOT NULL, c3 TEXT, UNIQUE KEY (c1(30))); +INSERT INTO t1 (c1, c2, c3) VALUE ('k1','2021-12-21','something'); +INSERT INTO t1 (c1, c2, c3) VALUE ('k3','2021-12-21','something else'); + +ALTER TABLE t1 +CHANGE c2 c2 DATETIME(3) NOT NULL AFTER c3, +ADD COLUMN c4 VARCHAR(18) NOT NULL, +ALGORITHM=INSTANT; + +SET GLOBAL innodb_log_checkpoint_now = ON; +SET GLOBAL innodb_page_cleaner_disabled_debug = 1; +SET GLOBAL innodb_dict_stats_disabled_debug = 1; +SET GLOBAL innodb_master_thread_disabled_debug = 1; +SET GLOBAL innodb_checkpoint_disabled = ON; + +UPDATE t1 SET c4 = 'value' WHERE c1 = 'k1'; + +--echo # Restart the server and reload the table to see if tables are corrupted. +--source include/kill_and_restart_mysqld.inc + +-- echo # Run a select to confirm that the database started up successfully +SELECT * FROM t1; + +# cleanup +DROP TABLE t1; + +--echo with prefix field in the index +CREATE TABLE t1 (c1 TINYTEXT COLLATE ascii_bin NOT NULL, c2 DATETIME(3) NOT NULL, c3 TEXT, PRIMARY KEY (c1(30))); +INSERT INTO t1 (c1, c2, c3) VALUE ('k1','2021-12-21','something'); +INSERT INTO t1 (c1, c2, c3) VALUE ('k3','2021-12-21','something else'); + +ALTER TABLE t1 +CHANGE c2 c2 DATETIME(3) NOT NULL AFTER c3, +ADD COLUMN c4 VARCHAR(18) NOT NULL, +ALGORITHM=INSTANT; + +SET GLOBAL innodb_log_checkpoint_now = ON; +SET GLOBAL innodb_page_cleaner_disabled_debug = 1; +SET GLOBAL innodb_dict_stats_disabled_debug = 1; +SET GLOBAL innodb_master_thread_disabled_debug = 1; +SET GLOBAL innodb_checkpoint_disabled = ON; + +UPDATE t1 SET c4 = 'value' WHERE c1 = 'k1'; + +--echo # Restart the server and reload the table to see if tables are corrupted. +--source include/kill_and_restart_mysqld.inc + +-- echo # Run a select to confirm that the database started up successfully +SELECT * FROM t1; + +# cleanup +DROP TABLE t1; diff --git a/storage/innobase/mtr/mtr0log.cc b/storage/innobase/mtr/mtr0log.cc index 6d205c7221a4..f6085bd723c2 100644 --- a/storage/innobase/mtr/mtr0log.cc +++ b/storage/innobase/mtr/mtr0log.cc @@ -515,18 +515,28 @@ constexpr size_t inst_col_info_size = 6; @param[in] is_comp true if COMP @param[in] is_versioned if table has row versions @param[in] is_instant true if table has INSTANT cols +@param[in] changed_order array indicating fields changed position @param[out] size_needed total size needed on REDO LOG */ -static void log_index_get_size_needed(const dict_index_t *index, size_t size, - uint16_t n, bool is_comp, - bool is_versioned, bool is_instant, - size_t &size_needed) { - auto size_for_versioned_fields = [](const dict_index_t *ind) { +static void log_index_get_size_needed( + const dict_index_t *index, size_t size, uint16_t n, bool is_comp, + bool is_versioned, bool is_instant, + const std::unique_ptr &changed_order, size_t &size_needed) { + auto size_for_versioned_fields = [n, + &changed_order](const dict_index_t *ind) { size_t _size = 0; /* 2 bytes for number of columns with version */ _size += 2; - size_t n_versioned_fields = ind->table->get_n_instant_add_cols() + - ind->table->get_n_instant_drop_cols(); + size_t n_versioned_fields = 0; + for (size_t i = 0; i < n; i++) { + dict_field_t *field = ind->get_field(i); + const dict_col_t *col = field->col; + if (col->is_instant_added() || col->is_instant_dropped() || + changed_order[i]) { + n_versioned_fields += 1; + } + } + ut_ad(n_versioned_fields != 0); _size += n_versioned_fields * inst_col_info_size; @@ -644,6 +654,7 @@ static bool close_and_reopen_log(byte *&log_ptr, const byte *&log_start, const byte *&log_end, mtr_t *&mtr, size_t &alloc, size_t &total) { mlog_close(mtr, log_ptr); + ut_a(total > (ulint)(log_ptr - log_start)); total -= log_ptr - log_start; alloc = total; @@ -673,7 +684,8 @@ template @param[in] func callback to check size reopen log buffer */ static bool log_index_fields(const dict_index_t *index, uint16_t n, bool is_versioned, std::vector &f, - bool *changed_order, byte *&log_ptr, F &func) { + const std::unique_ptr &changed_order, + byte *&log_ptr, F &func) { /* Write metadata for each field. Log the fields in their logical order. */ for (size_t i = 0; i < n; i++) { dict_field_t *field = index->get_field(i); @@ -799,9 +811,36 @@ bool mlog_open_and_write_index(mtr_t *mtr, const byte *rec, n = DICT_INDEX_SPATIAL_NODEPTR_SIZE; } + /* Ordinal position of an existing field can't be changed with INSTANT + algorithm. But when it is combined with ADD/DROP COLUMN, ordinal position + of a filed can be changed. This bool array of size #fields in index, + represents if ordinal position of an existing filed is changed. */ + + // Move the logic to find columns that changed order before calculating + // the buffer size + std::unique_ptr fields_with_changed_order = nullptr; + if (is_versioned) { + fields_with_changed_order = std::make_unique(n); + + uint16_t phy_pos = 0; + for (size_t i = 0; i < n; i++) { + dict_field_t *field = index->get_field(i); + const dict_col_t *col = field->col; + + if (col->is_instant_added() || col->is_instant_dropped()) { + continue; + } + if (col->get_col_phy_pos() >= phy_pos) { + phy_pos = col->get_col_phy_pos(); + } else { + fields_with_changed_order[i] = true; + } + } + } + size_t size_needed = 0; log_index_get_size_needed(index, size, n, is_comp, is_versioned, is_instant, - size_needed); + fields_with_changed_order, size_needed); size_t total = size_needed; size_t alloc = total; if (alloc > mtr_buf_t::MAX_DATA_SIZE) { @@ -846,37 +885,10 @@ bool mlog_open_and_write_index(mtr_t *mtr, const byte *rec, return true; }; - /* Ordinal position of an existing field can't be changed with INSTANT - algorithm. But when it is combined with ADD/DROP COLUMN, ordinal position - of a filed can be changed. This bool array of size #fields in index, - represents if ordinal position of an existing filed is changed. */ - bool *fields_with_changed_order = nullptr; - if (is_versioned) { - fields_with_changed_order = new bool[n]; - memset(fields_with_changed_order, false, (sizeof(bool) * n)); - - uint16_t phy_pos = 0; - for (size_t i = 0; i < n; i++) { - dict_field_t *field = index->get_field(i); - const dict_col_t *col = field->col; - - if (col->is_instant_added() || col->is_instant_dropped()) { - continue; - } else if (col->get_col_phy_pos() >= phy_pos) { - phy_pos = col->get_col_phy_pos(); - } else { - fields_with_changed_order[i] = true; - } - } - } - if (is_comp) { /* Write fields info. */ if (!log_index_fields(index, n, is_versioned, instant_fields_to_log, fields_with_changed_order, log_ptr, f)) { - if (is_versioned) { - delete[] fields_with_changed_order; - } return false; } } else if (is_versioned) { @@ -890,10 +902,6 @@ bool mlog_open_and_write_index(mtr_t *mtr, const byte *rec, } } - if (is_versioned) { - delete[] fields_with_changed_order; - } - if (!instant_fields_to_log.empty()) { ut_ad(is_versioned); /* Log INSTANT ADD/DROP fields */