diff --git a/mysql-test/suite/encryption/r/innodb-discard-import-change.result b/mysql-test/suite/encryption/r/innodb-discard-import-change.result new file mode 100644 index 0000000000000..7071f9eaf2008 --- /dev/null +++ b/mysql-test/suite/encryption/r/innodb-discard-import-change.result @@ -0,0 +1,105 @@ +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t2(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +insert into t1 values (NULL, 'verysecretmessage'); +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +insert into t5 select * from t1; +FLUSH TABLE t1,t2,t3,t4,t5 FOR EXPORT; +backup: t1 +backup: t2 +backup: t3 +backup: t4 +backup: t5 +t1.cfg +t1.frm +t1.ibd +t2.cfg +t2.frm +t2.ibd +t3.cfg +t3.frm +t3.ibd +t4.cfg +t4.frm +t4.ibd +t5.cfg +t5.frm +t5.ibd +UNLOCK TABLES; +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; +DROP TABLE t1; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; +create table t6(a int) engine=innodb; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; +restore: t1 .ibd and .cfg files +restore: t2 .ibd and .cfg files +restore: t3 .ibd and .cfg files +restore: t4 .ibd and .cfg files +restore: t5 .ibd and .cfg files +ALTER TABLE t1 IMPORT TABLESPACE; +SHOW CREATE TABLE t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `c1` bigint(20) NOT NULL AUTO_INCREMENT, + `b` char(200) DEFAULT NULL, + PRIMARY KEY (`c1`) +) ENGINE=InnoDB AUTO_INCREMENT=377 DEFAULT CHARSET=latin1 `encrypted`=yes `encryption_key_id`=4 +SELECT COUNT(*) FROM t1; +COUNT(*) +256 +ALTER TABLE t2 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t2; +COUNT(*) +256 +ALTER TABLE t3 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t3; +COUNT(*) +256 +ALTER TABLE t4 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t4; +COUNT(*) +256 +ALTER TABLE t5 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t5; +COUNT(*) +256 +# t1 encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t1.ibd +# t2 encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t2.ibd +# t3 page compressed expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t3.ibd +# t4 page compressed and encrypted expecting NOT FOUND +NOT FOUND /verysecretmessage/ in t4.ibd +# t5 normal expecting FOUND +FOUND /verysecretmessage/ in t5.ibd +DROP TABLE t1,t2,t3,t4,t5,t6; diff --git a/mysql-test/suite/encryption/t/innodb-discard-import-change.test b/mysql-test/suite/encryption/t/innodb-discard-import-change.test new file mode 100644 index 0000000000000..a278a8fba2910 --- /dev/null +++ b/mysql-test/suite/encryption/t/innodb-discard-import-change.test @@ -0,0 +1,131 @@ +-- source include/have_innodb.inc +-- source include/have_file_key_management_plugin.inc +# +# MDEV-11656: 'Data structure corruption' IMPORT TABLESPACE doesn't work for encrypted InnoDB tables if space_id changed +# + +call mtr.add_suppression("InnoDB: Table .* tablespace is set as discarded"); + +--disable_query_log +let $innodb_file_format_orig = `SELECT @@innodb_file_format`; +let $innodb_file_per_table_orig = `SELECT @@innodb_file_per_table`; +let $innodb_compression_algo = `SELECT @@innodb_compression_algorithm`; +--enable_query_log + +--disable_warnings +SET GLOBAL innodb_file_format = `Barracuda`; +SET GLOBAL innodb_file_per_table = ON; +SET GLOBAL innodb_compression_algorithm = 1; +--enable_warnings + +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t2(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=1; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; + +insert into t1 values (NULL, 'verysecretmessage'); +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t1(b) select b from t1; +insert into t2 select * from t1; +insert into t3 select * from t1; +insert into t4 select * from t1; +insert into t5 select * from t1; + +let MYSQLD_DATADIR =`SELECT @@datadir`; +FLUSH TABLE t1,t2,t3,t4,t5 FOR EXPORT; +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_backup_tablespaces("test", "t1","t2","t3","t4","t5"); +EOF +--list_files $MYSQLD_DATADIR/test +UNLOCK TABLES; + +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t2 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; + +# +# Now intentionally change space_id for t1,t3,t4,t5 +# +DROP TABLE t1; +DROP TABLE t3; +DROP TABLE t4; +DROP TABLE t5; + +create table t6(a int) engine=innodb; +create table t5(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb; +create table t3(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes; +create table t1(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb encrypted=yes encryption_key_id=4; +create table t4(c1 bigint not null primary key auto_increment, b char(200)) engine=innodb page_compressed=yes encrypted=yes encryption_key_id=4; + +ALTER TABLE t1 DISCARD TABLESPACE; +ALTER TABLE t3 DISCARD TABLESPACE; +ALTER TABLE t4 DISCARD TABLESPACE; +ALTER TABLE t5 DISCARD TABLESPACE; + +perl; +do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl"; +ib_discard_tablespaces("test", "t1","t2","t3","t4","t5"); +ib_restore_tablespaces("test", "t1","t2","t3","t4","t5"); +EOF + +ALTER TABLE t1 IMPORT TABLESPACE; +SHOW CREATE TABLE t1; +SELECT COUNT(*) FROM t1; +ALTER TABLE t2 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t2; +ALTER TABLE t3 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t3; +ALTER TABLE t4 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t4; +ALTER TABLE t5 IMPORT TABLESPACE; +SELECT COUNT(*) FROM t5; + +# +# Verify +# +--let $MYSQLD_TMPDIR = `SELECT @@tmpdir` +--let $MYSQLD_DATADIR = `SELECT @@datadir` +--let SEARCH_RANGE = 10000000 +--let t1_IBD = $MYSQLD_DATADIR/test/t1.ibd +--let t2_IBD = $MYSQLD_DATADIR/test/t2.ibd +--let t3_IBD = $MYSQLD_DATADIR/test/t3.ibd +--let t4_IBD = $MYSQLD_DATADIR/test/t4.ibd +--let t5_IBD = $MYSQLD_DATADIR/test/t5.ibd +--let SEARCH_PATTERN=verysecretmessage +--echo # t1 encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t1_IBD +-- source include/search_pattern_in_file.inc +--echo # t2 encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t2_IBD +-- source include/search_pattern_in_file.inc +--echo # t3 page compressed expecting NOT FOUND +-- let SEARCH_FILE=$t3_IBD +-- source include/search_pattern_in_file.inc +--echo # t4 page compressed and encrypted expecting NOT FOUND +-- let SEARCH_FILE=$t4_IBD +-- source include/search_pattern_in_file.inc +--echo # t5 normal expecting FOUND +-- let SEARCH_FILE=$t5_IBD +-- source include/search_pattern_in_file.inc + +DROP TABLE t1,t2,t3,t4,t5,t6; + +# reset system +--disable_warnings +--disable_query_log +EVAL SET GLOBAL innodb_file_per_table = $innodb_file_per_table_orig; +EVAL SET GLOBAL innodb_file_format = $innodb_file_format_orig; +EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algo; +--enable_query_log +--enable_warnings + diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 22352d94332d2..ce5c62a8c8b73 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -6533,6 +6533,7 @@ fil_iterate( for (offset = iter.start; offset < iter.end; offset += n_bytes) { byte* io_buffer = iter.io_buffer; + bool row_compressed = false; block->frame = io_buffer; @@ -6545,6 +6546,7 @@ fil_iterate( /* Zip IO is done in the compressed page buffer. */ io_buffer = block->page.zip.data; + row_compressed = true; } else { io_buffer = iter.io_buffer; } @@ -6585,8 +6587,9 @@ fil_iterate( for (ulint i = 0; i < n_pages_read; ++i) { ulint size = iter.page_size; dberr_t err = DB_SUCCESS; - byte* src = (readptr + (i * size)); - byte* dst = (io_buffer + (i * size)); + byte* src = readptr + (i * size); + byte* dst = io_buffer + (i * size); + bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -6610,8 +6613,12 @@ fil_iterate( if (decrypted) { updated = true; } else { - /* TODO: remove unnecessary memcpy's */ - memcpy(dst, src, size); + if (!page_compressed && !row_compressed) { + block->frame = src; + frame_changed = true; + } else { + memcpy(dst, src, size); + } } } @@ -6636,7 +6643,45 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - src = (io_buffer + (i * size)); + /* If tablespace is encrypted we use additional + temporary scratch area where pages are read + for decrypting readptr == crypt_io_buffer != io_buffer. + + Destination for decryption is a buffer pool block + block->frame == dst == io_buffer that is updated. + Pages that did not require decryption even when + tablespace is marked as encrypted are not copied + instead block->frame is set to src == readptr. + + For encryption we again use temporary scratch area + writeptr != io_buffer == dst + that is then written to the tablespace + + (1) For normal tables io_buffer == dst == writeptr + (2) For only page compressed tables + io_buffer == dst == writeptr + (3) For encrypted (and page compressed) + readptr != io_buffer == dst != writeptr + */ + + ut_ad(!encrypted && !page_compressed ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(page_compressed && !encrypted ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(encrypted ? + src != dst && dst != writeptr + (i * size):1); + + if (encrypted) { + memcpy(writeptr + (i * size), + row_compressed ? block->page.zip.data : + block->frame, size); + } + + if (frame_changed) { + block->frame = dst; + } + + src = io_buffer + (i * size); if (page_compressed) { ulint len = 0; @@ -6657,7 +6702,7 @@ fil_iterate( write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *dest = (writeptr + (i * size)); + byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6839,9 +6884,9 @@ fil_tablespace_iterate( void* crypt_io_buffer = NULL; if (iter.crypt_data != NULL) { crypt_io_buffer = mem_alloc( - iter.n_io_buffers * UNIV_PAGE_SIZE); + (2 + iter.n_io_buffers) * UNIV_PAGE_SIZE); iter.crypt_io_buffer = static_cast( - crypt_io_buffer); + ut_align(crypt_io_buffer, UNIV_PAGE_SIZE)); } err = fil_iterate(iter, &block, callback); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 5900bdb21402c..e7da4569f0d1f 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -6594,6 +6594,7 @@ fil_iterate( for (offset = iter.start; offset < iter.end; offset += n_bytes) { byte* io_buffer = iter.io_buffer; + bool row_compressed = false; block->frame = io_buffer; @@ -6606,6 +6607,7 @@ fil_iterate( /* Zip IO is done in the compressed page buffer. */ io_buffer = block->page.zip.data; + row_compressed = true; } else { io_buffer = iter.io_buffer; } @@ -6646,8 +6648,9 @@ fil_iterate( for (ulint i = 0; i < n_pages_read; ++i) { ulint size = iter.page_size; dberr_t err = DB_SUCCESS; - byte* src = (readptr + (i * size)); - byte* dst = (io_buffer + (i * size)); + byte* src = readptr + (i * size); + byte* dst = io_buffer + (i * size); + bool frame_changed = false; ulint page_type = mach_read_from_2(src+FIL_PAGE_TYPE); @@ -6671,8 +6674,12 @@ fil_iterate( if (decrypted) { updated = true; } else { - /* TODO: remove unnecessary memcpy's */ - memcpy(dst, src, size); + if (!page_compressed && !row_compressed) { + block->frame = src; + frame_changed = true; + } else { + memcpy(dst, src, size); + } } } @@ -6697,7 +6704,45 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - src = (io_buffer + (i * size)); + /* If tablespace is encrypted we use additional + temporary scratch area where pages are read + for decrypting readptr == crypt_io_buffer != io_buffer. + + Destination for decryption is a buffer pool block + block->frame == dst == io_buffer that is updated. + Pages that did not require decryption even when + tablespace is marked as encrypted are not copied + instead block->frame is set to src == readptr. + + For encryption we again use temporary scratch area + writeptr != io_buffer == dst + that is then written to the tablespace + + (1) For normal tables io_buffer == dst == writeptr + (2) For only page compressed tables + io_buffer == dst == writeptr + (3) For encrypted (and page compressed) + readptr != io_buffer == dst != writeptr + */ + + ut_ad(!encrypted && !page_compressed ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(page_compressed && !encrypted ? + src == dst && dst == writeptr + (i * size):1); + ut_ad(encrypted ? + src != dst && dst != writeptr + (i * size):1); + + if (encrypted) { + memcpy(writeptr + (i * size), + row_compressed ? block->page.zip.data : + block->frame, size); + } + + if (frame_changed) { + block->frame = dst; + } + + src = io_buffer + (i * size); if (page_compressed) { ulint len = 0; @@ -6718,7 +6763,7 @@ fil_iterate( write it back. Note that we should not encrypt the buffer that is in buffer pool. */ if (decrypted && encrypted) { - unsigned char *dest = (writeptr + (i * size)); + byte *dest = writeptr + (i * size); ulint space = mach_read_from_4( src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); ulint offset = mach_read_from_4(src + FIL_PAGE_OFFSET); @@ -6900,9 +6945,9 @@ fil_tablespace_iterate( void* crypt_io_buffer = NULL; if (iter.crypt_data != NULL) { crypt_io_buffer = mem_alloc( - iter.n_io_buffers * UNIV_PAGE_SIZE); + (2 + iter.n_io_buffers) * UNIV_PAGE_SIZE); iter.crypt_io_buffer = static_cast( - crypt_io_buffer); + ut_align(crypt_io_buffer, UNIV_PAGE_SIZE)); } err = fil_iterate(iter, &block, callback);