diff --git a/mysql-test/r/mysqltest.result b/mysql-test/r/mysqltest.result index f3103eecfc8e..dc6070999c19 100644 --- a/mysql-test/r/mysqltest.result +++ b/mysql-test/r/mysqltest.result @@ -1228,6 +1228,7 @@ ERROR HY000: MySQL server has gone away # # Bug#23743035: ADD A COPY_FILES_WILDCARD COMMAND # +README_tablespace_portable_linux.txt bug20683959loaddata.txt bug30435_10k_items.txt bug30435_5k.txt @@ -1252,6 +1253,7 @@ numbers.txt wl5766_data.txt words.dat words2.dat +README_tablespace_portable_linux.txt bug20683959loaddata.txt bug30435_10k_items.txt bug30435_5k.txt diff --git a/mysql-test/r/partition_innodb.result b/mysql-test/r/partition_innodb.result index 56c54473d6d1..c6739d529f03 100644 --- a/mysql-test/r/partition_innodb.result +++ b/mysql-test/r/partition_innodb.result @@ -1114,6 +1114,7 @@ t1 CREATE TABLE `t1` ( t1#p#p1.ibd t1#p#p0.ibd DROP TABLE t1; +# restart: --innodb-directories=MYSQL_TMP_DIR CREATE TABLESPACE ts1 ADD DATAFILE 'MYSQL_TMP_DIR/ts1.ibd' ENGINE=Innodb; CREATE TABLE `t1` ( `f1` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT, diff --git a/mysql-test/r/tablespace.result b/mysql-test/r/tablespace.result index 23c362c04e13..ee9db5afce39 100644 --- a/mysql-test/r/tablespace.result +++ b/mysql-test/r/tablespace.result @@ -577,6 +577,7 @@ SET AUTOCOMMIT=ON; # # Test ALTER TABLESPACE RENAME with partition tables CREATE TABLESPACE ts1 ADD DATAFILE 'ts1.ibd' Engine=InnoDB; +# restart: --innodb-directories=MYSQL_TMP_DIR CREATE TABLE tab (a INT, b INT) ENGINE = InnoDB ROW_FORMAT=DYNAMIC PARTITION BY RANGE(a) SUBPARTITION BY KEY(b) ( PARTITION p1 VALUES LESS THAN (100) TABLESPACE=ts1, diff --git a/mysql-test/suite/auth_sec/r/key_rotation_qa.result b/mysql-test/suite/auth_sec/r/key_rotation_qa.result index a07a249eed6e..6301ed2ecd65 100644 --- a/mysql-test/suite/auth_sec/r/key_rotation_qa.result +++ b/mysql-test/suite/auth_sec/r/key_rotation_qa.result @@ -1,22 +1,3 @@ -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Encryption can't find master key, please check the keyring plugin is loaded."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Plugin keyring_file reported: 'keyring_file initialization failure."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Function 'keyring_file' already exists"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Couldn't load plugin named 'keyring_file' with soname 'keyring_file.so'."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Couldn't load plugin named 'keyring_file' with soname 'keyring_file.dll'."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Can't generate new master key, please check the keyring plugin is loaded."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Encryption information in datafile"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number 2 in a file operation."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the system cannot find the path specified."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Could not find a valid tablespace file for"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Can't generate new master key for tablespace encryption, please check the keyring plugin is loaded."); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace `test/.*` because it could not be opened."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Failed to find tablespace for table"); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Can't read encryption key from file"); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot open table .* from the internal data dictionary of InnoDB though the .frm file for the table exists.*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Table .* in the InnoDB data dictionary has tablespace id .*, but tablespace with that id or name does not exist"); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Please refer to .* for how to resolve the issue"); -call mtr.add_suppression("Could not flush keys to keyring's backup"); ALTER INSTANCE ROTATE MYISAM MASTER KEY; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'MYISAM MASTER KEY' at line 1 ALTER INSTANCE ROTATE INNODB; diff --git a/mysql-test/suite/auth_sec/t/key_rotation_qa.test b/mysql-test/suite/auth_sec/t/key_rotation_qa.test index 6ea88ac37223..13e29df5777a 100644 --- a/mysql-test/suite/auth_sec/t/key_rotation_qa.test +++ b/mysql-test/suite/auth_sec/t/key_rotation_qa.test @@ -4,6 +4,7 @@ # and key rotation testing. --source include/no_valgrind_without_big.inc +--disable_query_log call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Encryption can't find master key, please check the keyring plugin is loaded."); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Plugin keyring_file reported: 'keyring_file initialization failure."); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Function 'keyring_file' already exists"); @@ -23,6 +24,8 @@ call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot open table .* call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Table .* in the InnoDB data dictionary has tablespace id .*, but tablespace with that id or name does not exist"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Please refer to .* for how to resolve the issue"); call mtr.add_suppression("Could not flush keys to keyring's backup"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Tablespace [0-9]+, name 'test.t[0-9]+', unable to open file '.*test.t[0-9]+.ibd' - Data structure corruption"); +--enable_query_log # Invalid syntax --error ER_PARSE_ERROR diff --git a/mysql-test/suite/innodb/include/load_tsof_data.inc b/mysql-test/suite/innodb/include/load_tsof_data.inc deleted file mode 100644 index 5a4859280d15..000000000000 --- a/mysql-test/suite/innodb/include/load_tsof_data.inc +++ /dev/null @@ -1,180 +0,0 @@ -# suite/ddl/include/load_tsof_data.inc -# - -# $when_taken must sit right - ---enable_abort_on_error - ---disable_query_log -if($script_debug) -{ - --enable_query_log -} - -# Add the new state to the bookkeeping files - -# Result of SELECT MD5(...) -# ------------------------- -# SELECT MD5(NULL) : NULL -# SELECT MD5('') : d41d8cd98f00b204e9800998ecf8427e -# SELECT MD5(LOAD_FILE()) variations -# secure_file_priv sits right -# - file is readable by all and zero bytes -# long : d41d8cd98f00b204e9800998ecf8427e -# - file is not readable by all : NULL -# Observed under Linux where some additional tablespaces.open.4 was created -# by InnoDB with "-rw-rw----" whereas the already existing were with -# "-rw-r--r--" from whatever reason (MTR during server setup?). -# - file does not exist : NULL -# secure_file_priv sits wrong : NULL (no warning!) -# So in case we take care that -# - secure_file_priv sits right -# - any tablespaces.open.* gets treated with chmod 644 -# than any NULL implies some not existing tablespaces.open.. -# -let $TSOF_DIR= $MYSQLD_DATADIR; -# if($when_taken == $point_postcrash) -# { - # let $TSOF_DIR= $MYSQLTEST_VARDIR/tsof_copies/; -# } -let $max_num= 5; -let $num= 1; -let $null_reached= 0; -while ($num <= $max_num) -{ - let $tsof_name= tablespaces.open.$num; - let $tsof_path= $TSOF_DIR$tsof_name; - - # Fore debugging the script. - if ($script_debug) - { - --echo $tsof_path - # 1. The --error 0,1,2 is because the file $tsof_name might not exist. - # This is to be expected for the higher numbers. - # 2. The "cd $MYSQLD_DATADIR;" allows to use $tsof_name and that causes - # that the output is shorter and free of the local box specific - # setting of $MYSQLD_DATADIR. Just imagine the "ls -ld " output for - # tablespaces.open.4 - # versus - # /work/mysql-trunk/mysql-test/var/mysqld.1/data/tablespaces.open.4 - # 3. "--exec ...." and not "exec ...;" is required because - # - we have two commands, "cd $TSOF_DIR" and "ls -ld $tsof_name" which - # need to be separated by a ';' in order to work proper in the OS. - # - In case of "exec" without a preceding "--" mysqltest assumes that - # only the line end marks the end of the OS command sequence. - # Some additional ';' at end would be interpreted as "empty" OS cmd. - --error 0,1,2 - --exec cd $TSOF_DIR; ls -ld $tsof_name - --error 0,1,2,3 - --exec cd $TSOF_DIR; md5sum $tsof_name - } - --error 0,1,2 - chmod 0644 $tsof_path; - eval SET @md5_value = MD5(LOAD_FILE('$tsof_path')); - let $md5_is_null= `SELECT @md5_value IS NULL`; - if(!$md5_is_null) - { - # For testing the current script - # let $null_reached= 1; - if ($null_reached) - { - # There was a tablespaces_open file with lower number which did not - # exist. Finding now some existing tablespaces_open file with higher - # number means that we have a hole in numbering which should not exist - # at all. - --echo # ERROR: Hole in the numbering of tablespaces_open files. - --list_files $TSOF_DIR tablespaces.open.* - --echo # Abort - exit; - } - # Always store $tsof_name and never $tsof_path. - # The first is "free" of $MYSQLD_DATADIR which - # - is testing box specific - # - increases the length of the string without giving something valuable. - eval - INSERT INTO tablespaces_open_content (object, when_taken, md5_value) - SELECT CONCAT('./', '$tsof_name'), $when_taken, @md5_value; - } - if ($md5_is_null) - { - # No INSERT because the file $tsof_path did not exist. - let $null_reached= 1; - } - inc $num; -} -if (!$md5_is_null) -{ - # Being here implies that - # - there was no abort of the test because of error - # - $max_num files were looked at - # - @md5_value belongs to tablespaces.open.<$max_num> - # In case @md5_value IS NOT NULL than the file tablespaces.open.10 existed, - # $max_num needs to be increased. - --echo # ERROR: $the_file with highest number tried exists. - --echo # collect_tsof_data.inc must be adjusted (increase max_num). - --echo # Abort - exit; -} - - -# For testing the scripts especially the next check. -if(0) -{ - --enable_query_log - --echo # ! error injection for script debugging ! - --echo # So some abort of test because of error might show up. - DELETE FROM tablespaces_open_content; -} -if (`SELECT COUNT(*) <> 0 FROM tsof_dropped`) -{ - --echo # ERROR: One or more tablespaces.open.* files were dropped. - SELECT * FROM tsof_dropped ORDER BY object; - SELECT when_taken, object FROM tablespaces_open_content - ORDER BY object, when_taken; - SELECT object FROM tablespaces_open_files - ORDER BY object; - --echo # Abort - exit; -} - -if(`SELECT COUNT(*) <> 0 FROM tsof_added`) -{ - # Case: Some additional tablespaces_open file showed up. - # ------------------------------------------------------ - # 1. Reset counter2 to 0 for the already known tablespaces_open files. - # =~ Declare a restart of secondary measurement queue between previous - # and current collection of data. - UPDATE tablespaces_open_files SET counter2 = 0; - # 2. Add the new tablespaces_open files to tablespaces_open_files with - # counter1 = 1 - # == Update the first measurement queue because that file was used. - # counter2 = 1 - # == The file was used after the restart of the secondary measurement - # queue. - INSERT INTO tablespaces_open_files (object, counter1, counter2) - SELECT object, 1, 1 FROM tsof_added; -} - -if($when_taken == $point_after) -{ - # Case: Some tablespaces_open file was changed. - # --------------------------------------------- - # Increment counter1 and counter2 for such tablespaces_open files. - # Charging for : 'before' ==>> 'after' - UPDATE tablespaces_open_files - SET counter1 = counter1 + 1, counter2 = counter2 + 1 - WHERE object IN (SELECT object FROM tablespaces_open_content_diff_a_b - WHERE state = 'changed'); -} - ---enable_query_log - -#### For testing the scripts -if($script_debug) -{ - --echo # INFO: when_taken : $when_taken - SELECT * FROM tablespaces_open_content ORDER BY object, when_taken; - SELECT * FROM tablespaces_open_content_diff_a_b ORDER BY state, object; - SELECT * FROM tablespaces_open_files ORDER BY object; -} - diff --git a/mysql-test/suite/innodb/r/alter_kill.result b/mysql-test/suite/innodb/r/alter_kill.result index 5d2e90208903..898510b8e688 100644 --- a/mysql-test/suite/innodb/r/alter_kill.result +++ b/mysql-test/suite/innodb/r/alter_kill.result @@ -63,7 +63,7 @@ XA END 'x'; XA PREPARE 'x'; # Kill the server # Attempt to start without an *.ibd file. -Pattern "\[ERROR\] \[[^]]*\] InnoDB:.*tablespace \d+ .*not found:.*test.bug16735660.ibd.*" found +Pattern "\[ERROR\].*InnoDB: Could not find any file associated with the tablespace ID:.*" found # restart SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED; SELECT * FROM bug16735660; diff --git a/mysql-test/suite/innodb/r/create_tablespace.result b/mysql-test/suite/innodb/r/create_tablespace.result index c80540952922..fc981e78cea3 100644 --- a/mysql-test/suite/innodb/r/create_tablespace.result +++ b/mysql-test/suite/innodb/r/create_tablespace.result @@ -3,6 +3,10 @@ # SET DEFAULT_STORAGE_ENGINE=InnoDB; SET NAMES utf8; +# restart:--innodb-directories=MYSQL_TMP_DIR +SHOW VARIABLES LIKE 'innodb_directories'; +Variable_name Value +innodb_directories MYSQL_TMP_DIR # Strict-mode has no effect on CREATE TABLESPACE. # It rejects all invalid input, as if strict mode is always ON. SHOW VARIABLES LIKE 'innodb_strict_mode'; @@ -43,10 +47,14 @@ CREATE TABLESPACE test/s_bad ADD DATAFILE 's_bad.ibd'; ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '/s_bad ADD DATAFILE 's_bad.ibd'' at line 1 CREATE TABLESPACE `test/s_bad` ADD DATAFILE 's_bad.ibd'; ERROR 42000: InnoDB: A general tablespace name cannot contain '/'. -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'delete' returned OS error .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'stat' returned OS error .*"); CREATE TABLESPACE `s_too_long_file_name` ADD DATAFILE '../xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ibd'; -ERROR HY000: Failed to create TABLESPACE s_too_long_file_name +ERROR HY000: Incorrect File Name '../xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ibd'. +SHOW WARNINGS; +Level Code Message +Error 3121 Incorrect File Name '../xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ibd'. +Error 3121 CREATE TABLESPACE data file must be in one of these directories './;MYSQL_TMP_DIR/'. +Error 1528 Failed to create TABLESPACE s_too_long_file_name +Error 1031 Table storage engine for 's_too_long_file_name' doesn't have this option CREATE TABLESPACE s_bad ADD DATAFILE 's_bad.ibd' FILE_BLOCK_SIZE=1k FILE_BLOCK_SIZE=2k; ERROR HY000: It is not allowed to specify FILE_BLOCK_SIZE more than once CREATE TABLESPACE s_bad ADD DATAFILE 's_bad.ibd' FILE_BLOCK_SIZE=3k; @@ -274,10 +282,9 @@ Error 3121 Incorrect File Name 's_dir/s_subdir/.ibd'. Error 1528 Failed to create TABLESPACE s_bad Error 1031 Table storage engine for 's_bad' doesn't have this option CREATE TABLESPACE s_bad ADD DATAFILE 's_dir/s_bad.ibs'; -ERROR HY000: Incorrect File Name 's_dir/s_bad.ibs'. +ERROR HY000: An IBD filepath must end with `.ibd`. SHOW WARNINGS; Level Code Message -Error 3121 Incorrect File Name 's_dir/s_bad.ibs'. Error 3121 An IBD filepath must end with `.ibd`. Error 1528 Failed to create TABLESPACE s_bad Error 1031 Table storage engine for 's_bad' doesn't have this option @@ -1642,7 +1649,7 @@ SET GLOBAL innodb_strict_mode=default; # # Check CREATE TABLESPACE is handled properly in read-only mode CREATE TABLESPACE s1 ADD DATAFILE 's1.ibd' ENGINE InnoDB; -# restart: --innodb_read_only +# restart: --innodb_read_only --innodb-directories=MYSQL_TMP_DIR CREATE TABLESPACE s2 ADD DATAFILE 's2.ibd' ENGINE InnoDB; ERROR HY000: Table 'tablespaces' is read only SHOW WARNINGS; @@ -1656,7 +1663,7 @@ Level Code Message Error 1529 Failed to drop TABLESPACE s1 Error 1874 InnoDB is in read only mode. # Restart in normal mode for cleanup. -# restart +# restart --innodb-directories=MYSQL_TMP_DIR DROP TABLESPACE s1; # # Bug#26199233 create like ignores innodb_file_per_table diff --git a/mysql-test/suite/innodb/r/create_tablespace_8k.result b/mysql-test/suite/innodb/r/create_tablespace_8k.result index f6c10b0f5059..cdd13cf53b8a 100644 --- a/mysql-test/suite/innodb/r/create_tablespace_8k.result +++ b/mysql-test/suite/innodb/r/create_tablespace_8k.result @@ -867,6 +867,8 @@ test/t_zip4k_to_file_per_table test/t_zip4k_to_file_per_table 39 5 Compressed 40 DROP TABLE `t_zip1k_to_file_per_table`; DROP TABLE `t_zip2k_to_file_per_table`; DROP TABLE `t_zip4k_to_file_per_table`; +call mtr.add_suppression("\\[ERROR\\].*InnoDB: '.*s_.k.ibd' read failed! - attempted to read 4 bytes, read only 0 bytes at offset 8226"); +call mtr.add_suppression("\\[Warning\\].*InnoDB: Ignoring '.*s_.k.ibd' invalid tablespace ID in the header"); # # Clean-up. # diff --git a/mysql-test/suite/innodb/r/innodb-multiple-tablespaces.result b/mysql-test/suite/innodb/r/innodb-multiple-tablespaces.result deleted file mode 100644 index 6205e805a8de..000000000000 --- a/mysql-test/suite/innodb/r/innodb-multiple-tablespaces.result +++ /dev/null @@ -1,461 +0,0 @@ -SET default_storage_engine=InnoDB; -# -# Test when tablespaces can be found at multiple places -# SYS_DATAFILES will refer to the file at alt_dir. -# Link File will refer to the file at new_dir. -# Tablename Default SYS_DATAFILES Link_File -# yyy Yes Yes Yes -# nyy No Yes Yes -# yny Yes No Yes -# yyn Yes Yes No -# nyw No Yes WrongFile -# nwy No WrongFile Yes -# wny WrongFile No Yes -# ynw Yes No WrongFile -# wyn WrongFile Yes No -# ywn Yes WrongFile No -# nnn No No No -# www WrongFile WrongFile WrongFile -# -CREATE TABLE yyy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO yyy VALUES (1, 'yyy'); -CREATE TABLE nyy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO nyy VALUES (1, 'nyy'); -CREATE TABLE yny (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO yny VALUES (1, 'yny'); -CREATE TABLE yyn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO yyn VALUES (1, 'yyn'); -CREATE TABLE nyw (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO nyw VALUES (1, 'nyw'); -CREATE TABLE nwy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO nwy VALUES (1, 'nwy'); -CREATE TABLE wny (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO wny VALUES (1, 'wny'); -CREATE TABLE ynw (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO ynw VALUES (1, 'ynw'); -CREATE TABLE wyn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO wyn VALUES (1, 'wyn'); -CREATE TABLE ywn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO ywn VALUES (1, 'ywn'); -CREATE TABLE nnn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO nnn VALUES (1, 'nnn'); -CREATE TABLE www (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO www VALUES (1, 'www'); -CREATE TABLE nolink (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO nolink VALUES (1, 'no link file'); -# -# Shutdown the server, copy and remove files. -# ----- MYSQLD_DATADIR/test ----- MYSQL_TMP_DIR/alt_dir/test -nnn.ibd -nolink.ibd -nwy.ibd -nyw.ibd -nyy.ibd -wny.ibd -www.ibd -wyn.ibd -ynw.ibd -yny.ibd -ywn.ibd -yyn.ibd -yyy.ibd ----- MYSQL_TMP_DIR/new_dir/test -# YYY; Tablespace found in 3 places -# NYY; Tablespace found in alt_dir and new_dir -# YNY; Tablespace found in default and new_dir -# YYN; Tablespace found in default and alt_dir -# NYW; Copy the wrong file to new_dir -# NWY; Move the wrong file to alt_dir, good one to new_dir. -# WNY; Move the wrong file to default, good one to new_dir, delete it form alt_dir -# YNW; Move the file to default, wrong one to new_dir, delete it form alt_dir -# WYN; Copy the wrong file to default -# YWN; Move the file to default, wrong one to alt_dir -# NNN; Delete the tablespace -# WWW; Put the wrong file in all three locations -# Make a backup of this tablespace to use later. ----- MYSQLD_DATADIR/test -wny.ibd -wyn.ibd -ynw.ibd -yny.ibd -ywn.ibd -yyn.ibd -yyy.ibd ----- MYSQL_TMP_DIR/alt_dir/test -nolink.ibd -nolink.ibd.bak -nyw.ibd -nyy.ibd -wyn.ibd -yyn.ibd -yyy.ibd ----- MYSQL_TMP_DIR/new_dir/test -nwy.ibd -nyw.ibd -nyy.ibd -wny.ibd -ynw.ibd -yny.ibd -yyy.ibd -# -# Start the server and show the tablespaces. -# -# restart -=== information_schema.innodb_tablespaces and innodb_datafiles === -Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/yyy Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/yyy.ibd -test/nyy Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/nyy.ibd -test/yny Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/yny.ibd -test/yyn Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/yyn.ibd -test/nyw Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/nyw.ibd -test/nwy Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/nwy.ibd -test/wny Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/wny.ibd -test/ynw Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/ynw.ibd -test/wyn Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/wyn.ibd -test/ywn Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/ywn.ibd -test/nnn Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/nnn.ibd -test/www Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/www.ibd -test/nolink Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/nolink.ibd -=== information_schema.files === -Space_Name File_Type Engine Status Tablespace_Name Path -test/yyy TABLESPACE InnoDB NORMAL test/yyy MYSQL_TMP_DIR/alt_dir/test/yyy.ibd -test/nyy TABLESPACE InnoDB NORMAL test/nyy MYSQL_TMP_DIR/alt_dir/test/nyy.ibd -test/yyn TABLESPACE InnoDB NORMAL test/yyn MYSQL_TMP_DIR/alt_dir/test/yyn.ibd -test/nyw TABLESPACE InnoDB NORMAL test/nyw MYSQL_TMP_DIR/alt_dir/test/nyw.ibd -test/wyn TABLESPACE InnoDB NORMAL test/wyn MYSQL_TMP_DIR/alt_dir/test/wyn.ibd -test/nolink TABLESPACE InnoDB NORMAL test/nolink MYSQL_TMP_DIR/alt_dir/test/nolink.ibd -Warnings: -Warning 1812 Tablespace is missing for table test/yny. -Warning 1812 Tablespace is missing for table test/nwy. -Warning 1812 Tablespace is missing for table test/wny. -Warning 1812 Tablespace is missing for table test/ynw. -Warning 1812 Tablespace is missing for table test/ywn. -Warning 1812 Tablespace is missing for table test/nnn. -Warning 1812 Tablespace is missing for table test/www. -SELECT * FROM yyy; -c1 c2 -1 yyy -SELECT * FROM nyy; -c1 c2 -1 nyy -SELECT * FROM yny; -ERROR HY000: Tablespace is missing for table `test`.`yny`. -SELECT * FROM yyn; -c1 c2 -1 yyn -SELECT * FROM nyw; -c1 c2 -1 nyw -SELECT * FROM nwy; -ERROR HY000: Tablespace is missing for table `test`.`nwy`. -SELECT * FROM wny; -ERROR HY000: Tablespace is missing for table `test`.`wny`. -SELECT * FROM ynw; -ERROR HY000: Tablespace is missing for table `test`.`ynw`. -SELECT * FROM wyn; -c1 c2 -1 wyn -SELECT * FROM ywn; -ERROR HY000: Tablespace is missing for table `test`.`ywn`. -SELECT * FROM nnn; -ERROR HY000: Tablespace is missing for table `test`.`nnn`. -SELECT * FROM www; -ERROR HY000: Tablespace is missing for table `test`.`www`. -SELECT * FROM nolink; -c1 c2 -1 no link file -SHOW CREATE TABLE yyy; -Table Create Table -yyy CREATE TABLE `yyy` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE nyy; -Table Create Table -nyy CREATE TABLE `nyy` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE yny; -ERROR HY000: Tablespace is missing for table `test`.`yny`. -SHOW CREATE TABLE yyn; -Table Create Table -yyn CREATE TABLE `yyn` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE nyw; -Table Create Table -nyw CREATE TABLE `nyw` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE nwy; -ERROR HY000: Tablespace is missing for table `test`.`nwy`. -SHOW CREATE TABLE wny; -ERROR HY000: Tablespace is missing for table `test`.`wny`. -SHOW CREATE TABLE ynw; -ERROR HY000: Tablespace is missing for table `test`.`ynw`. -SHOW CREATE TABLE wyn; -Table Create Table -wyn CREATE TABLE `wyn` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE ywn; -ERROR HY000: Tablespace is missing for table `test`.`ywn`. -SHOW CREATE TABLE nnn; -ERROR HY000: Tablespace is missing for table `test`.`nnn`. -SHOW CREATE TABLE www; -ERROR HY000: Tablespace is missing for table `test`.`www`. -SHOW CREATE TABLE nolink; -Table Create Table -nolink CREATE TABLE `nolink` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -# -# List of files before DROP TABLES -# ----- MYSQLD_DATADIR/test -wny.ibd -wyn.ibd -ynw.ibd -yny.ibd -ywn.ibd -yyn.ibd -yyy.ibd ----- MYSQL_TMP_DIR/alt_dir/test -nolink.ibd -nolink.ibd.bak -nyw.ibd -nyy.ibd -wyn.ibd -yyn.ibd -yyy.ibd ----- MYSQL_TMP_DIR/new_dir/test -nwy.ibd -nyw.ibd -nyy.ibd -wny.ibd -ynw.ibd -yny.ibd -yyy.ibd -# -# Restart the server and DROP the tablespaces. -# -# restart -DROP TABLE yyy; -DROP TABLE nyy; -DROP TABLE yny; -DROP TABLE yyn; -DROP TABLE nyw; -DROP TABLE nwy; -DROP TABLE wny; -DROP TABLE ynw; -DROP TABLE wyn; -DROP TABLE ywn; -DROP TABLE nnn; -DROP TABLE www; -DROP TABLE nolink; -# -# List of files not deleted by the DROP TABLES -# ----- MYSQLD_DATADIR/test -wny.ibd -wyn.ibd -ynw.ibd -yny.ibd -ywn.ibd -yyn.ibd -yyy.ibd ----- MYSQL_TMP_DIR/alt_dir/test -nolink.ibd.bak ----- MYSQL_TMP_DIR/new_dir/test -nwy.ibd -nyw.ibd -nyy.ibd -wny.ibd -ynw.ibd -yny.ibd -yyy.ibd -# -# List of files after removing leftover files -# ----- MYSQLD_DATADIR/test -ynw.ibd -ywn.ibd ----- MYSQL_TMP_DIR/alt_dir/test -nolink.ibd.bak ----- MYSQL_TMP_DIR/new_dir/test -# -# Create some tables again and this time, crash instead of shutdown -# -# Test recovery when tablespaces can be found at multiple places. -# The data dictionary is unavailable during recovery. -# 'nolink.ibd.bak' is the source of the 'wrong' tablespaces. -# -# Tablename Default_Tablespace Remote_Tablespace -# ny No Yes -# wy Wrong Yes -# yn Yes No -# yw Yes Wrong -# yy Yes Yes (both the same file) -# -CREATE TABLE ny (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO ny VALUES (1, 'ny'); -CREATE TABLE wy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO wy VALUES (1, 'wy'); -CREATE TABLE yn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO yn VALUES (1, 'yn'); -CREATE TABLE yw (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO yw VALUES (1, 'yw'); -CREATE TABLE yy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; -INSERT INTO yy VALUES (1, 'yy'); -# -# Crash the server, copy and remove files. -# -SET GLOBAL innodb_master_thread_disabled_debug=1; -BEGIN; -INSERT INTO ny VALUES (2, 'ny'); -INSERT INTO wy VALUES (2, 'wy'); -INSERT INTO yn VALUES (2, 'yn'); -INSERT INTO yw VALUES (2, 'yw'); -INSERT INTO yy VALUES (2, 'yy'); -SELECT * FROM ny; -c1 c2 -1 ny -2 ny -SELECT * FROM wy; -c1 c2 -1 wy -2 wy -SELECT * FROM yn; -c1 c2 -1 yn -2 yn -SELECT * FROM yw; -c1 c2 -1 yw -2 yw -SELECT * FROM yy; -c1 c2 -1 yy -2 yy -# Kill the server -# -# Now that the engine is not running, move files around to test various scenarios. -# -# NY; Tablespace found in alt_dir but not the default directory. -# WY; The wrong tablespace is found in the default directory -# and the correct one in alt_dir. -# YW; Tablespace is found in the default directory but the wrong file in alt_dir. -# YN; Tablespace found the default directory but not in alt_dir. -# YY; Found in both default directory and alt-dir. ----- MYSQLD_DATADIR/test -wy.ibd -yn.ibd -ynw.ibd -yw.ibd -ywn.ibd -yy.ibd ----- MYSQL_TMP_DIR/alt_dir/test -nolink.ibd.bak -ny.ibd -wy.ibd -yw.ibd -yy.ibd -Pattern "InnoDB: Could not find any file associated with the tablespace \d+" found -Pattern "InnoDB: Tablespace file '.*.tmp.*alt_dir.test.yw.ibd' ID mismatch, expected \d+ but found \d+" found -# restoring yn.ibd, yw.ibd -# restart -SELECT * FROM ny; -c1 c2 -1 ny -SELECT * FROM wy; -c1 c2 -1 wy -SELECT * FROM yn; -c1 c2 -1 yn -SELECT * FROM yw; -c1 c2 -1 yw -SELECT * FROM yy; -c1 c2 -1 yy -SHOW CREATE TABLE ny; -Table Create Table -ny CREATE TABLE `ny` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE wy; -Table Create Table -wy CREATE TABLE `wy` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE yn; -Table Create Table -yn CREATE TABLE `yn` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE yw; -Table Create Table -yw CREATE TABLE `yw` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -SHOW CREATE TABLE yy; -Table Create Table -yy CREATE TABLE `yy` ( - `c1` int(11) NOT NULL, - `c2` text, - PRIMARY KEY (`c1`) -) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir/' -=== information_schema.innodb_tablespaces and innodb_datafiles === -Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/ny Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/ny.ibd -test/wy Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/wy.ibd -test/yn Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/yn.ibd -test/yw Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/yw.ibd -test/yy Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/yy.ibd -=== information_schema.files === -Space_Name File_Type Engine Status Tablespace_Name Path -test/ny TABLESPACE InnoDB NORMAL test/ny MYSQL_TMP_DIR/alt_dir/test/ny.ibd -test/wy TABLESPACE InnoDB NORMAL test/wy MYSQL_TMP_DIR/alt_dir/test/wy.ibd -test/yn TABLESPACE InnoDB NORMAL test/yn MYSQL_TMP_DIR/alt_dir/test/yn.ibd -test/yw TABLESPACE InnoDB NORMAL test/yw MYSQL_TMP_DIR/alt_dir/test/yw.ibd -test/yy TABLESPACE InnoDB NORMAL test/yy MYSQL_TMP_DIR/alt_dir/test/yy.ibd -# -# List of files before removing unused files -# -DROP TABLE ny; -DROP TABLE wy; -DROP TABLE yn; -DROP TABLE yw; -DROP TABLE yy; -# -# List of files after DROP TABLES -# ----- MYSQLD_DATADIR/test -wy.ibd -ynw.ibd -ywn.ibd -yy.ibd ----- MYSQL_TMP_DIR/alt_dir/test -nolink.ibd.bak diff --git a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result index 297b69e22b08..832bc343e988 100644 --- a/mysql-test/suite/innodb/r/innodb-wl5522-debug.result +++ b/mysql-test/suite/innodb/r/innodb-wl5522-debug.result @@ -45,6 +45,8 @@ SELECT * FROM test_wl5522.t1; ERROR HY000: Tablespace has been discarded for table 't1' ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE; ERROR HY000: Lost connection to MySQL server during query +unlink: t1.ibd +unlink: t1.cfg SET SESSION debug="-d,ib_import_before_commit_crash"; SET SESSION debug="+d,ib_import_before_checkpoint_crash"; SELECT COUNT(*) FROM test_wl5522.t1; diff --git a/mysql-test/suite/innodb/r/innodb_tsof_use_1.result b/mysql-test/suite/innodb/r/innodb_tsof_use_1.result deleted file mode 100644 index 089a2f63be6b..000000000000 --- a/mysql-test/suite/innodb/r/innodb_tsof_use_1.result +++ /dev/null @@ -1,274 +0,0 @@ -### Build infrastructure required for the current test ------------ START -SET SQL_MODE = 'traditional'; -SET GLOBAL innodb_log_checkpoint_now = ON; -CREATE SCHEMA db_test_a; -CREATE TABLESPACE ts1 ADD DATAFILE 'ts1.ibd' Engine = InnoDB; -CREATE TABLE db_test_a.t_aux (col1 INT, col2 VARCHAR(100)) Engine = InnoDB; -CREATE TABLE db_test_a.t_aux_10 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 10; -CREATE TABLE db_test_a.t_aux_9 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 9; -CREATE TABLE db_test_a.t_aux_8 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 8; -CREATE TABLE db_test_a.t_aux_7 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 7; -CREATE TABLE db_test_a.t_aux_6 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 6; -CREATE TABLE db_test_a.t_aux_5 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 5; -CREATE TABLE db_test_a.t_aux_4 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 4; -CREATE TABLE db_test_a.t_aux_3 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 3; -CREATE TABLE db_test_a.t_aux_2 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 2; -CREATE TABLE db_test_a.t_aux_1 (col1 INT) -Engine = InnoDB TABLESPACE = ts1; -INSERT INTO db_test_a.t_aux -SET col2 = CONCAT('0-', REPEAT('z', 90)), col1 = 1; -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -CALL mtr.add_suppression("InnoDB: File '.*tablespaces.open..*' size is .* Must be at least"); -CALL mtr.add_suppression("InnoDB: File '.*tablespaces.open..*' size is .* should be at least"); -CALL mtr.add_suppression("InnoDB: No space ID to filename mapping file found"); -CALL mtr.add_suppression("InnoDB: Cannot load table db_test_a"); -# Make backups of the existing tablespaces.open.* files. -# We will use these files later. -# Generate entries in tablespaces.open.* files and also log by running DDL+DML. The printing of statements is disabled. -# restart -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `innodb_file_per_table` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -### Build infrastructure required for the current test -------------- END -# ======================================================== -# Restart after previous smooth shutdown. -# There is no valid tablespaces.open.* -# If we find one we use that to verify the location of the UNDO -# tablespaces. If we don't find one we don't verify the location -# of the undo tablespaces. -# The server just comes up without error messages mentioning that state. -# No crash -> no look for tablespaces.open.* files -# restart -Pattern ".* \[Note\] InnoDB: Starting crash recovery." not found -Pattern ".* \[Note\] InnoDB: Unable to read from '.*tablespaces.open.*', the space ID to filename mapping file" found -Pattern ".* \[ERROR\] InnoDB: No space ID to filename mapping file found" found -# ======================================================== -# Restart after previous -# 1. Have no open transaction -# 2. Enforce making an InnnoDB checkpoint -# 3. Kill of server -# There is no valid tablespaces.open.* file at all but we need none. -# The server just comes up without error messages mentioning that state. -# The impact of crash is so minimal that InnoDB does detect it at all. -# -> no look for tablespaces.open.* files -SET GLOBAL innodb_log_checkpoint_now = ON; -# Kill the server -# restart -Pattern ".* \[Note\] InnoDB: Starting crash recovery." not found -Pattern ".* \[Note\] InnoDB: Unable to read from '.*tablespaces.open.*', the space ID to filename mapping file" found -Pattern ".* \[ERROR\] InnoDB: No space ID to filename mapping file found" found -# ======================================================== -# Restart with crash recovery -# There is no tablespaces.open.* file but we need at least one valid. -# A simple restart without extra options fails. -SET GLOBAL innodb_checkpoint_disabled = 1; -# Generate entries in tablespaces.open.* files and also log by running DDL+DML. The printing of statements is disabled. -# Kill the server -# The restart attempt has failed like expected. -Pattern ".* \[Note\] InnoDB: Starting crash recovery." found -Pattern ".* \[Note\] InnoDB: Unable to read from '.*tablespaces.open.*', the space ID to filename mapping file" found -Pattern ".* \[ERROR\] InnoDB: No space ID to filename mapping file found" found -# restart:--innodb-scan-directories= -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -# ======================================================== -# Restart with crash recovery -# There is no tablespaces.open.* file but a restart with -# "--innodb-scan-directories=" -# reports that bad state and passes because it does not rely -# on at least one valid tablespaces.open.* file. -SET GLOBAL innodb_log_checkpoint_now = ON; -SET GLOBAL innodb_checkpoint_disabled = 1; -# Generate entries in tablespaces.open.* files and also log by running DDL+DML. The printing of statements is disabled. -# Kill the server -# restart:--innodb-scan-directories= -Pattern ".* \[Note\] InnoDB: Starting crash recovery." found -Pattern ".* \[Note\] InnoDB: Unable to read from '.*tablespaces.open.*', the space ID to filename mapping file" found -Pattern ".* \[ERROR\] InnoDB: No space ID to filename mapping file found" found -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `innodb_file_per_table` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -# ======================================================== -# Restart with crash recovery -# One of the tablespaces.open.* with complete rotten entry. -SET GLOBAL innodb_log_checkpoint_now = ON; -SET GLOBAL innodb_checkpoint_disabled = 1; -# Generate entries in tablespaces.open.* files and also log by running DDL+DML. The printing of statements is disabled. -CREATE TABLE db_test_a . t1 (col1 INT NOT NULL) -ENGINE = 'InnoDB' TABLESPACE = innodb_file_per_table; -SET SESSION DEBUG = "+d,ib_tablespace_open_write_corrupt_0"; -ALTER TABLE db_test_a . t1 TABLESPACE = ts1; -ERROR HY000: Lost connection to MySQL server during query -# The restart attempt has failed like expected. -# So even one valid '.*tablespaces.open.*' is not sufficient for success. -Pattern ".* \[Note\] InnoDB: Starting crash recovery." not found -Pattern ".* \[ERROR\] InnoDB: Unsupported file format .* found in tablespace ID to filename mapping file: '.*tablespaces.open.*'. You can use --innodb-scan-directories to recover .*" found -Pattern ".* \[ERROR\] \[FATAL\] InnoDB: Unable to read the space ID to filename mapping file\(s\)." found -Pattern ".* \[ERROR\] InnoDB: Assertion failure: ut0ut.cc:" found -# The crash recovery fails even if -# "--innodb-scan-directories=" -# is assigned as long as the tablespaces.open.* file with complete -# rotten entry exists. So we delete all tablespaces.open.*. -# restart:--innodb-scan-directories= -DROP TABLE db_test_a . t1; -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -# ======================================================== -# Restart with crash recovery -# One of the tablespaces.open.* with incomplete header -# Generate entries in tablespaces.open.* files and also log by running DDL+DML. The printing of statements is disabled. -CREATE TABLE db_test_a . t1 (col1 INT NOT NULL) -ENGINE = 'InnoDB' TABLESPACE = innodb_file_per_table; -SET SESSION DEBUG = "+d,ib_tablespace_open_write_corrupt_1"; -ALTER TABLE db_test_a . t1 TABLESPACE = ts1; -ERROR HY000: Lost connection to MySQL server during query -# restart -# InnoDB warning about too small tablespaces.open.* file found in -# server error log. -SHOW CREATE TABLE db_test_a . t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `col1` int(11) NOT NULL -) /*!50100 TABLESPACE `innodb_file_per_table` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -DROP TABLE db_test_a . t1; -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `innodb_file_per_table` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -# ======================================================== -# Restart with crash recovery -# One of the tablespaces.open.* with incomplete data part -# Generate entries in tablespaces.open.* files and also log by running DDL+DML. The printing of statements is disabled. -CREATE TABLE db_test_a . t1 (col1 INT NOT NULL) -ENGINE = 'InnoDB' TABLESPACE = innodb_file_per_table; -SET SESSION DEBUG = "+d,ib_tablespace_open_write_corrupt_2"; -ALTER TABLE db_test_a . t1 TABLESPACE = ts1; -ERROR HY000: Lost connection to MySQL server during query -# restart -# InnoDB warning about too small tablespaces.open.* file found in -# server error log. -SHOW CREATE TABLE db_test_a . t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `col1` int(11) NOT NULL -) /*!50100 TABLESPACE `innodb_file_per_table` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -DROP TABLE db_test_a . t1; -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -# ======================================================== -# Restart with crash recovery -# Both tablespaces.open.* files are outdated. -# Generate entries in tablespaces.open.* files and also log by running DDL+DML. The printing of statements is disabled. -CREATE TABLE db_test_a . t1 (col1 INT NOT NULL) -ENGINE = 'InnoDB' TABLESPACE = innodb_file_per_table; -ALTER TABLE db_test_a . t1 TABLESPACE = ts1; -# Kill the server -# restart -SHOW CREATE TABLE db_test_a . t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `col1` int(11) NOT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -DROP TABLE db_test_a . t1; -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `innodb_file_per_table` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -# ======================================================== -# Shutdown smooth -# Delete all tablespaces.open.* files. -# Restart with --innodb-scan-directories -# restart:--innodb-scan-directories= -# Generate entries in tablespaces.open.* files and also log by running DDL+DML. The printing of statements is disabled. -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -# We should have now one tablespaces.open.* file without defects. -# Let's try the server crash just before tablespaces.open.* write. -CREATE TABLE db_test_a . t1 (col1 INT NOT NULL) -ENGINE = 'InnoDB' TABLESPACE = innodb_file_per_table; -SET SESSION DEBUG = "+d,ib_tablespace_open_crash_before_write"; -ALTER TABLE db_test_a . t1 TABLESPACE = ts1; -ERROR HY000: Lost connection to MySQL server during query -# restart -SHOW CREATE TABLE db_test_a . t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `col1` int(11) NOT NULL -) /*!50100 TABLESPACE `innodb_file_per_table` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -ALTER TABLE db_test_a . t1 TABLESPACE = ts1; -SHOW CREATE TABLE db_test_a . t1; -Table Create Table -t1 CREATE TABLE `t1` ( - `col1` int(11) NOT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -# We should have now two tablespaces.open.* files without defects. -DROP TABLE db_test_a . t1; -SHOW CREATE TABLE db_test_a.t_aux_10; -Table Create Table -t_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 -SELECT COUNT(*) = 9 AS 'expect_1' FROM db_test_a.t_aux -WHERE col2 = CONCAT('8-', REPEAT('z', 90)); -expect_1 -1 -SELECT COUNT(*) = 1 AS 'expect_1' FROM db_test_a.t_aux -WHERE col2 = CONCAT('0-', REPEAT('z', 90)) AND col1 = 1; -expect_1 -1 -SELECT 't_aux_10 CREATE TABLE `t_aux_10` ( - `col1` int(11) DEFAULT NULL -) /*!50100 TABLESPACE `ts1` */ ENGINE=InnoDB DEFAULT CHARSET=utf8mb4' LIKE '%ts1%' AS "expect_1"; -expect_1 -1 -DROP SCHEMA db_test_a; -DROP TABLESPACE ts1; diff --git a/mysql-test/suite/innodb/r/innodb_tsof_use_2.result b/mysql-test/suite/innodb/r/innodb_tsof_use_2.result deleted file mode 100644 index 8cfbb31c7f9f..000000000000 --- a/mysql-test/suite/innodb/r/innodb_tsof_use_2.result +++ /dev/null @@ -1,19 +0,0 @@ -### Build infrastructure typical for the current suite ------------ START -# In this phase printing of statements and responses is disabled. -### Build infrastructure typical for the current suite -------------- END -CREATE SCHEMA db_test_a; -CREATE TABLE db_test_a . t0 (col1 INT) TABLESPACE = innodb_file_per_table -ENGINE = InnoDB; -# Run a big DDL round for the table "db_test_a . t0". -# RENAME TABLE t0 -> t1 -> ... -> -# Caused by using TABLESPACE = innodb_file_per_table tablespaces.open.* -# files need to be used. -# In this phase printing of statements and responses is disabled. -# For all checks : PASS -# 1. Every RENAME causes that the set of tablespaces.open.* files gets -# modified (modify such a file or rather rare add one). -# 2. No tablespaces.open.* file was ever deleted. -# 3. The numbering of tablespaces.open.* files is dense 1,2,... -# 4. The tablespaces.open.* files are used round robin. -DROP TABLE db_test_a . ; -# Cleanup diff --git a/mysql-test/suite/innodb/r/log_alter_table.result b/mysql-test/suite/innodb/r/log_alter_table.result index b5cd50c74370..4f74d96a2392 100644 --- a/mysql-test/suite/innodb/r/log_alter_table.result +++ b/mysql-test/suite/innodb/r/log_alter_table.result @@ -13,7 +13,7 @@ ALTER TABLE t1 ADD PRIMARY KEY(a), ALGORITHM=INPLACE; ALTER TABLE t1 DROP INDEX b, ADD INDEX (b); # Kill the server # restart: --debug=d,ib_log -Pattern "scan .*:.*log rec MLOG_FILE_CREATE2.*page .*:0" found +Pattern "scan .*:.*log rec MLOG_FILE_CREATE.*page .*:0" found Pattern "scan .*:.*log rec MLOG_INDEX_LOAD" found CHECK TABLE t1; Table Op Msg_type Msg_text diff --git a/mysql-test/suite/innodb/r/log_based_discovery.result b/mysql-test/suite/innodb/r/log_based_discovery.result deleted file mode 100644 index 9420097e0ef8..000000000000 --- a/mysql-test/suite/innodb/r/log_based_discovery.result +++ /dev/null @@ -1,42 +0,0 @@ -# create bootstrap file -# Stop the MTR default DB server -# Run the bootstrap command of datadir1 -# Start the DB server with datadir1 -CREATE TABLE tab1(c1 INT, c2 VARCHAR(30)); -INSERT INTO tab1 VALUES(1, 'Test consistency undo*'); -SELECT * FROM tab1; -c1 c2 -1 Test consistency undo* -# Stop the DB server with datadir1 -# Run the bootstrap command of datadir2 -# Start the DB server with datadir2 -CREATE TABLE tab2(c1 INT, c2 VARCHAR(30)); -INSERT INTO tab2 VALUES(1, 'Test consistency undo*'); -SELECT * FROM tab2; -c1 c2 -1 Test consistency undo* -# Stop the DB server with datadir2 -# DB1 server started with the wrong path of undo tablespaces. -# Expect errors during recovery. -Pattern "Cannot create .*undo_001 because .*undo_001 already uses Space ID=4294967279! Did you change innodb_undo_directory" found -# Start the DB server with right path, expect no recovery errors -SELECT @@innodb_undo_tablespaces; -@@innodb_undo_tablespaces -5 -SELECT @@innodb_data_home_dir; -@@innodb_data_home_dir -MYSQL_TMP_DIR/innodb_data_home_dir1 -SELECT @@innodb_undo_directory; -@@innodb_undo_directory -MYSQL_TMP_DIR/innodb_undo_data_dir1 -SELECT * FROM tab1; -c1 c2 -1 Test consistency undo* -DROP TABLE tab1; -# Stop the DB server with datadir1 -# Start the DB server with datadir2 for cleanup -SELECT * FROM tab2; -c1 c2 -1 Test consistency undo* -DROP TABLE tab2; -# restart diff --git a/mysql-test/suite/innodb/r/log_corruption.result b/mysql-test/suite/innodb/r/log_corruption.result index 8ff4edb027d0..14d5ec6c1095 100644 --- a/mysql-test/suite/innodb/r/log_corruption.result +++ b/mysql-test/suite/innodb/r/log_corruption.result @@ -1,37 +1,37 @@ -# redo log from before MySQL 5.7.9 -Pattern "InnoDB: Unsupported redo log format\. The redo log was created before MySQL 5\.7\.9\." found -# redo log from MySQL 5.7.9, with corrupted log block -Pattern "InnoDB: Upgrade after a crash is not supported. This redo log was created with malicious intentions, or perhaps, and it appears corrupted" found -# redo log from "after" MySQL 5.7.9, but with invalid header checksum +# redo log from before we started versioning in MySQL 5.7.9 +Pattern "InnoDB: Unsupported redo log format \(0\). The redo log was created before MySQL 5\.7\.9.*" found +# redo log from before MySQL 8.0.3, with corrupted log block +Pattern "InnoDB: Upgrade after a crash is not supported.*" found +# redo log from MySQL 8.0.3, but with invalid header checksum Pattern "InnoDB: Invalid redo log header checksum" found # distant future redo log format, with valid header checksum -Pattern "InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/.*/en/upgrading-downgrading.html" found -# redo log from MySQL 5.7.9, with corrupted log checkpoint -Pattern "InnoDB: No valid checkpoint found .corrupted redo log" found +Pattern "InnoDB: Unknown redo log format \(4294967295\).*" found +# redo log from MySQL 8.0.3, with corrupted log checkpoint +Pattern "InnoDB: No valid checkpoint found.*corrupted redo log" found # valid 5.7.9 header, valid checkpoint 1, all-zero (invalid) checkpoint 2, -# invalid block checksum (MLOG_CHECKPOINT cannot be read), +# invalid block checksum, # thus we cannot determine if the redo log is logically clean. Pattern "InnoDB: Database upgrade cannot be accomplished with innodb_force_recovery > 0" found # --innodb-force-recovery=6 (skip the entire redo log) Pattern "InnoDB: Database upgrade cannot be accomplished in read-only mode" found -# valid header, valid checkpoint 1, all-zero (invalid) checkpoint 2, invalid block number +# valid 5.7.9 header, valid checkpoint 1, all-zero (invalid) checkpoint 2, invalid block number # Start with innodb-force-recovery=5 Pattern "InnoDB: Database upgrade cannot be accomplished with innodb_force_recovery > 0" found # Start with --innodb-force-recovery=6 (skip the entire redo log) Pattern "InnoDB: Database upgrade cannot be accomplished in read-only mode" found -# valid 8.0.0 header, valid checkpoint 1, all-zero (invalid) checkpoint 2, invalid block checksum +# valid 8.0.3 header, valid checkpoint 1, all-zero (invalid) checkpoint 2, invalid block checksum Pattern "InnoDB: Log block 2372 at lsn 1213952 has valid header, but checksum field contains 144444122, should be 3362026715" found Pattern "Data Dictionary initialization failed" found # --innodb-force-recovery=6 (skip the entire redo log) Pattern "InnoDB: Database upgrade cannot be accomplished in read-only mode" found # valid 5.7.9 header, valid checkpoint 1, no matching MLOG_CHECKSUM -Pattern "InnoDB: Upgrade after a crash is not supported\. This redo log was created with malicious intentions, or perhaps\." found +Pattern "InnoDB: Upgrade after a crash is not supported\. This redo log was created with malicious intentions, or perhaps\." not found Pattern "Data Dictionary initialization failed" found # --innodb-force-recovery=6 (skip the entire redo log) Pattern "InnoDB: Database upgrade cannot be accomplished in read-only mode" found # upgrade: valid 5.7.9 header, valid checkpoint 1, logically non empty # redo log -Pattern "InnoDB: Upgrade after a crash is not supported\. This redo log was created with malicious intentions, or perhaps\." found +Pattern "InnoDB: Upgrade after a crash is not supported.*" found # Test a corrupted record. # current header, valid checkpoint 1, all-zero (invalid) checkpoint 2 Pattern "InnoDB: ############### CORRUPT LOG RECORD FOUND ###############" found diff --git a/mysql-test/suite/innodb/r/log_file_name.result b/mysql-test/suite/innodb/r/log_file_name.result deleted file mode 100644 index 4fcd5e0c29b6..000000000000 --- a/mysql-test/suite/innodb/r/log_file_name.result +++ /dev/null @@ -1,124 +0,0 @@ -# Test tablespace discovery during crash recovery -# including the detection of duplicate tablespaces. -# Clear old log file -# restart -# Do up some DDL and DML to recover -SET GLOBAL innodb_file_per_table=ON; -SET GLOBAL innodb_master_thread_disabled_debug=1; -CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE t3(a INT PRIMARY KEY) ENGINE=InnoDB; -SET GLOBAL innodb_checkpoint_disabled=1; -BEGIN; -INSERT INTO t3 VALUES (2008),(08),(25); -INSERT INTO t1 VALUES (1964),(12),(25); -RENAME TABLE t1 TO t2; -UPDATE t2 SET a=2000 where a=1964; -COMMIT; -# Kill the server -# Fault 0 (no real fault): Orphan file with duplicate space_id. -# Fault 1: Two dirty files with the same space_id. -# Attempt to start mysqld. Recovery will fail -# Remove orphaned file from fault 0 -# Should startup fine -# restart -DROP TABLE t2; -DROP TABLE t3; -CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE t3(a INT PRIMARY KEY) ENGINE=InnoDB; -SET GLOBAL innodb_checkpoint_disabled=1; -BEGIN; -INSERT INTO t3 VALUES (2008),(08),(25); -INSERT INTO t1 VALUES (1964),(12),(25); -RENAME TABLE t1 TO t2; -UPDATE t2 SET a=2000 where a=1964; -COMMIT; -# Kill the server -# Fault 2: Wrong space_id in a dirty file, and a missing file. -# Attempt to start mysqld. Recovery will fail -Pattern "InnoDB: Tablespace file '.*t1.ibd' ID mismatch, expected \d+ but found \d+" found -# Fault 3: Wrong space_id in dirty file(s) -# Swap t2.ibd and t3.ibd. -# Attempt to start mysqld. Recovery will fail -Pattern "InnoDB: Tablespace file '.*t2.ibd' ID mismatch, expected \d+ but found \d+" found -Pattern "InnoDB: Tablespace file '.*t3.ibd' ID mismatch, expected \d+ but found \d+" found -Pattern "InnoDB: Some files associated with the tablespace \d+ were not found: '.*t2.ibd'" found -Pattern "InnoDB: Some files associated with the tablespace \d+ were not found: '.*t3.ibd'" found -# Swap back t3.ibd, but hide t2.ibd. -# Fault 4: Missing data file -# Attempt to start mysqld. Recovery will fail -Pattern "InnoDB: Some files associated with the tablespace \d+ were not found: '.*t2.ibd'" found -Pattern "InnoDB: Some files associated with the tablespace \d+ were not found" found -# Fault 5: Empty data file -# Attempt to start mysqld. Recovery will fail -Pattern "\[ERROR\] \[[^]]*\] InnoDB: Datafile .*t2.*\. Cannot determine the space ID from the first 64 pages" found -# Restore t2.ibd -# restart -SELECT * FROM t2; -a -12 -25 -2000 -SELECT * FROM t3; -a -8 -25 -2008 -SHOW TABLES; -Tables_in_test -t2 -t3 -DROP TABLE t2,t3; -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -ERROR HY000: Tablespace '`test`.`t0`' exists. -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -ERROR HY000: Tablespace '`test`.`t0`' exists. -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -DROP TABLE t0; -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -ERROR HY000: Tablespace '`test`.`t0`' exists. -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -ERROR HY000: Tablespace '`test`.`t0`' exists. -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -DROP TABLE t0; -SET GLOBAL innodb_master_thread_disabled_debug=1; -SET GLOBAL innodb_checkpoint_disabled=1; -CREATE TABLE u1(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE u2(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE u3(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE u4(a INT PRIMARY KEY) ENGINE=InnoDB; -INSERT INTO u4 VALUES(1); -RENAME TABLE u4 TO u5; -RENAME TABLE u5 TO u6; -INSERT INTO u6 VALUES(2); -# Kill the server -# Fault 6: Check file exists use case during RENAME -# Copy u6.ibd -> u4.ibd -# We now have the original file and the final file (u4 -> u5 -> u6) -# Recovery should should rename u4 -> u5 and abort on u5 -> u6 -# Attempt to start mysqld. Recovery will fail -Pattern "InnoDB: Attempted to open a previously opened tablespace. Previous tablespace .*u5 at filepath: .*u5.ibd uses space ID: \d+. Cannot open filepath: .*u6.ibd which uses the same space ID" found -# Fix the problem by removing u5.ibd -# Fault 7: Check all zero data file -# Remove u1.ibd, u2.ibd & u3.ibd -# Make the header page of u1.ibd consists of zero bytes -# Write "" to u2.ibd -# Attempt to start mysqld. Recovery will fail -Pattern "\[ERROR\] \[[^]]*\] InnoDB: Datafile .*u1.*\. Cannot determine the space ID from the first 64 pages" found -Pattern "\[ERROR\] \[[^]]*\] InnoDB: Cannot read first page of .*u2.ibd.*" found -# Fault 8: Missing or wrong data file -# Remove empty u2.ibd -# Attempt to start mysqld. Recovery will fail -Pattern "\[ERROR\] \[[^]]*\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd" found -# Remove empty u6.ibd -# Attempt to start mysqld with force-recovery=1. Recovery will fail -Pattern "\[ERROR\] \[[^]]*\] unknown option '--innodb-nonexistent-option'" found -Pattern "\[ERROR\] \[[^]]*\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd" found -Pattern "InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace" found -# restart -DROP TABLE u1,u2,u3,u6; -# List of files: -SHOW TABLES; -Tables_in_test -SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLES -WHERE NAME NOT LIKE 'SYS_%' AND NAME NOT LIKE 'sys/%' AND NAME NOT LIKE 'mysql/%'; -TABLE_ID NAME FLAG N_COLS SPACE ROW_FORMAT ZIP_PAGE_SIZE SPACE_TYPE diff --git a/mysql-test/suite/innodb/r/log_file_name_discover.result b/mysql-test/suite/innodb/r/log_file_name_discover.result index 07f0c749213a..b93cefdbf8e4 100644 --- a/mysql-test/suite/innodb/r/log_file_name_discover.result +++ b/mysql-test/suite/innodb/r/log_file_name_discover.result @@ -1,28 +1,19 @@ SET GLOBAL innodb_file_per_table=ON; +CREATE DATABASE discover; +use discover; SET GLOBAL innodb_master_thread_disabled_debug=1; SET SESSION debug="+d,fil_ibd_create_log"; -CREATE TABLE t1(c INT) engine=InnoDB; +CREATE TABLE discover.t1(c INT) engine=InnoDB; ERROR HY000: Lost connection to MySQL server during query # restart -call mtr.add_suppression("InnoDB: Table .*test.*t1.*does not exist in the InnoDB internal data dictionary though MySQL is trying to drop it"); -DROP TABLE IF EXISTS t1; -Warnings: -Note 1051 Unknown table 'test.t1' -CREATE TABLE t2(c INT) ENGINE=InnoDB; +CREATE TABLE discover.t2(c INT) ENGINE=InnoDB; BEGIN; INSERT INTO t2 VALUES(1); INSERT INTO t2 SELECT * FROM t2; INSERT INTO t2 SELECT * FROM t2; INSERT INTO t2 SELECT * FROM t2; -INSERT INTO t2 SELECT * FROM t2; -INSERT INTO t2 SELECT * FROM t2; -INSERT INTO t2 SELECT * FROM t2; -INSERT INTO t2 SELECT * FROM t2; # Kill the server -"Removing tablespaces.open.* files" -"Restarting with --innodb-scan-directories=." -# restart: --innodb-scan-directories=.; +# restart SELECT * FROM t2; c -DROP TABLE t2; -call mtr.add_suppression("InnoDB: No space ID to filename mapping file found"); +DROP DATABASE discover; diff --git a/mysql-test/suite/innodb/r/readahead.result b/mysql-test/suite/innodb/r/readahead.result index c31cee5919f7..82997d1b41df 100644 --- a/mysql-test/suite/innodb/r/readahead.result +++ b/mysql-test/suite/innodb/r/readahead.result @@ -5,5 +5,7 @@ INDEX(b(5), c(10), a) ) charset latin1 ENGINE=INNODB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; # restart SET GLOBAL innodb_random_read_ahead = 1; +call mtr.add_suppression("\\[ERROR\\].*InnoDB: '.*t1.ibd' read failed! - attempted to read 4 bytes, read only 0 bytes at offset 16418"); +call mtr.add_suppression("\\[Warning\\].*InnoDB: Ignoring '.*t1.ibd' invalid tablespace ID in the header"); DROP TABLE t1; SET GLOBAL innodb_random_read_ahead = default; diff --git a/mysql-test/suite/innodb/r/tablespace_per_table_not_windows.result b/mysql-test/suite/innodb/r/tablespace_per_table_not_windows.result index 67f67a036fc6..342284d431bb 100644 --- a/mysql-test/suite/innodb/r/tablespace_per_table_not_windows.result +++ b/mysql-test/suite/innodb/r/tablespace_per_table_not_windows.result @@ -3,13 +3,6 @@ # the database name with the table name to make a unique table name. # SET default_storage_engine=InnoDB; -# Suppress errors about too long filenames in error log. -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number .* in a file operation."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Error number .* means '.*'"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot create file .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'create' returned OS error .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'delete' returned OS error .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'stat' returned OS error .*"); # # MySQL limits each database and tablename identifier to 64 characters # of up to 3 bytes per character, corresponding to 192 bytes. diff --git a/mysql-test/suite/innodb/r/tablespace_portability.result b/mysql-test/suite/innodb/r/tablespace_portability.result index 940c3961aa9e..84e14b89e249 100644 --- a/mysql-test/suite/innodb/r/tablespace_portability.result +++ b/mysql-test/suite/innodb/r/tablespace_portability.result @@ -7,6 +7,10 @@ # on any OS, we create the tablespaces here. # SET DEFAULT_STORAGE_ENGINE=InnoDB; +# restart:--innodb-directories=MYSQL_TMP_DIR +SHOW VARIABLES LIKE 'innodb_directories'; +Variable_name Value +innodb_directories MYSQL_TMP_DIR # # Try to create a tablespace in a read-only directory. # diff --git a/mysql-test/suite/innodb/r/tablespace_rename.result b/mysql-test/suite/innodb/r/tablespace_rename.result index 7bfba85f1f0d..31ddb4bf6635 100644 --- a/mysql-test/suite/innodb/r/tablespace_rename.result +++ b/mysql-test/suite/innodb/r/tablespace_rename.result @@ -11,7 +11,7 @@ SHOW TABLES; Tables_in_test t1 # -# Test crash before writing MLOG_FILE_OPEN t2. +# Test crash before writing MLOG_FILE_RENAME t2. # On recovery the rename never happened. # SET SESSION debug="+d,ib_crash_rename_log_1"; diff --git a/mysql-test/suite/innodb/t/alter_kill.test b/mysql-test/suite/innodb/t/alter_kill.test index 380dafa86866..be7617532584 100644 --- a/mysql-test/suite/innodb/t/alter_kill.test +++ b/mysql-test/suite/innodb/t/alter_kill.test @@ -21,6 +21,7 @@ call mtr.add_suppression("InnoDB: \(The error means\|If you are\)"); call mtr.add_suppression("Skip re-populating collations and character sets tables in InnoDB read-only mode."); call mtr.add_suppression("InnoDB: Error number [0-9]* means 'Resource temporarily unavailable'"); call mtr.add_suppression("Skip updating information_schema metadata in InnoDB read-only mode."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Tablespace [0-9]+, name 'test.*bug16720368', unable to open file '.*test.*bug16720368.ibd' - Data structure corruption"); call mtr.add_suppression("Skipped updating resource group metadata in InnoDB read only mode."); -- enable_query_log @@ -143,7 +144,7 @@ let SEARCH_FILE= $MYSQLTEST_VARDIR/log/my_restart.err; -- error 1,42 -- exec $MYSQLD_CMD --core-file --console > $SEARCH_FILE 2>&1; -let SEARCH_PATTERN= \[ERROR\] \[[^]]*\] InnoDB:.*tablespace \d+ .*not found:.*test.bug16735660.ibd.*; +let SEARCH_PATTERN= \[ERROR\].*InnoDB: Could not find any file associated with the tablespace ID:.*; -- source include/search_pattern.inc -- move_file $MYSQLD_DATADIR/bug16735660.omg $MYSQLD_DATADIR/test/bug16735660.ibd diff --git a/mysql-test/suite/innodb/t/alter_missing_tablespace.test b/mysql-test/suite/innodb/t/alter_missing_tablespace.test index 41bb0a4b2329..602949f4ca0a 100644 --- a/mysql-test/suite/innodb/t/alter_missing_tablespace.test +++ b/mysql-test/suite/innodb/t/alter_missing_tablespace.test @@ -19,6 +19,7 @@ EOF --source include/start_mysqld.inc --disable_query_log +call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace .*, name '.*', file '.*' is missing"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot calculate statistics for table `test`\.`t` because the \.ibd file is missing"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot delete tablespace [0-9]+ in DISCARD TABLESPACE: Tablespace not found"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace .* because it could not be opened"); diff --git a/mysql-test/suite/innodb/t/create_tablespace.test b/mysql-test/suite/innodb/t/create_tablespace.test index 3ba5be7e701f..c9bbc941d947 100644 --- a/mysql-test/suite/innodb/t/create_tablespace.test +++ b/mysql-test/suite/innodb/t/create_tablespace.test @@ -1,5 +1,5 @@ # -# WL#6205 - A series of tests to show the correct behavior for +# A series of tests to show the correct behavior for # CREATE TABLESPACE and associated SQL statements. # @@ -12,6 +12,16 @@ SET NAMES utf8; LET $MYSQLD_DATADIR = `select @@datadir`; LET $INNODB_PAGE_SIZE = `select @@innodb_page_size`; +--disable_query_log +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +LET $restart_parameters = restart:--innodb-directories=$MYSQL_TMP_DIR; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--source include/restart_mysqld.inc +--enable_query_log + +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +SHOW VARIABLES LIKE 'innodb_directories'; + --echo # Strict-mode has no effect on CREATE TABLESPACE. --echo # It rejects all invalid input, as if strict mode is always ON. SHOW VARIABLES LIKE 'innodb_strict_mode'; @@ -70,10 +80,10 @@ CREATE TABLESPACE test/s_bad ADD DATAFILE 's_bad.ibd'; --error ER_WRONG_TABLESPACE_NAME CREATE TABLESPACE `test/s_bad` ADD DATAFILE 's_bad.ibd'; LET $TWOFIFTYFIVE=../xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx; -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'delete' returned OS error .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'stat' returned OS error .*"); ---error ER_CREATE_FILEGROUP_FAILED +--error ER_WRONG_FILE_NAME eval CREATE TABLESPACE `s_too_long_file_name` ADD DATAFILE '$TWOFIFTYFIVE.ibd'; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +SHOW WARNINGS; --error ER_FILEGROUP_OPTION_ONLY_ONCE CREATE TABLESPACE s_bad ADD DATAFILE 's_bad.ibd' FILE_BLOCK_SIZE=1k FILE_BLOCK_SIZE=2k; --error ER_ILLEGAL_HA_CREATE_OPTION @@ -1003,30 +1013,15 @@ SET GLOBAL innodb_strict_mode=default; --rmdir $MYSQL_TMP_DIR/test --rmdir $MYSQL_TMP_DIR/tablespace.ibd ---disable_query_log -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .*\.ibd: 'create' returned OS error .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the system cannot find the path specified"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means mysqld does not have the access rights to the directory. It may also be you have created a subdirectory of the same name as a data file."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number .* in a file operation"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Error number .* means"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot create file"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Invalid use of ':' in"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The file .* already exists though the corresponding table did not exist."); ---enable_query_log --echo # --echo # Bug #20763179 SEGV WHEN CREATE TABLESPACE IS EXECUTED IN READ ONLY MODE --echo # --echo # Check CREATE TABLESPACE is handled properly in read-only mode ---disable_query_log -call mtr.add_suppression("Skip re-populating collations and character sets tables in InnoDB read-only mode."); -call mtr.add_suppression("Skip updating information_schema metadata in InnoDB read-only mode."); -call mtr.add_suppression("Skipped updating resource group metadata in InnoDB read only mode."); ---enable_query_log - CREATE TABLESPACE s1 ADD DATAFILE 's1.ibd' ENGINE InnoDB; -let $restart_parameters = restart: --innodb_read_only; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +let $restart_parameters = restart: --innodb_read_only --innodb-directories=$MYSQL_TMP_DIR; --source include/restart_mysqld.inc --error ER_OPEN_AS_READONLY CREATE TABLESPACE s2 ADD DATAFILE 's2.ibd' ENGINE InnoDB; @@ -1036,7 +1031,8 @@ DROP TABLESPACE s1; SHOW WARNINGS; --echo # Restart in normal mode for cleanup. -let $restart_parameters = restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +let $restart_parameters = restart --innodb-directories=$MYSQL_TMP_DIR; --source include/restart_mysqld.inc DROP TABLESPACE s1; @@ -1203,3 +1199,21 @@ SHOW CREATE TABLE after_alter_like_t_impl_tmp; DROP SCHEMA s1; DROP TABLESPACE s1; SET GLOBAL innodb_file_per_table= DEFAULT; + +--disable_query_log +call mtr.add_suppression("\\[Warning\\].*Skip re-populating collations and character sets tables in InnoDB read-only mode"); +call mtr.add_suppression("\\[Warning\\].*Skip updating information_schema metadata in InnoDB read-only mode"); +call mtr.add_suppression("\\[Warning\\].*Skipped updating resource group metadata in InnoDB read only mode"); +call mtr.add_suppression("\\[Warning\\].*InnoDB: '.*tablespace.ibd' is a directory, can't delete!"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: File .* 'delete' returned OS error .*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: File .* 'stat' returned OS error .*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: File .*\.ibd: 'create' returned OS error .*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: The error means the system cannot find the path specified"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: The error means mysqld does not have the access rights to the directory. It may also be you have created a subdirectory of the same name as a data file."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Operating system error number .* in a file operation"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Error number .* means"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Cannot create file"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Invalid use of ':' in"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: The file .* already exists though the corresponding table did not exist."); +--enable_query_log + diff --git a/mysql-test/suite/innodb/t/create_tablespace_16k.test b/mysql-test/suite/innodb/t/create_tablespace_16k.test index bb3b3de136ca..37ec3c798aa1 100644 --- a/mysql-test/suite/innodb/t/create_tablespace_16k.test +++ b/mysql-test/suite/innodb/t/create_tablespace_16k.test @@ -640,7 +640,10 @@ call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the sy call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot open datafile for read-only"); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot delete tablespace .* because it is not found in the tablespace memory cache"); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Failed to find tablespace for table `test`.`t_missing` in the cache. Attempting to load the tablespace with space id .*"); +call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace .*, name 's_missing', file 'delete_me.ibd' is missing!"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace `s_missing` because it could not be opened"); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Unable to open tablespace .* \\(flags=.*, filename=delete_me.ibd\\);"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Trying to access missing tablespace .*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: '.*s_.k.ibd' read failed! - attempted to read 4 bytes, read only 0 bytes at offset 16418"); +call mtr.add_suppression("\\[Warning\\].*InnoDB: Ignoring '.*s_.k.ibd' invalid tablespace ID in the header"); --enable_query_log diff --git a/mysql-test/suite/innodb/t/create_tablespace_8k.test b/mysql-test/suite/innodb/t/create_tablespace_8k.test index 6d47a6a49997..1d96a712c241 100644 --- a/mysql-test/suite/innodb/t/create_tablespace_8k.test +++ b/mysql-test/suite/innodb/t/create_tablespace_8k.test @@ -475,6 +475,9 @@ DROP TABLE `t_zip1k_to_file_per_table`; DROP TABLE `t_zip2k_to_file_per_table`; DROP TABLE `t_zip4k_to_file_per_table`; +call mtr.add_suppression("\\[ERROR\\].*InnoDB: '.*s_.k.ibd' read failed! - attempted to read 4 bytes, read only 0 bytes at offset 8226"); +call mtr.add_suppression("\\[Warning\\].*InnoDB: Ignoring '.*s_.k.ibd' invalid tablespace ID in the header"); + --echo # --echo # Clean-up. --echo # diff --git a/mysql-test/suite/innodb/t/create_tablespace_debug.test b/mysql-test/suite/innodb/t/create_tablespace_debug.test index a6e5b82a78db..41e4edd768be 100644 --- a/mysql-test/suite/innodb/t/create_tablespace_debug.test +++ b/mysql-test/suite/innodb/t/create_tablespace_debug.test @@ -3,6 +3,8 @@ # --source include/have_debug.inc +--source include/not_valgrind.inc + SET DEFAULT_STORAGE_ENGINE=InnoDB; # Set these up for show_i_s_tablespaces.inc @@ -119,12 +121,14 @@ SELECT count(*) INTO @matching_is_tbs_row_cnt FROM information_schema.innodb_tab SELECT @is_tbs_row_cnt = @matching_is_tbs_row_cnt ; --disable_query_log -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number .* in a file operation"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means that another program is using InnoDB's files"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the system cannot find the path specified."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot open datafile for read-only"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Could not find a valid tablespace file for"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .*s_def.ibd: 'delete' returned OS error [0-9]*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Attempted to open a previously opened tablespace. Previous tablespace innodb_general_[0-9]* at filepath: .*s1.ibd uses space ID: [0-9]*. Cannot open filepath: .*s1.ibd which uses the same space ID."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Operating system error number .* in a file operation"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: The error means that another program is using InnoDB's files"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: The error means the system cannot find the path specified."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Cannot open datafile for read-only"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Could not find a valid tablespace file for"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: File .*s_def.ibd: 'delete' returned OS error [0-9]*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Failed to delete the datafile of tablespace [0-9]+, file '.*s_def.ibd'!"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Attempted to open a previously opened tablespace. Previous tablespace innodb_general_[0-9]* at filepath: .*s1.ibd uses space ID: [0-9]*. Cannot open filepath: .*s1.ibd which uses the same space ID."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Tablespace [0-9]+, name '.*test.*t[0-9]+', unable to open file '.*test.*t[0-4]+.ibd' - Data structure corruption"); --enable_query_log diff --git a/mysql-test/suite/innodb/t/ddl_crash_alter_partition.test b/mysql-test/suite/innodb/t/ddl_crash_alter_partition.test index ba4bf197ce56..9b22f30bf7db 100644 --- a/mysql-test/suite/innodb/t/ddl_crash_alter_partition.test +++ b/mysql-test/suite/innodb/t/ddl_crash_alter_partition.test @@ -112,6 +112,7 @@ call mtr.add_suppression("Could not find a valid tablespace file for"); call mtr.add_suppression("Operating system error number 2 in a file operation."); call mtr.add_suppression("The error means the system cannot find the path specified."); call mtr.add_suppression("Cannot open datafile for read-only:"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Tablespace.*, name 'test.t[12].*', unable to open file '.*test.t[12].*\\.ibd' - Data structure corruption"); --enable_query_log let $create_statement= CREATE TABLE $table_name diff --git a/mysql-test/suite/innodb/t/ddl_crash_basic.test b/mysql-test/suite/innodb/t/ddl_crash_basic.test index 2843a4d7bc53..2e42a316180e 100644 --- a/mysql-test/suite/innodb/t/ddl_crash_basic.test +++ b/mysql-test/suite/innodb/t/ddl_crash_basic.test @@ -9,6 +9,7 @@ # ... and the crash recovery has finally success. --disable_query_log CALL mtr.add_suppression("\\[ERROR\\] InnoDB: Could not find a valid tablespace file for"); +CALL mtr.add_suppression("\\[ERROR\\].*InnoDB: Tablespace.*, name 'test.t1', unable to open file '.*test.t1.ibd' - Data structure corruption"); --enable_query_log SET SESSION debug= '+d,skip_dd_table_access_check'; diff --git a/mysql-test/suite/innodb/t/innodb-multiple-tablespaces.test b/mysql-test/suite/innodb/t/innodb-multiple-tablespaces.test deleted file mode 100644 index 925b80c89bea..000000000000 --- a/mysql-test/suite/innodb/t/innodb-multiple-tablespaces.test +++ /dev/null @@ -1,453 +0,0 @@ -# -# These test make sure that tables are visible after rebooting -# - ---source include/have_debug.inc - -# Avoid CrashReporter popup on Mac -#--source include/not_crashrep.inc - -SET default_storage_engine=InnoDB; - ---disable_query_log -LET $MYSQLD_DATADIR = `select @@datadir`; -LET $INNODB_PAGE_SIZE = `select @@innodb_page_size`; -LET $data_directory = DATA DIRECTORY='$MYSQL_TMP_DIR/alt_dir'; ---enable_query_log - ---mkdir $MYSQL_TMP_DIR/alt_dir ---mkdir $MYSQL_TMP_DIR/alt_dir/test ---mkdir $MYSQL_TMP_DIR/new_dir ---mkdir $MYSQL_TMP_DIR/new_dir/test - ---disable_query_log -# These errors are expected in the error log for this test. -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot open datafile for read-only:"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Failed to find tablespace for table `test`.`.*` in the cache. Attempting to load"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number 2 in a file operation."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the system cannot find the path specified."); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot calculate statistics for table `test`\.`.*` because the \.ibd file is missing"); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace `test/.*` because it could not be opened."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Unable to open tablespace .* \\(flags=.*, filename="); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Trying to access missing tablespace .*"); ---enable_query_log - ---echo # ---echo # Test when tablespaces can be found at multiple places ---echo # SYS_DATAFILES will refer to the file at alt_dir. ---echo # Link File will refer to the file at new_dir. ---echo # Tablename Default SYS_DATAFILES Link_File ---echo # yyy Yes Yes Yes ---echo # nyy No Yes Yes ---echo # yny Yes No Yes ---echo # yyn Yes Yes No ---echo # nyw No Yes WrongFile ---echo # nwy No WrongFile Yes ---echo # wny WrongFile No Yes ---echo # ynw Yes No WrongFile ---echo # wyn WrongFile Yes No ---echo # ywn Yes WrongFile No ---echo # nnn No No No ---echo # www WrongFile WrongFile WrongFile ---echo # - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE yyy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO yyy VALUES (1, 'yyy'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE nyy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO nyy VALUES (1, 'nyy'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE yny (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO yny VALUES (1, 'yny'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE yyn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO yyn VALUES (1, 'yyn'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE nyw (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO nyw VALUES (1, 'nyw'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE nwy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO nwy VALUES (1, 'nwy'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE wny (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO wny VALUES (1, 'wny'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE ynw (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO ynw VALUES (1, 'ynw'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE wyn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO wyn VALUES (1, 'wyn'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE ywn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO ywn VALUES (1, 'ywn'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE nnn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO nnn VALUES (1, 'nnn'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE www (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO www VALUES (1, 'www'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE nolink (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO nolink VALUES (1, 'no link file'); - ---echo # ---echo # Shutdown the server, copy and remove files. ---echo # ---source include/shutdown_mysqld.inc - ---echo ---- MYSQLD_DATADIR/test ---list_files $MYSQLD_DATADIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test ---list_files $MYSQL_TMP_DIR/alt_dir/test ---echo ---- MYSQL_TMP_DIR/new_dir/test ---list_files $MYSQL_TMP_DIR/new_dir/test - ---echo # YYY; Tablespace found in 3 places ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQLD_DATADIR/test/yyy.ibd ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQL_TMP_DIR/new_dir/test/yyy.ibd - ---echo # NYY; Tablespace found in alt_dir and new_dir ---copy_file $MYSQL_TMP_DIR/alt_dir/test/nyy.ibd $MYSQL_TMP_DIR/new_dir/test/nyy.ibd - ---echo # YNY; Tablespace found in default and new_dir ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yny.ibd $MYSQLD_DATADIR/test/yny.ibd ---move_file $MYSQL_TMP_DIR/alt_dir/test/yny.ibd $MYSQL_TMP_DIR/new_dir/test/yny.ibd - ---echo # YYN; Tablespace found in default and alt_dir ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yyn.ibd $MYSQLD_DATADIR/test/yyn.ibd - ---echo # NYW; Copy the wrong file to new_dir ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQL_TMP_DIR/new_dir/test/nyw.ibd - ---echo # NWY; Move the wrong file to alt_dir, good one to new_dir. ---move_file $MYSQL_TMP_DIR/alt_dir/test/nwy.ibd $MYSQL_TMP_DIR/new_dir/test/nwy.ibd -#--copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQL_TMP_DIR/alt_dir/test/nwy.ibd - ---echo # WNY; Move the wrong file to default, good one to new_dir, delete it form alt_dir ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQLD_DATADIR/test/wny.ibd ---move_file $MYSQL_TMP_DIR/alt_dir/test/wny.ibd $MYSQL_TMP_DIR/new_dir/test/wny.ibd - ---echo # YNW; Move the file to default, wrong one to new_dir, delete it form alt_dir ---copy_file $MYSQL_TMP_DIR/alt_dir/test/ynw.ibd $MYSQLD_DATADIR/test/ynw.ibd ---remove_file $MYSQL_TMP_DIR/alt_dir/test/ynw.ibd ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQL_TMP_DIR/new_dir/test/ynw.ibd - ---echo # WYN; Copy the wrong file to default ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQLD_DATADIR/test/wyn.ibd - ---echo # YWN; Move the file to default, wrong one to alt_dir ---move_file $MYSQL_TMP_DIR/alt_dir/test/ywn.ibd $MYSQLD_DATADIR/test/ywn.ibd -#--copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQL_TMP_DIR/alt_dir/test/ywn.ibd - ---echo # NNN; Delete the tablespace ---remove_file $MYSQL_TMP_DIR/alt_dir/test/nnn.ibd - ---echo # WWW; Put the wrong file in all three locations -#--copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQLD_DATADIR/test/www.ibd ---remove_file $MYSQL_TMP_DIR/alt_dir/test/www.ibd -#--copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQL_TMP_DIR/alt_dir/test/www.ibd -#--copy_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd $MYSQL_TMP_DIR/new_dir/test/www.ibd - ---echo # Make a backup of this tablespace to use later. ---copy_file $MYSQL_TMP_DIR/alt_dir/test/nolink.ibd $MYSQL_TMP_DIR/alt_dir/test/nolink.ibd.bak - ---echo ---- MYSQLD_DATADIR/test ---list_files $MYSQLD_DATADIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test ---list_files $MYSQL_TMP_DIR/alt_dir/test ---echo ---- MYSQL_TMP_DIR/new_dir/test ---list_files $MYSQL_TMP_DIR/new_dir/test - ---echo # ---echo # Start the server and show the tablespaces. ---echo # ---source include/start_mysqld.inc - ---source suite/innodb/include/show_i_s_tablespaces.inc - -SELECT * FROM yyy; -SELECT * FROM nyy; ---error ER_TABLESPACE_MISSING -SELECT * FROM yny; -SELECT * FROM yyn; -SELECT * FROM nyw; ---error ER_TABLESPACE_MISSING -SELECT * FROM nwy; ---error ER_TABLESPACE_MISSING -SELECT * FROM wny; ---error ER_TABLESPACE_MISSING -SELECT * FROM ynw; -SELECT * FROM wyn; ---error ER_TABLESPACE_MISSING -SELECT * FROM ywn; ---error ER_TABLESPACE_MISSING -SELECT * FROM nnn; ---error ER_TABLESPACE_MISSING -SELECT * FROM www; -SELECT * FROM nolink; - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE yyy; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE nyy; ---error ER_TABLESPACE_MISSING -SHOW CREATE TABLE yny; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE yyn; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE nyw; ---error ER_TABLESPACE_MISSING -SHOW CREATE TABLE nwy; ---error ER_TABLESPACE_MISSING -SHOW CREATE TABLE wny; ---error ER_TABLESPACE_MISSING -SHOW CREATE TABLE ynw; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE wyn; ---error ER_TABLESPACE_MISSING -SHOW CREATE TABLE ywn; ---error ER_TABLESPACE_MISSING -SHOW CREATE TABLE nnn; ---error ER_TABLESPACE_MISSING -SHOW CREATE TABLE www; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE nolink; - ---echo # ---echo # List of files before DROP TABLES ---echo # ---echo ---- MYSQLD_DATADIR/test ---list_files $MYSQLD_DATADIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test ---list_files $MYSQL_TMP_DIR/alt_dir/test ---echo ---- MYSQL_TMP_DIR/new_dir/test ---list_files $MYSQL_TMP_DIR/new_dir/test - ---echo # ---echo # Restart the server and DROP the tablespaces. ---echo # ---source include/restart_mysqld.inc - -DROP TABLE yyy; -DROP TABLE nyy; -DROP TABLE yny; -DROP TABLE yyn; -DROP TABLE nyw; -DROP TABLE nwy; -DROP TABLE wny; -DROP TABLE ynw; -DROP TABLE wyn; -DROP TABLE ywn; -DROP TABLE nnn; -DROP TABLE www; -DROP TABLE nolink; - ---echo # ---echo # List of files not deleted by the DROP TABLES ---echo # ---echo ---- MYSQLD_DATADIR/test ---list_files $MYSQLD_DATADIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test ---list_files $MYSQL_TMP_DIR/alt_dir/test ---echo ---- MYSQL_TMP_DIR/new_dir/test ---list_files $MYSQL_TMP_DIR/new_dir/test - -#--remove_file $MYSQLD_DATADIR/test/www.ibd ---remove_file $MYSQLD_DATADIR/test/wny.ibd ---remove_file $MYSQLD_DATADIR/test/wyn.ibd ---remove_file $MYSQLD_DATADIR/test/yny.ibd ---remove_file $MYSQLD_DATADIR/test/yyn.ibd ---remove_file $MYSQLD_DATADIR/test/yyy.ibd ---remove_file $MYSQL_TMP_DIR/new_dir/test/nwy.ibd ---remove_file $MYSQL_TMP_DIR/new_dir/test/wny.ibd -#--remove_file $MYSQL_TMP_DIR/alt_dir/test/nyy.ibd -#--remove_file $MYSQL_TMP_DIR/alt_dir/test/www.ibd -#--remove_file $MYSQL_TMP_DIR/alt_dir/test/ywn.ibd -#--remove_file $MYSQL_TMP_DIR/alt_dir/test/yyn.ibd -#--remove_file $MYSQL_TMP_DIR/alt_dir/test/yyy.ibd ---remove_file $MYSQL_TMP_DIR/new_dir/test/nyw.ibd ---remove_file $MYSQL_TMP_DIR/new_dir/test/nyy.ibd -#--remove_file $MYSQL_TMP_DIR/new_dir/test/www.ibd ---remove_file $MYSQL_TMP_DIR/new_dir/test/ynw.ibd ---remove_file $MYSQL_TMP_DIR/new_dir/test/yny.ibd ---remove_file $MYSQL_TMP_DIR/new_dir/test/yyy.ibd - ---echo # ---echo # List of files after removing leftover files ---echo # ---echo ---- MYSQLD_DATADIR/test ---list_files $MYSQLD_DATADIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test ---list_files $MYSQL_TMP_DIR/alt_dir/test ---echo ---- MYSQL_TMP_DIR/new_dir/test ---list_files $MYSQL_TMP_DIR/new_dir/test - ---echo # ---echo # Create some tables again and this time, crash instead of shutdown ---echo # ---echo # Test recovery when tablespaces can be found at multiple places. ---echo # The data dictionary is unavailable during recovery. ---echo # 'nolink.ibd.bak' is the source of the 'wrong' tablespaces. ---echo # ---echo # Tablename Default_Tablespace Remote_Tablespace ---echo # ny No Yes ---echo # wy Wrong Yes ---echo # yn Yes No ---echo # yw Yes Wrong ---echo # yy Yes Yes (both the same file) ---echo # - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE ny (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO ny VALUES (1, 'ny'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE wy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO wy VALUES (1, 'wy'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE yn (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO yn VALUES (1, 'yn'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE yw (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO yw VALUES (1, 'yw'); - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -eval CREATE TABLE yy (c1 INT KEY, c2 TEXT) ENGINE=InnoDB $data_directory; -INSERT INTO yy VALUES (1, 'yy'); - ---echo # ---echo # Crash the server, copy and remove files. ---echo # - -#Ensure that master thread doesnt do a checkpoint -SET GLOBAL innodb_master_thread_disabled_debug=1; - -BEGIN; -INSERT INTO ny VALUES (2, 'ny'); -INSERT INTO wy VALUES (2, 'wy'); -INSERT INTO yn VALUES (2, 'yn'); -INSERT INTO yw VALUES (2, 'yw'); -INSERT INTO yy VALUES (2, 'yy'); - -SELECT * FROM ny; -SELECT * FROM wy; -SELECT * FROM yn; -SELECT * FROM yw; -SELECT * FROM yy; - ---source include/kill_mysqld.inc - ---echo # ---echo # Now that the engine is not running, move files around to test various scenarios. ---echo # - ---echo # NY; Tablespace found in alt_dir but not the default directory. - ---echo # WY; The wrong tablespace is found in the default directory ---echo # and the correct one in alt_dir. ---copy_file $MYSQL_TMP_DIR/alt_dir/test/nolink.ibd.bak $MYSQLD_DATADIR/test/wy.ibd - ---echo # YW; Tablespace is found in the default directory but the wrong file in alt_dir. ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yw.ibd $MYSQLD_DATADIR/test/yw.ibd ---remove_file $MYSQL_TMP_DIR/alt_dir/test/yw.ibd ---copy_file $MYSQL_TMP_DIR/alt_dir/test/nolink.ibd.bak $MYSQL_TMP_DIR/alt_dir/test/yw.ibd - ---echo # YN; Tablespace found the default directory but not in alt_dir. ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yn.ibd $MYSQLD_DATADIR/test/yn.ibd ---remove_file $MYSQL_TMP_DIR/alt_dir/test/yn.ibd - ---echo # YY; Found in both default directory and alt-dir. ---copy_file $MYSQL_TMP_DIR/alt_dir/test/yy.ibd $MYSQLD_DATADIR/test/yy.ibd - ---echo ---- MYSQLD_DATADIR/test ---list_files $MYSQLD_DATADIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test ---list_files $MYSQL_TMP_DIR/alt_dir/test - -let SEARCH_FILE= $MYSQLTEST_VARDIR/log/my_restart.err; -let $mysqld=$MYSQLD_CMD --core-file --console --log-error-verbosity=3 >$SEARCH_FILE 2>&1; - -# TODO: This could fail to refuse startup, in case there was a log -# checkpoint after the INSERT INTO yn. ---error 1,42 ---exec $mysqld - -let SEARCH_PATTERN= InnoDB: Could not find any file associated with the tablespace \d+; ---source include/search_pattern.inc - -let SEARCH_PATTERN= InnoDB: Tablespace file '.*.tmp.*alt_dir.test.yw.ibd' ID mismatch, expected \d+ but found \d+; ---source include/search_pattern.inc - ---echo # restoring yn.ibd, yw.ibd ---copy_file $MYSQLD_DATADIR/test/yn.ibd $MYSQL_TMP_DIR/alt_dir/test/yn.ibd ---remove_file $MYSQLD_DATADIR/test/yn.ibd ---remove_file $MYSQL_TMP_DIR/alt_dir/test/yw.ibd ---copy_file $MYSQLD_DATADIR/test/yw.ibd $MYSQL_TMP_DIR/alt_dir/test/yw.ibd ---remove_file $MYSQLD_DATADIR/test/yw.ibd - - ---source include/start_mysqld.inc - -SELECT * FROM ny; -SELECT * FROM wy; -SELECT * FROM yn; -SELECT * FROM yw; -SELECT * FROM yy; - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE ny; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE wy; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE yn; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE yw; ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SHOW CREATE TABLE yy; - ---source suite/innodb/include/show_i_s_tablespaces.inc - ---echo # ---echo # List of files before removing unused files ---echo # - -DROP TABLE ny; -DROP TABLE wy; -DROP TABLE yn; -DROP TABLE yw; -DROP TABLE yy; - ---echo # ---echo # List of files after DROP TABLES ---echo # ---echo ---- MYSQLD_DATADIR/test ---list_files $MYSQLD_DATADIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test ---list_files $MYSQL_TMP_DIR/alt_dir/test - ---remove_file $MYSQLD_DATADIR/test/wy.ibd ---remove_file $MYSQLD_DATADIR/test/yy.ibd ---remove_file $MYSQLD_DATADIR/test/ynw.ibd ---remove_file $MYSQLD_DATADIR/test/ywn.ibd ---remove_file $MYSQL_TMP_DIR/alt_dir/test/nolink.ibd.bak - ---rmdir $MYSQL_TMP_DIR/alt_dir/test ---rmdir $MYSQL_TMP_DIR/alt_dir ---rmdir $MYSQL_TMP_DIR/new_dir/test ---rmdir $MYSQL_TMP_DIR/new_dir diff --git a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test index a752466c8d45..3fcb3bb69408 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5522-debug.test +++ b/mysql-test/suite/innodb/t/innodb-wl5522-debug.test @@ -112,6 +112,11 @@ SELECT * FROM test_wl5522.t1; --error 2013 ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE; +perl; +require 'include/innodb-util.inc'; +ib_unlink_tablespace("test_wl5522", "t1"); +EOF + --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect @@ -138,6 +143,7 @@ ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE; # delete the old tablespace files and drop the table, # recreate the table and do a proper import. -- source include/wait_until_disconnected.inc + perl; require 'include/innodb-util.inc'; ib_unlink_tablespace("test_wl5522", "t1"); @@ -1248,6 +1254,7 @@ call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Trying to import a tab call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Unsupported tablespace format"); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number 32 in a file operation."); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means that another program is using InnoDB's files."); +call mtr.add_suppression("\\[Warning\\].*InnoDB: Tablespace.*, name 'test_wl5522.t1', file '.*test_wl5522.t1.ibd' is missing!"); --enable_query_log #cleanup diff --git a/mysql-test/suite/innodb/t/innodb-wl5980-discard.test b/mysql-test/suite/innodb/t/innodb-wl5980-discard.test index a463eedba174..4126478040d2 100644 --- a/mysql-test/suite/innodb/t/innodb-wl5980-discard.test +++ b/mysql-test/suite/innodb/t/innodb-wl5980-discard.test @@ -697,4 +697,5 @@ call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace for table .* is set as discarded"); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Unable to open tablespace .* \\(flags=.*, filename="); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Trying to access missing tablespace .*"); +call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace .*, name '.*', file '.*' is missing!"); -- enable_query_log diff --git a/mysql-test/suite/innodb/t/innodb-wl6445.test b/mysql-test/suite/innodb/t/innodb-wl6445.test index bb2b4fea5cf4..82186ccaca70 100644 --- a/mysql-test/suite/innodb/t/innodb-wl6445.test +++ b/mysql-test/suite/innodb/t/innodb-wl6445.test @@ -63,3 +63,7 @@ let $restart_parameters = restart; DROP TABLE WL6445.t1; DROP DATABASE WL6445; + +--disable_query_log +call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace .*, name '.*', file '.*' is missing"); +--enable_query_log diff --git a/mysql-test/suite/innodb/t/innodb_tsof_use_1.test b/mysql-test/suite/innodb/t/innodb_tsof_use_1.test deleted file mode 100644 index f343d2681001..000000000000 --- a/mysql-test/suite/innodb/t/innodb_tsof_use_1.test +++ /dev/null @@ -1,569 +0,0 @@ -# Purpose -# ------- -# 1. Check that the system behaves right during crash recovery after -# - suffering from more or less bad physical writes into some -# tablespaces.open.* file immediate followed by some server crash -# Basically: Emulate effects of a power loss and some small temporary -# hardware failure. -# - having had some server crash and missing all tablespaces.open.* files -# Emulate a mistake of the user. -# - having had some server crash and meeting outdated tablespaces.open.* -# files -# Emulate a mistake of the user. -# regarding: Does the server come up with success or not. -# I do not claim that all of the cases above must end with success. -# 2. Check that the system responds with the right notes, warnings and error -# messages in the server error log depending on the case met. -# Checking the presence of error log messages regarding defects of the -# tablespaces.open.* files -# not found , too small , unknown format -# is first priority. -# Checking the presence of other messages like -# [Note] InnoDB: Tablespace file './db_test_a/t1.ibd' ID mismatch, -# expected 86 but found 102 -# is only nice to have. -# -# WARNING -# ------- -# When improving or extending this test its easy to implement nice to have -# checks (see 2. above) and also make restart attempts without extra options -# at most places but such tests have a strong tendency to be unstable under -# high load. -# Please be aware of that the current test goes often without control over -# when InnoDB makes checkpoints and that has some dramatic impact if -# - the final result of crash recovery attempts without extra options -# (Example: "--innodb-scan-directories=$MYSQLD_DATADIR") end up with -# is that the server comes up or aborts. -# Example for all tablespaces.open.* files unusable (missing or rotten) -# and more an impression than some exact description: -# InnoDB will "criticize" the state of the tablespaces.open.* files first. -# Than recovering will be tried. -# During the recovery process InnoDB might detect that it cannot progress -# without valid data from at least one tablespaces.open.* file. -# If that happens than we get an abort of the recovery and need to go -# with "--innodb-scan-directories=$MYSQLD_DATADIR". -# - we have messages like -# [Note] InnoDB: Tablespace file './db_test_a/t1.ibd' ID mismatch, -# expected but found -# or others or not at all. -# It could be tried to get control over the checkpointing via -# SET GLOBAL innodb_log_checkpoint_now = ON; -# SET GLOBAL innodb_checkpoint_disabled = 1; -# but that leads often to sideeffects like that the server aborts because -# there is no more free space in the log or similar. -# -# Hints -# ----- -# 1. It is NOT the goal of the current test to cover all scenarios/DDLs where -# tablespaces.open.* files get used and might be harmed like described -# above. Just two main reasons: -# - Cramming many purposes into one test makes the test huge and vulnerable -# against mistakes in many areas outside of the current WL9499. -# - As long as the implementation of the new data dictionary is not finished -# we would suffer from many problems with known reason. -# Therefore we use mostly the move of tables between tablespaces only! -# 2. "search_pattern*.inc" gets used and not "assert_grep.inc" because -# the latter requires that we know the number of hits ($assert_count) -# in advance. But during test development were several cases where the number -# of hits was unstable but with guarantee > 0. -# The use of "search_pattern.inc" for processing the server error log -# forces to delete that log before the next restart attempt in order to get -# rid of warnings, error, notes belonging to the previous server startup. -# 3. Certain "UPDATE db_test_a.t_aux SET col2 = .." (autocommit is on) -# and also moves of tables to other tablespaces serve only to achieve that -# InnoDB processes "tablespaces.open.*" files during restart. -# During experimenting were cases where this did not happen. -# 4. The often used sequence CREATE TABLE, "test it", DROP TABLE might look -# "oversized" because we could reuse that already created table. -# But it ensures that any "test it" meets a "fresh" table and so we cannot -# suffer from some previous not detected error. It is the task of other -# tests to detect such errors. -# -################################################################################ - -# We use SET ... DEBUG ... ---source include/have_debug.inc - -# With valgrind the test falls asleep after the first injected crash. ---source include/not_valgrind.inc - -# The test is not compatible with binlogging ---source include/not_log_bin.inc - -# In case you are interested in the full content of the server error logs than -# set script_debug to 1. ---let $script_debug= 0; - ---echo ### Build infrastructure required for the current test ------------ START -# Half arbitrary decision to go with 'traditional' -SET SQL_MODE = 'traditional'; - -SET GLOBAL innodb_log_checkpoint_now = ON; - -# SCHEMAs and a TABLESPACE used within the current test. -CREATE SCHEMA db_test_a; -CREATE TABLESPACE ts1 ADD DATAFILE 'ts1.ibd' Engine = InnoDB; -CREATE TABLE db_test_a.t_aux (col1 INT, col2 VARCHAR(100)) Engine = InnoDB; -let $flip_count= 0; -let $max_num= 10; -let $num= $max_num; -while($num) -{ - eval - CREATE TABLE db_test_a.t_aux_$num (col1 INT) - Engine = InnoDB TABLESPACE = ts1; - eval - INSERT INTO db_test_a.t_aux - SET col2 = CONCAT('$flip_count-', REPEAT('z', 90)), col1 = $num; - dec $num; -} -SHOW CREATE TABLE db_test_a.t_aux_10; - -# There will be messages within the server error log which MTR needs to ignore -let $p= InnoDB: File '.*tablespaces.open..*' size is .*; -let $suppress_pattern= $p Must be at least; -eval CALL mtr.add_suppression("$suppress_pattern"); -let $p= InnoDB: File '.*tablespaces.open..*' size is .*; -let $suppress_pattern= $p should be at least; -eval CALL mtr.add_suppression("$suppress_pattern"); -# [ERROR] InnoDB: No space ID to filename mapping file found -let $p= InnoDB: No space ID to filename mapping file found; -let $suppress_pattern= $p; -eval CALL mtr.add_suppression("$suppress_pattern"); -# WL#9535 TODO: Remove this suppression. Currently, some mysql temporary -# tables could be left during a crashed ALTER TABLE. -let $p= InnoDB: Cannot load table db_test_a; -let $suppress_pattern= $p; -eval CALL mtr.add_suppression("$suppress_pattern"); - -# Frequent used statements and messages -let $create_table= CREATE TABLE db_test_a . t1 (col1 INT NOT NULL) - ENGINE = 'InnoDB' TABLESPACE = innodb_file_per_table; -let $check_cmd= SHOW CREATE TABLE db_test_a . t1; -let $drop_standard_table= DROP TABLE db_test_a . t1; -let $ddl_cmd= ALTER TABLE db_test_a . t1 TABLESPACE = ts1; - ---let $start_message= # Wait till the server crash is finished and the automatic ---let $start_message= $start_message restart with crash recovery has happened. ---let $comment_line0= # ======================================================== ---let $comment_line1= # Restart with crash recovery - -let $MYSQLD_DATADIR= `SELECT @@datadir`; -let $error_log= $MYSQLTEST_VARDIR/log/mysqld.1.err; -let SEARCH_FILE= $error_log; -let $tsof_copy_dir= $MYSQLTEST_VARDIR/tsof_copies; ---mkdir $tsof_copy_dir - ---echo # Make backups of the existing tablespaces.open.* files. ---echo # We will use these files later. ---source suite/innodb/include/tablespacesopen_1.inc -# Attention: -# I had cases with SET GLOBAL innodb_checkpoint_disabled = 1; around begin of -# the test. And than the shutdown aborted. ---source ./include/shutdown_mysqld.inc ---copy_files_wildcard $MYSQLD_DATADIR $tsof_copy_dir tablespaces.open.* ---source include/start_mysqld.inc -SHOW CREATE TABLE db_test_a.t_aux_10; - ---echo ### Build infrastructure required for the current test -------------- END - - ---echo $comment_line0 ---echo # Restart after previous smooth shutdown. ---echo # There is no valid tablespaces.open.* ---echo # If we find one we use that to verify the location of the UNDO ---echo # tablespaces. If we don't find one we don't verify the location ---echo # of the undo tablespaces. ---echo # The server just comes up without error messages mentioning that state. ---echo # No crash -> no look for tablespaces.open.* files ---source ./include/shutdown_mysqld.inc ---remove_files_wildcard $MYSQLD_DATADIR tablespaces.open.* ---remove_file $SEARCH_FILE ---source include/start_mysqld.inc -# The note which follows must be not found. ---let SEARCH_PATTERN= .* \[Note\] InnoDB: Starting crash recovery. ---source include/search_pattern.inc -# The note which follows must be found. ---let $p= .* \[Note\] InnoDB: Unable to read from '.*tablespaces.open.*', ---let SEARCH_PATTERN= $p the space ID to filename mapping file ---source include/search_pattern.inc -# The error message which follows must be found. ---let $p= .* \[ERROR\] InnoDB: No space ID to filename mapping file found ---let SEARCH_PATTERN= $p ---source include/search_pattern.inc - - ---echo $comment_line0 ---echo # Restart after previous ---echo # 1. Have no open transaction ---echo # 2. Enforce making an InnnoDB checkpoint ---echo # 3. Kill of server ---echo # There is no valid tablespaces.open.* file at all but we need none. ---echo # The server just comes up without error messages mentioning that state. ---echo # The impact of crash is so minimal that InnoDB does detect it at all. ---echo # -> no look for tablespaces.open.* files -SET GLOBAL innodb_log_checkpoint_now = ON; ---source include/expect_crash.inc ---source include/kill_mysqld.inc ---remove_files_wildcard $MYSQLD_DATADIR tablespaces.open.* ---remove_file $SEARCH_FILE ---source include/start_mysqld.inc -# The note which follows must be not found. ---let SEARCH_PATTERN= .* \[Note\] InnoDB: Starting crash recovery. ---source include/search_pattern.inc -# The note which follows must be not found. ---let $p= .* \[Note\] InnoDB: Unable to read from '.*tablespaces.open.*', ---let SEARCH_PATTERN= $p the space ID to filename mapping file ---source include/search_pattern.inc -# The error message which follows must be not found. ---let $p= .* \[ERROR\] InnoDB: No space ID to filename mapping file found ---let SEARCH_PATTERN= $p ---source include/search_pattern.inc - - ---echo $comment_line0 ---echo $comment_line1 ---echo # There is no tablespaces.open.* file but we need at least one valid. ---echo # A simple restart without extra options fails. -SET GLOBAL innodb_checkpoint_disabled = 1; ---source suite/innodb/include/tablespacesopen_1.inc ---source include/expect_crash.inc ---source include/kill_mysqld.inc ---remove_files_wildcard $MYSQLD_DATADIR tablespaces.open.* ---remove_file $SEARCH_FILE ---error 1 ---exec $MYSQLD_CMD --restart > $error_log 2>&1 ---echo # The restart attempt has failed like expected. -if($script_debug) -{ - --cat_file $SEARCH_FILE - # The restart attempt above might have created some tablespaces.open.* file. - # But that is not deterministic. - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -# The note which follows must be found. ---let SEARCH_PATTERN= .* \[Note\] InnoDB: Starting crash recovery. ---source include/search_pattern.inc -# The note which follows must be found. ---let $p= .* \[Note\] InnoDB: Unable to read from '.*tablespaces.open.*', ---let SEARCH_PATTERN= $p the space ID to filename mapping file ---source include/search_pattern.inc -# The error message which follows must be found. ---let $p= .* \[ERROR\] InnoDB: No space ID to filename mapping file found ---let SEARCH_PATTERN= $p ---source include/search_pattern.inc -# The next sequence is disabled because getting that message is unstable. -if(0) -{ -# [ERROR] InnoDB: Could not find any file associated with the tablespace 62 ---let $p= .* \[ERROR\] InnoDB: Could not find any file associated with ---let SEARCH_PATTERN= $p the tablespace .* ---source include/search_pattern.inc -} ---remove_file $SEARCH_FILE -# The restart attempt above might have created some tablespaces.open.* file. -# We remove it in order to make the test hopefully more deterministic. ---remove_files_wildcard $MYSQLD_DATADIR tablespaces.open.* ---let $restart_parameters= restart:--innodb-scan-directories=$MYSQLD_DATADIR ---replace_result $MYSQLD_DATADIR ---source include/start_mysqld.inc ---let $restart_parameters= restart -if($script_debug) -{ - --cat_file $SEARCH_FILE - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -# If the notes, warnings and error messages will show up again seems to depend -# on what InnoDb was able to achieve in the first restart attempt and this is -# unstable. So we do not check for them again. -SHOW CREATE TABLE db_test_a.t_aux_10; - - ---echo $comment_line0 ---echo $comment_line1 ---echo # There is no tablespaces.open.* file but a restart with ---echo # "--innodb-scan-directories=" ---echo # reports that bad state and passes because it does not rely ---echo # on at least one valid tablespaces.open.* file. -SET GLOBAL innodb_log_checkpoint_now = ON; -SET GLOBAL innodb_checkpoint_disabled = 1; ---source suite/innodb/include/tablespacesopen_1.inc ---source include/expect_crash.inc ---source include/kill_mysqld.inc ---remove_files_wildcard $MYSQLD_DATADIR tablespaces.open.* ---remove_file $SEARCH_FILE ---let $restart_parameters= restart:--innodb-scan-directories=$MYSQLD_DATADIR ---replace_result $MYSQLD_DATADIR ---source include/start_mysqld.inc ---let $restart_parameters= restart -if($script_debug) -{ - --cat_file $SEARCH_FILE - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -# The note which follows must be found. ---let SEARCH_PATTERN= .* \[Note\] InnoDB: Starting crash recovery. ---source include/search_pattern.inc -# The note which follows must be found. ---let $p= .* \[Note\] InnoDB: Unable to read from '.*tablespaces.open.*', ---let SEARCH_PATTERN= $p the space ID to filename mapping file ---source include/search_pattern.inc -# The error message which follows must be found. ---let $p= .* \[ERROR\] InnoDB: No space ID to filename mapping file found ---let SEARCH_PATTERN= $p ---source include/search_pattern.inc -SHOW CREATE TABLE db_test_a.t_aux_10; - - ---echo $comment_line0 ---echo $comment_line1 ---echo # One of the tablespaces.open.* with complete rotten entry. -# Take care that we have two clean tablespaces.open.* files. -# If not we could end up with one not existing, the other rotten, the -# automatic crash recovery without extra options needs at least one valid -# end therefore fails. -SET GLOBAL innodb_log_checkpoint_now = ON; -SET GLOBAL innodb_checkpoint_disabled = 1; ---source suite/innodb/include/tablespacesopen_1.inc -if($script_debug) -{ - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -eval $create_table; ---source include/expect_crash.inc -SET SESSION DEBUG = "+d,ib_tablespace_open_write_corrupt_0"; ---error CR_SERVER_LOST -eval $ddl_cmd; ---remove_file $SEARCH_FILE -# So even one valid '.*tablespaces.open.*' is not sufficient. ---error 2 ---exec $MYSQLD_CMD --restart > $error_log 2>&1 -if($script_debug) -{ - --cat_file $SEARCH_FILE - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} ---echo # The restart attempt has failed like expected. ---echo # So even one valid '.*tablespaces.open.*' is not sufficient for success. ---let SEARCH_PATTERN= .* \[Note\] InnoDB: Starting crash recovery. ---source include/search_pattern.inc -# [ERROR] InnoDB: Unsupported file format 808530483 found in tablespace ID to -# filename mapping file: '.*tablespaces.open.2'. You can use -# --innodb-scan-directories to recover if the tablespaces.open.* -# files are unreadable or corrupt. ---let $p= .* \[ERROR\] InnoDB: Unsupported file format .* found in tablespace ID ---let $p= $p to filename mapping file: '.*tablespaces.open.*'. ---let $p= $p You can use --innodb-scan-directories to recover .* ---let SEARCH_PATTERN= $p ---source include/search_pattern.inc -# The next message is too rigorous. -# At least one '.*tablespaces.open.*' file is valid. ---let $p= .* \[ERROR\] \[FATAL\] InnoDB: Unable to read the space ID to ---let SEARCH_PATTERN= $p filename mapping file\(s\). ---source include/search_pattern.inc ---let SEARCH_PATTERN= .* \[ERROR\] InnoDB: Assertion failure: ut0ut.cc: ---source include/search_pattern.inc ---remove_file $SEARCH_FILE ---echo # The crash recovery fails even if ---echo # "--innodb-scan-directories=" ---echo # is assigned as long as the tablespaces.open.* file with complete ---echo # rotten entry exists. So we delete all tablespaces.open.*. ---remove_files_wildcard $MYSQLD_DATADIR tablespaces.open.* ---let $restart_parameters= restart:--innodb-scan-directories=$MYSQLD_DATADIR ---replace_result $MYSQLD_DATADIR ---source include/start_mysqld.inc ---let $restart_parameters= restart -if($script_debug) -{ - --cat_file $SEARCH_FILE - # The restart attempt above might have created some tablespaces.open.* file. - # But that is not deterministic. So we will not check that. - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -eval $drop_standard_table; -SHOW CREATE TABLE db_test_a.t_aux_10; - - ---echo $comment_line0 ---echo $comment_line1 ---echo # One of the tablespaces.open.* with incomplete header ---source suite/innodb/include/tablespacesopen_1.inc -eval $create_table; ---source include/expect_crash.inc -SET SESSION DEBUG = "+d,ib_tablespace_open_write_corrupt_1"; ---error CR_SERVER_LOST -eval $ddl_cmd; ---remove_file $SEARCH_FILE ---source include/start_mysqld.inc -if($script_debug) -{ - --cat_file $SEARCH_FILE - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -# [Warning] InnoDB: File '.*tablespaces.open.2' size is 11 bytes. -# Must be at least 12 bytes ---let $p= .* \[Warning\] InnoDB: File '.*tablespaces.open.*' size is .* bytes. ---let $p= $p Must be at least .* bytes ---let SEARCH_PATTERN= $p ---echo # InnoDB warning about too small tablespaces.open.* file found in ---echo # server error log. ---source include/search_pattern.inc -eval $check_cmd; -eval $drop_standard_table; -SHOW CREATE TABLE db_test_a.t_aux_10; - - ---echo $comment_line0 ---echo $comment_line1 ---echo # One of the tablespaces.open.* with incomplete data part ---source suite/innodb/include/tablespacesopen_1.inc -eval $create_table; ---source include/expect_crash.inc -SET SESSION DEBUG = "+d,ib_tablespace_open_write_corrupt_2"; ---error CR_SERVER_LOST -eval $ddl_cmd; ---remove_file $SEARCH_FILE ---let $restart_parameters= restart ---source include/start_mysqld.inc -if($script_debug) -{ - --cat_file $SEARCH_FILE - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -# [Warning] InnoDB: File '.*tablespaces.open.1' size is 1342 bytes, -# should be at least 1343 bytes ---let $p= .* \[Warning\] InnoDB: File '.*tablespaces.open.*' size is .* bytes, ---let $p= $p should be at least .* bytes ---let SEARCH_PATTERN= $p ---source include/search_pattern.inc ---echo # InnoDB warning about too small tablespaces.open.* file found in ---echo # server error log. -eval $check_cmd; -eval $drop_standard_table; -SHOW CREATE TABLE db_test_a.t_aux_10; - - ---echo $comment_line0 ---echo $comment_line1 ---echo # Both tablespaces.open.* files are outdated. ---source suite/innodb/include/tablespacesopen_1.inc -eval $create_table; -eval $ddl_cmd; ---source include/expect_crash.inc ---source include/kill_mysqld.inc ---remove_files_wildcard $MYSQLD_DATADIR tablespaces.open.* ---copy_files_wildcard $tsof_copy_dir $MYSQLD_DATADIR tablespaces.open.* ---remove_file $SEARCH_FILE ---source include/start_mysqld.inc -if($script_debug) -{ - --cat_file $SEARCH_FILE - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -# There is no guarantee that the next error message shows up. -# Therefore the corresponding command sequence is disabled till some -# more reliable way is found. -if(0) -{ -# [Note] InnoDB: Tablespace file './db_test_a/t1.ibd' ID mismatch, -# expected 86 but found 102 ---let $p= .* \[Note\] InnoDB: Tablespace file './db_test_a/t1.ibd' ID mismatch, ---let $p= $p expected .* but found .* ---let SEARCH_PATTERN= $p ---source include/search_pattern.inc ---echo # InnoDB note about Tablespace file './db_test_a/t1.ibd' ID mismatches ---echo # found in server error log. -# There were several tables db_test_a . t1 TABLESPACE innodb_file_per_table -# (--> './db_test_a/t1.ibd') created, moved to general tablespace t1 with -# success or crash, but all time finally dropped in history. -} -eval $check_cmd; -eval $drop_standard_table; -SHOW CREATE TABLE db_test_a.t_aux_10; - - -# Intentional sideeffect of the next sub test which follows: -# Most probably one of the current tablespaces.open.* files is exceptional evil. -# In order to avoid any thinkable bad impact on successing tests we take care -# to get with guarantee new usable tablespaces.open.* files. ---echo $comment_line0 ---echo # Shutdown smooth ---source include/shutdown_mysqld.inc ---echo # Delete all tablespaces.open.* files. ---remove_files_wildcard $MYSQLD_DATADIR tablespaces.open.* ---remove_file $SEARCH_FILE ---echo # Restart with --innodb-scan-directories ---let $restart_parameters= restart:--innodb-scan-directories=$MYSQLD_DATADIR ---replace_result $MYSQLD_DATADIR ---source include/start_mysqld.inc ---let $restart_parameters= restart -if($script_debug) -{ - --cat_file $SEARCH_FILE - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} ---source suite/innodb/include/tablespacesopen_1.inc -SHOW CREATE TABLE db_test_a.t_aux_10; -if($script_debug) -{ - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} ---echo # We should have now one tablespaces.open.* file without defects. ---echo # Let's try the server crash just before tablespaces.open.* write. -eval $create_table; ---source include/expect_crash.inc -SET SESSION DEBUG = "+d,ib_tablespace_open_crash_before_write"; ---error CR_SERVER_LOST -eval $ddl_cmd; ---remove_file $SEARCH_FILE ---source include/start_mysqld.inc -if($script_debug) -{ - --cat_file $SEARCH_FILE - --error 0, 2 - --exec ls -ld $MYSQLD_DATADIR/tablespaces.open.* -} -eval $check_cmd; -eval $ddl_cmd; -eval $check_cmd; ---echo # We should have now two tablespaces.open.* files without defects. -eval $drop_standard_table; -SHOW CREATE TABLE db_test_a.t_aux_10; - -# Some checks -# ----------- -# $flip_count is the number of suite/innodb/include/tablespacesopen_1.inc calls. -# Sourcing suite/innodb/include/tablespacesopen_1.inc -# - changes the col2 values of 9 rows. -eval -SELECT COUNT(*) = 9 AS 'expect_1' FROM db_test_a.t_aux -WHERE col2 = CONCAT('$flip_count-', REPEAT('z', 90)); -# - changes the col2 value of the row with col1 = 1 but this gets reverted by -# ROLLBACK. So we should find in col2 the value from the INSERT. -SELECT COUNT(*) = 1 AS 'expect_1' FROM db_test_a.t_aux -WHERE col2 = CONCAT('0-', REPEAT('z', 90)) AND col1 = 1; -# - flips the tablespaces used by the db_test_a.t_aux_* tables from -# "innodb_file_per_table" to "gts1" and back. -let $next_ts= `SELECT IF(MOD($flip_count,2) = 0, - 'ts1', 'innodb_file_per_table')`; -let $show_val= `SHOW CREATE TABLE db_test_a.t_aux_10`; -eval SELECT '$show_val' LIKE '%$next_ts%' AS "expect_1"; - - -# Cleanup -# ------- -DROP SCHEMA db_test_a; -DROP TABLESPACE ts1; ---remove_files_wildcard $tsof_copy_dir tablespaces.open.* ---rmdir $tsof_copy_dir - diff --git a/mysql-test/suite/innodb/t/innodb_tsof_use_2.test b/mysql-test/suite/innodb/t/innodb_tsof_use_2.test deleted file mode 100644 index a11bab422cbc..000000000000 --- a/mysql-test/suite/innodb/t/innodb_tsof_use_2.test +++ /dev/null @@ -1,397 +0,0 @@ -# Check how the set of tablespace.open. files (WL#9499) gets used. -# ------------------------------------------------------------------------ -# 1. Per theory/design/... tablespaces.open.* files get written in case for -# example some InnoDB table created with TABLESPACE = innodb_file_per_table -# gets renamed. -# We use here that case and check hereby if the system does exact that. -# It is expected that n of such renames cause n of such writes. -# 2. The numbering of the tablespaces.open. files must be dense -# without holes. 1, 2, 3, .... -# 3. The system might add a tablespaces.open. file but it must never -# 4. The use of the tablespaces.open.* files should be round robin. -# --> Equal distribution of writes over the tablespaces.open.* files. -# Checking for this property alone cannot not reveal if its really round -# robin or all time round robin but -# - the equal distribution persists independent of the number of renames -# and the system does not know in advance how many renames will come -# - derived tests show that it seems to be exact round robin. - ---source include/have_debug.inc - -# The current test uses some a bit complicated infrastructure which makes -# trouble with the not that perfect implemented sp|cursor|view - protocols. ---source include/no_cursor_protocol.inc ---source include/no_sp_protocol.inc ---source include/no_view_protocol.inc -# -# The InnoDB tables belonging to the test infrastructure require quite big -# key sizes, column file_name VARCHAR(511), which InnoDB page sizes like 2k -# or 4k do not allow. -# Solutions which would remove that limitation or look interesting have often -# unfortunate side effects or do not help: -# We need up to ~ 10 characters for internal purposes. -# a) Don't use CHARSET UTF8 for the table. -# Risky because the information_schema tables which are used as source -# of data use UTF8. Trouble around string comparison is feared. -# b) Use a VARCHAR column significant smaller than 511 characters. -# Risky because -# - maximum observed: -# Currently at least '/' goes up till -# ~ 70 characters for some performance_schema tables -# - Maximum 1 in theory: -# 'table_schema' and 'table_name' in information_schema . table are -# both VARCHAR(64) --> 128 characters -# - Maximum 2 in theory: -# 'file_name' in information_schema . files is VARCHAR(4000). -# - who knows what we will get in future -# - in some standard MySQL setup (non user defined schemas and tables) -# - in some test within the current suite -# I guess the worst case is ~ 300 -# c) Have all the infrastructure tables in some tablespace which has -# its "own" sufficient big enough InnoDB page size no matter what the -# default page size is. Impossible in 5.7. -# d) Use an artificial key instead of 'file_name', normalize the tables -# and have "fun" with joins. IMHO just too expensive. ---source include/have_innodb_min_8k.inc - - -##### Script debugging -# Set $script_debug to 1 in case needing to debug the scripts. -# Far way more queries and result sets will get printed and so some diff to the -# file with expected results will show up. -# MTR will valuate the test run as "failed" because the file with the expected -# results was generated with $script_debug set to 0. -# Protocols of test runs with $script_debug set to 1 were neither -# - portable (MySQL setup on local box, especially paths) -# nor -# - reproducible (timestamps, LSN or MD5 values of it or ...) -let $script_debug= 0; - - ---disable_query_log -if($script_debug) -{ - --enable_query_log -} ---echo ### Build infrastructure typical for the current suite ------------ START -if (!$script_debug) -{ - --echo # In this phase printing of statements and responses is disabled. - --disable_query_log - --disable_result_log -} -let $MYSQLD_DATADIR= `SELECT @@datadir`; -let MYSQLD_DATADIR= $MYSQLD_DATADIR; - -# LOAD_FILE might be run later per tablespace.open. file. -# In order to be able to handle huge files of that kind max_allowed_packet -# must be big. The value will be flipped back at test end. -let $max_allowed_packet_save= `SELECT @@global.max_allowed_packet`; -SET GLOBAL max_allowed_packet = 1024 * 1024 * 64; - -# We want to discover if the set of tablespace.open. files gets changed -# by executing the query (usually DDL). In order to not measure the impact of -# the asynchronous InnoDB checkpoints we need to disable temporary the -# checkpointing. The value will be flipped back at test end. -let $innodb_checkpoint_disabled_save= `SELECT @@innodb_checkpoint_disabled`; - - -# SCHEMA for keeping the infrastructure (base tables and views) of the test -# ------------------------------------------------------------------------- -# Advantage: Simplified cleanup via dropping that schema at test end. -CREATE SCHEMA test_infrastructure; -USE test_infrastructure; - -# Points within the work flow where data could be collected -- Variables -# ---------------------------------------------------------------------- -# Before starting the query of interest. -let $point_before= ' before'; -# After the query of interest crashed the server and before restarting the -# server which invokes automatic crash recovery. -let $point_postcrash= 'postcrash'; -# After finishing the query of interest without crash or after restarting the -# server (automatic crash recovery is finished). -let $point_after= ' after'; -# User defined point 'baseline' is optional and can be used for freezing one -# state we later refer to during testing. -let $point_baseline= ' baseline'; -let $point_list= $point_before, $point_postcrash, $point_after, $point_baseline; - -# WL#9535 TODO: Remove this suppression. Currently, some mysql temporary -# tables could be left during a crashed ALTER TABLE. -let $p= InnoDB: Cannot load table db_test_a; -let $suppress_pattern= $p; -eval CALL mtr.add_suppression("$suppress_pattern"); - -# For the tablespaces.open. files used by InnoDB. -eval -CREATE TABLE tablespaces_open_content ( - when_taken SET($point_list) NOT NULL, - object VARCHAR(511), - md5_value CHAR(32), - PRIMARY KEY(object, when_taken) -) ENGINE = InnoDB CHARSET = UTF8; - -CREATE TABLE tablespaces_open_files ( - object VARCHAR(511), - counter1 INT DEFAULT 0, - counter2 INT DEFAULT 0, - PRIMARY KEY(object) -) ENGINE = InnoDB CHARSET = UTF8; - -CREATE VIEW tablespaces_open_content_diff_a_b AS -SELECT ' added' AS state, object FROM tablespaces_open_content -WHERE when_taken = ' after' - AND object NOT IN (SELECT object FROM tablespaces_open_content - WHERE when_taken = ' before') -UNION -SELECT 'dropped' AS state, object FROM tablespaces_open_content -WHERE when_taken = ' before' - AND object NOT IN (SELECT object FROM tablespaces_open_content - WHERE when_taken = ' after') -UNION -SELECT 'changed' AS state, object FROM tablespaces_open_content AS t1 -WHERE when_taken = ' before' - AND EXISTS (SELECT 1 FROM tablespaces_open_content - WHERE when_taken = ' after' - AND object = t1.object AND md5_value <> t1.md5_value); - - -# tablespaces.open.* files get never dropped! -# The AND EXISTS ... makes the view safe against too early use. -CREATE VIEW tsof_dropped AS -SELECT object FROM tablespaces_open_files -WHERE object NOT IN (SELECT object FROM tablespaces_open_content) - AND EXISTS (SELECT 1 FROM tablespaces_open_content); -CREATE VIEW tsof_added AS -SELECT DISTINCT(object) FROM tablespaces_open_content -WHERE object NOT IN (SELECT object FROM tablespaces_open_files); - -# Create a directory for storing copies of the tablespaces.open.* files ---mkdir $MYSQLTEST_VARDIR/tsof_copies - - -# Set the file system/OS specific line terminator -# ----------------------------------------------- -# It gets currently used in suite/innodb/include/load_tsof_data.inc. -let $line_terminator= '\n'; -if (`SELECT CONVERT(@@version_compile_os USING latin1) - IN ('Win32','Win64','Windows') = 1`) -{ - # When running on WIN than the Perl sniplet above has produced the - # WIN typic '\r\n' at line end. - let $line_terminator= '\r\n'; -} - - ---enable_query_log ---enable_result_log ---echo ### Build infrastructure typical for the current suite -------------- END - -CREATE SCHEMA db_test_a; -CREATE TABLE db_test_a . t0 (col1 INT) TABLESPACE = innodb_file_per_table -ENGINE = InnoDB; ---disable_query_log -if($script_debug) -{ - --enable_query_log -} - - -# Collect the current state -let $when_taken= $point_before; ---source suite/innodb/include/load_tsof_data.inc ---disable_query_log -if($script_debug) -{ - --enable_query_log -} - -SET GLOBAL innodb_log_checkpoint_now = ON; -SET GLOBAL innodb_checkpoint_disabled = 1; - -# We are before beginning a sub test/beginning the measurement. -# So reset the columns "counter*" to 0. -UPDATE tablespaces_open_files SET counter1 = 0, counter2 = 0; -COMMIT; - -let $rounds= 37; -let $t_num= 0; -let $t_num_up= 1; -let $has_failed= 0; ---echo # Run a big DDL round for the table "db_test_a . t0". ---echo # RENAME TABLE t0 -> t1 -> ... -> ---echo # Caused by using TABLESPACE = innodb_file_per_table tablespaces.open.* ---echo # files need to be used. -if(!$script_debug) -{ - --echo # In this phase printing of statements and responses is disabled. -} -while ($t_num < $rounds) -{ - let $source_table= t$t_num; - let $target_table= t$t_num_up; - eval RENAME TABLE db_test_a . $source_table TO db_test_a . $target_table; - let $when_taken= $point_after; - --source suite/innodb/include/load_tsof_data.inc - --disable_query_log - # If enabled the next block allows to see which tablespaces.open.* file was - # used for the last RENAME. - # 2016-11 : It seems to be round robin and - # - the number of the tablespaces.open.* file used first depends on - # server history since restart or similar ==> not good predictable - # - the queue is like ..., min, min+1, ..., max-1, max, min, min+1, ... - # - at some point of time some additional tablespaces.open. - # might show up. This will not change the logics in the queue. - if ($script_debug) - { - --enable_query_log - SELECT * FROM tablespaces_open_content_diff_a_b - WHERE state IN ('changed', 'added'); - --disable_query_log - } - # The BETWEEN $t_num_up AND $t_num_up + ... was introduced because of the - # following observation (2016-11) during high load tests: - # - In most cases only one tablespaces.open.* file was touched per one DDL - # and "Number of touches == number of finished loops" was valid. - # - In rare cases two tablespaces.open.* files were touched per one DDL. - # Variants: - # - Two tablespaces.open.* files changed. - # - One tablespaces.open.* file changed and one added. - # Impact: Number of touches >= number of finished loops. - # - When that happens in the 37 rounds if at all is unstable. - # Cases observed: - # - In first loop --> Minimal diff if any at all is 1! - # - In middle loop - # - In last loop - # - After the fix the instability was no more met. - # I guess that there is some natural reason ala InnoDB performs some - # asynchronous action which requires that some tablespaces.open.* must - # get touched exact for that action. Savepoint? - # Such asynchronous actions are usually triggered by the amount of - # - time elapsed since last action - # - left over free resources (last action freed a lot) - # - data requiring some flush somewhere in future (last action flushed) - # Their number during running the current test depends most probably on - # the number of loops finished. So some linear growth is quite likely. - let $pass= `SELECT SUM(counter1) BETWEEN $t_num_up - AND $t_num_up + 1 - FROM tablespaces_open_files`; - # For debugging: Simulate an error - if(0) - { - if($t_num == 10) - { - --echo # ERROR_INJECTION - let $pass= 0; - } - } - if (!$pass) - { - let $has_failed= 1; - --echo # FAIL: The number of "touches" of tablespaces.open.* files does - --echo # not equal the number of DDLs + small overhead. - # Enforce that we leave the loop - let $rounds= 0; - } - let $pass= `SELECT MAX(counter2) BETWEEN MIN(counter2) AND MIN(counter2) + 1 - FROM tablespaces_open_files`; - if (!$pass) - { - let $has_failed= 1; - --echo # FAIL: The number of DDLs does not cause some equal distributed - --echo # number of touches of tablespaces.open.* files. - --echo # MAX(counter2) is NOT BETWEEN MIN(counter2) - --echo # AND MIN(counter2) + 1 - # Enforce that we leave the loop - let $rounds= 0; - } - # For excessive experimenting/observing - if (0) - { - if(`SELECT COUNT(*) > 1 FROM tablespaces_open_content_diff_a_b - WHERE state IN ('changed', 'added')`) - { - --echo # INFO: More than one touched tablespaces.open.* file found - SELECT * FROM tablespaces_open_content_diff_a_b - WHERE state IN ('changed', 'added'); - --echo # Number of DDLs(RENAME TABLE ...) executed : $t_num_up - } - } - inc $t_num; - inc $t_num_up; - - eval - DELETE FROM test_infrastructure . tablespaces_open_content - WHERE when_taken = $point_before; - eval - UPDATE test_infrastructure . tablespaces_open_content - SET when_taken = $point_before - WHERE when_taken = $point_after; - COMMIT; -} ---enable_query_log - -let $print_details= $script_debug; -if ($has_failed) -{ - let $print_details= 1; -} -if($print_details) -{ - --echo # Test details ======================================= - --echo # Number of DDLs(RENAME TABLE ...) executed : $t_num - let $val= `SELECT SUM(counter1) FROM tablespaces_open_files`; - --echo # Total (all files) number of tablespaces.open.* touches: $val - let $val= `SELECT MAX(counter2) FROM tablespaces_open_files`; - --echo # Max number of touches per tablespaces.open.* per file : $val - let $val= `SELECT Min(counter2) FROM tablespaces_open_files`; - --echo # Min number of touches per tablespaces.open.* per file : $val - --echo # Aggregate bookkeeping table (tablespaces_open_files) ----- - SELECT SUM(counter1), MAX(counter2), MIN(counter2) - FROM tablespaces_open_files; - SELECT * FROM tablespaces_open_files ORDER BY file_name; - --echo # Sub test bookkeeping table (tablespaces_open_content) ---- - SELECT * FROM tablespaces_open_content - WHERE when_taken = 'after' - ORDER BY file_name; - --echo # tablespaces.open.* below data directory ------------------ - --list_files $MYSQLD_DATADIR/ tablespaces.open.* -} -if ($has_failed) -{ - exit; -} -if (!$has_failed) -{ - --echo # For all checks : PASS - --echo # 1. Every RENAME causes that the set of tablespaces.open.* files gets - --echo # modified (modify such a file or rather rare add one). - --echo # 2. No tablespaces.open.* file was ever deleted. - --echo # 3. The numbering of tablespaces.open.* files is dense 1,2,... - --echo # 4. The tablespaces.open.* files are used round robin. -} ---replace_result $target_table -eval DROP TABLE db_test_a . $target_table; - ---echo # Cleanup - ---disable_query_log -if ($script_debug) -{ - --enable_query_log -} -USE test; -eval -SET GLOBAL max_allowed_packet = $max_allowed_packet_save; -eval -SET GLOBAL innodb_checkpoint_disabled = $innodb_checkpoint_disabled_save; ---disable_warnings -DROP SCHEMA IF EXISTS db_test_a; -DROP SCHEMA IF EXISTS db_test_b; -DROP SCHEMA test_infrastructure; ---enable_query_log ---remove_files_wildcard $MYSQLTEST_VARDIR/tsof_copies * ---rmdir $MYSQLTEST_VARDIR/tsof_copies - - diff --git a/mysql-test/suite/innodb/t/log_alter_table.test b/mysql-test/suite/innodb/t/log_alter_table.test index 5f6064f7d3ba..8f3144f4fa17 100644 --- a/mysql-test/suite/innodb/t/log_alter_table.test +++ b/mysql-test/suite/innodb/t/log_alter_table.test @@ -28,11 +28,11 @@ ALTER TABLE t1 DROP INDEX b, ADD INDEX (b); --source include/start_mysqld.inc let SEARCH_FILE = $MYSQLTEST_VARDIR/log/mysqld.1.err; -# Look for at least one MLOG_FILE_CREATE2 in the error log. +# Look for at least one MLOG_FILE_CREATE in the error log. # Theoretically, it may have been written by this test or an earlier test. # FIXME: redirect the error log of the restart to a new file, # and ensure that we have exactly 2 records there. -let SEARCH_PATTERN=scan .*:.*log rec MLOG_FILE_CREATE2.*page .*:0; +let SEARCH_PATTERN=scan .*:.*log rec MLOG_FILE_CREATE.*page .*:0; --source include/search_pattern.inc # Look for at least one MLOG_INDEX_LOAD in the error log. # Theoretically, it may have been written by this test or an earlier test. diff --git a/mysql-test/suite/innodb/t/log_based_discovery.test b/mysql-test/suite/innodb/t/log_based_discovery.test deleted file mode 100644 index fb96bffe78ab..000000000000 --- a/mysql-test/suite/innodb/t/log_based_discovery.test +++ /dev/null @@ -1,155 +0,0 @@ -# This testcase will test the functionality of DB inconsistency check -# for ibdata* and undo* files. Create 2 DBs datadir1 & datadir2 in -# different paths Create ibdata* files and undo* files in different paths -# Crash the datadir1 DB server Swap the undo file path of the datadir1 -# to datadir2. Try to restrat the datadir1 server ,expect an error - ---source include/not_valgrind.inc ---source include/have_debug.inc - -let $MYSQLD_BASEDIR= `select @@basedir`; -let $START_PAGE_SIZE= `select @@innodb_page_size`; -let $LOG_FILE_SIZE= `select @@innodb_log_file_size`; - -# Create 2 different paths for ibdata* & undo* files both DBs ---mkdir $MYSQL_TMP_DIR/innodb_undo_data_dir1 ---mkdir $MYSQL_TMP_DIR/innodb_undo_data_dir2 ---mkdir $MYSQL_TMP_DIR/innodb_data_home_dir1 ---mkdir $MYSQL_TMP_DIR/innodb_data_home_dir2 ---mkdir $MYSQL_TMP_DIR/datadir1 ---mkdir $MYSQL_TMP_DIR/datadir2 ---mkdir $MYSQL_TMP_DIR/datadir1/redo ---mkdir $MYSQL_TMP_DIR/datadir2/redo - -# Set 2 different paths for --datadir -let $MYSQLD_DATA_DIR1 = $MYSQL_TMP_DIR/datadir1/data; -let $MYSQLD_DATA_DIR2 = $MYSQL_TMP_DIR/datadir2/data; - -# Set 2 different paths for undo* files. -let $MYSQLD_UNDO_DATA_DIR1 = $MYSQL_TMP_DIR/innodb_undo_data_dir1; -let $MYSQLD_UNDO_DATA_DIR2 = $MYSQL_TMP_DIR/innodb_undo_data_dir2; - -# Set 2 different paths for ibdata* files. -let $MYSQLD_HOME_DATA_DIR1 = $MYSQL_TMP_DIR/innodb_data_home_dir1; -let $MYSQLD_HOME_DATA_DIR2 = $MYSQL_TMP_DIR/innodb_data_home_dir2; - -# Set 2 different paths for the redo log (and tablespaces.open.*) files. -let $MYSQLD_REDO_DIR1 = $MYSQL_TMP_DIR/datadir1/redo; -let $MYSQLD_REDO_DIR2 = $MYSQL_TMP_DIR/datadir2/redo; - -let BOOTSTRAP_SQL=$MYSQL_TMP_DIR/boot.sql; - ---echo # create bootstrap file -write_file $BOOTSTRAP_SQL; -CREATE DATABASE test; -EOF - ---echo # Stop the MTR default DB server ---source include/shutdown_mysqld.inc - -# Set the bootstrap parameters for datadir1 -let NEW_CMD = $MYSQLD --no-defaults --innodb_dedicated_server=OFF --initialize-insecure --innodb_log_group_home_dir=$MYSQLD_REDO_DIR1 --innodb_log_file_size=$LOG_FILE_SIZE --innodb_page_size=$START_PAGE_SIZE --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR1 --innodb_undo_directory=$MYSQLD_UNDO_DATA_DIR1 --innodb_undo_tablespaces=5 --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATA_DIR1 --init-file=$BOOTSTRAP_SQL --secure-file-priv="" >$MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1; - ---echo # Run the bootstrap command of datadir1 ---exec $NEW_CMD - ---echo # Start the DB server with datadir1 ---disable_result_log ---let $restart_parameters="restart: --innodb_page_size=$START_PAGE_SIZE --innodb_log_group_home_dir=$MYSQLD_REDO_DIR1 --innodb_log_file_size=$LOG_FILE_SIZE --innodb_undo_tablespaces=5 --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR1 --innodb_undo_directory=$MYSQLD_UNDO_DATA_DIR1 --datadir=$MYSQLD_DATA_DIR1" ---source include/start_mysqld.inc ---enable_result_log - -CREATE TABLE tab1(c1 INT, c2 VARCHAR(30)); - -INSERT INTO tab1 VALUES(1, 'Test consistency undo*'); - -SELECT * FROM tab1; - ---echo # Stop the DB server with datadir1 ---source include/shutdown_mysqld.inc - -# Set the bootstrap parameters for datadir2 -let NEW_CMD = $MYSQLD --no-defaults --innodb_dedicated_server=OFF --initialize-insecure --innodb_page_size=$START_PAGE_SIZE --innodb_log_group_home_dir=$MYSQLD_REDO_DIR2 --innodb_log_file_size=$LOG_FILE_SIZE --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR2 --innodb_undo_directory=$MYSQLD_UNDO_DATA_DIR2 --innodb_undo_tablespaces=5 --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATA_DIR2 --init-file=$BOOTSTRAP_SQL --secure-file-priv="" >$MYSQLTEST_VARDIR/tmp/bootstrap.log 2>&1; - ---echo # Run the bootstrap command of datadir2 ---exec $NEW_CMD - ---echo # Start the DB server with datadir2 ---disable_result_log ---let $restart_parameters="restart: --innodb_page_size=$START_PAGE_SIZE --innodb_log_group_home_dir=$MYSQLD_REDO_DIR2 --innodb_log_file_size=$LOG_FILE_SIZE --innodb_undo_tablespaces=5 --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR2 --innodb_undo_directory=$MYSQLD_UNDO_DATA_DIR2 --datadir=$MYSQLD_DATA_DIR2" ---source include/start_mysqld.inc ---enable_result_log - -CREATE TABLE tab2(c1 INT, c2 VARCHAR(30)); - -INSERT INTO tab2 VALUES(1, 'Test consistency undo*'); - -SELECT * FROM tab2; - ---echo # Stop the DB server with datadir2 ---source include/shutdown_mysqld.inc - -# swap the undo* file path of datadir1 to datadir2 (interchange) -let NEW_CMD = $MYSQLD --no-defaults --innodb_dedicated_server=OFF --innodb_page_size=$START_PAGE_SIZE --innodb_log_group_home_dir=$MYSQLD_REDO_DIR1 --innodb_log_file_size=$LOG_FILE_SIZE --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR1 --innodb_undo_directory=$MYSQLD_UNDO_DATA_DIR2 --innodb_undo_tablespaces=5 --basedir=$MYSQLD_BASEDIR --datadir=$MYSQLD_DATA_DIR1 --secure-file-priv="" --unknown-setting --console >$MYSQLTEST_VARDIR/tmp/mysqld_log.err 2>&1; - ---echo # DB1 server started with the wrong path of undo tablespaces. ---echo # Expect errors during recovery. ---error 1,42 ---exec $NEW_CMD - -# Search for particular error -let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/mysqld_log.err; -let SEARCH_PATTERN=Cannot create .*undo_001 because .*undo_001 already uses Space ID=4294967279! Did you change innodb_undo_directory; - ---source include/search_pattern.inc ---remove_file $MYSQLTEST_VARDIR/tmp/mysqld_log.err - ---echo # Start the DB server with right path, expect no recovery errors ---disable_result_log ---let $restart_parameters="restart: --innodb_page_size=$START_PAGE_SIZE --innodb_log_group_home_dir=$MYSQLD_REDO_DIR1 --innodb_log_file_size=$LOG_FILE_SIZE --innodb_undo_tablespaces=5 --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR1 --innodb_undo_directory=$MYSQLD_UNDO_DATA_DIR1 --datadir=$MYSQLD_DATA_DIR1" ---source include/start_mysqld.inc ---enable_result_log - -# Check datadir1 consistency ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SELECT @@innodb_undo_tablespaces; - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SELECT @@innodb_data_home_dir; - ---replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR -SELECT @@innodb_undo_directory; - -SELECT * FROM tab1; -# Cleanup -DROP TABLE tab1; - ---echo # Stop the DB server with datadir1 ---source include/shutdown_mysqld.inc - ---echo # Start the DB server with datadir2 for cleanup ---disable_result_log ---let $restart_parameters="restart: --innodb_page_size=$START_PAGE_SIZE --innodb_log_group_home_dir=$MYSQLD_REDO_DIR2 --innodb_log_file_size=$LOG_FILE_SIZE --innodb_undo_tablespaces=5 --innodb_data_home_dir=$MYSQLD_HOME_DATA_DIR2 --innodb_undo_directory=$MYSQLD_UNDO_DATA_DIR2 --datadir=$MYSQLD_DATA_DIR2" ---source include/start_mysqld.inc ---enable_result_log - -SELECT * FROM tab2; - -# Cleanup -DROP TABLE tab2; - -# restart the server with MTR default ---let $restart_parameters= ---source include/restart_mysqld.inc - ---remove_file $BOOTSTRAP_SQL - -# Remove residue files ---force-rmdir $MYSQL_TMP_DIR/datadir1/redo ---force-rmdir $MYSQL_TMP_DIR/datadir2/redo ---force-rmdir $MYSQL_TMP_DIR/datadir1 ---force-rmdir $MYSQL_TMP_DIR/datadir2 ---force-rmdir $MYSQL_TMP_DIR/innodb_data_home_dir1 ---force-rmdir $MYSQL_TMP_DIR/innodb_data_home_dir2 ---force-rmdir $MYSQL_TMP_DIR/innodb_undo_data_dir1 ---force-rmdir $MYSQL_TMP_DIR/innodb_undo_data_dir2 diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test index 1a37a4d9b62d..6889762c1c59 100644 --- a/mysql-test/suite/innodb/t/log_corruption.test +++ b/mysql-test/suite/innodb/t/log_corruption.test @@ -8,22 +8,22 @@ EOF let SEARCH_FILE = $newdir/my_restart.err; let $args=--no-defaults --innodb_dedicated_server=OFF --datadir=$newdir --secure-file-priv="" --loose-skip-auto-generate-certs --loose-skip-sha256-password-auto-generate-rsa-keys --loose-console --loose-skip-log-bin --log-error-verbosity=3 > $SEARCH_FILE 2>&1 ; ---echo # redo log from before MySQL 5.7.9 +--echo # redo log from before we started versioning in MySQL 5.7.9 --exec unzip $MYSQL_TEST_DIR/suite/innodb/t/log_corruption.zip -d $newdir > $SEARCH_FILE --error 1,42 --exec $MYSQLD $args -let SEARCH_PATTERN=InnoDB: Unsupported redo log format\\. The redo log was created before MySQL 5\\.7\\.9\\.; +let SEARCH_PATTERN=InnoDB: Unsupported redo log format \(0\). The redo log was created before MySQL 5\.7\.9.*; --source include/search_pattern.inc ---echo # redo log from MySQL 5.7.9, with corrupted log block +--echo # redo log from before MySQL 8.0.3, with corrupted log block --remove_file $newdir/ib_logfile0 --exec unzip $MYSQL_TEST_DIR/suite/innodb/t/log_corruption0.zip -d $newdir > $SEARCH_FILE --error 1,42 --exec $MYSQLD $args -let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported. This redo log was created with malicious intentions, or perhaps, and it appears corrupted; +let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported.*; --source include/search_pattern.inc ---echo # redo log from "after" MySQL 5.7.9, but with invalid header checksum +--echo # redo log from MySQL 8.0.3, but with invalid header checksum --remove_file $newdir/ib_logfile0 --exec unzip $MYSQL_TEST_DIR/suite/innodb/t/log_corruption1.zip -d $newdir > $SEARCH_FILE --error 1,42 @@ -36,19 +36,19 @@ let SEARCH_PATTERN=InnoDB: Invalid redo log header checksum; --exec unzip $MYSQL_TEST_DIR/suite/innodb/t/log_corruption2.zip -d $newdir > $SEARCH_FILE --error 1,42 --exec $MYSQLD $args -let SEARCH_PATTERN=InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\\. Please follow the instructions at http://dev.mysql.com/doc/refman/.*/en/upgrading-downgrading.html; +let SEARCH_PATTERN=InnoDB: Unknown redo log format \(4294967295\).*; --source include/search_pattern.inc ---echo # redo log from MySQL 5.7.9, with corrupted log checkpoint +--echo # redo log from MySQL 8.0.3, with corrupted log checkpoint --remove_file $newdir/ib_logfile0 --exec unzip $MYSQL_TEST_DIR/suite/innodb/t/log_corruption3.zip -d $newdir > $SEARCH_FILE --error 1,42 --exec $MYSQLD $args -let SEARCH_PATTERN=InnoDB: No valid checkpoint found .corrupted redo log; +let SEARCH_PATTERN=InnoDB: No valid checkpoint found.*corrupted redo log; --source include/search_pattern.inc --echo # valid 5.7.9 header, valid checkpoint 1, all-zero (invalid) checkpoint 2, ---echo # invalid block checksum (MLOG_CHECKPOINT cannot be read), +--echo # invalid block checksum, --echo # thus we cannot determine if the redo log is logically clean. --remove_file $newdir/ib_logfile0 --exec unzip $MYSQL_TEST_DIR/suite/innodb/t/log_corruption4.zip -d $newdir > $SEARCH_FILE @@ -65,7 +65,7 @@ let SEARCH_PATTERN=InnoDB: Database upgrade cannot be accomplished with innodb_f let SEARCH_PATTERN=InnoDB: Database upgrade cannot be accomplished in read-only mode; --source include/search_pattern.inc ---echo # valid header, valid checkpoint 1, all-zero (invalid) checkpoint 2, invalid block number +--echo # valid 5.7.9 header, valid checkpoint 1, all-zero (invalid) checkpoint 2, invalid block number --remove_file $newdir/ib_logfile0 --exec unzip $MYSQL_TEST_DIR/suite/innodb/t/log_corruption4a.zip -d $newdir > $SEARCH_FILE # Anything below innodb_force_recovery=6 must find a valid redo log. @@ -82,7 +82,7 @@ let SEARCH_PATTERN=InnoDB: Database upgrade cannot be accomplished with innodb_f let SEARCH_PATTERN=InnoDB: Database upgrade cannot be accomplished in read-only mode; --source include/search_pattern.inc ---echo # valid 8.0.0 header, valid checkpoint 1, all-zero (invalid) checkpoint 2, invalid block checksum +--echo # valid 8.0.3 header, valid checkpoint 1, all-zero (invalid) checkpoint 2, invalid block checksum --remove_file $newdir/ib_logfile0 # Make server believe this is 8.0 datadir --move_file $newdir/mysql/plugin.frm $newdir/mysql/plugin.frm.old @@ -125,10 +125,10 @@ let SEARCH_PATTERN=InnoDB: Database upgrade cannot be accomplished in read-only --exec unzip $MYSQL_TEST_DIR/suite/innodb/t/log_corruption4d.zip -d $newdir > $SEARCH_FILE --error 1,42 --exec $MYSQLD $args --innodb-log-file-size=4M -let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported\\. This redo log was created with malicious intentions, or perhaps\\.; +let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported.*; --source include/search_pattern.inc -# Replace database with 1M redo logs from before MySQL 5.7.9 +# Replace database with 1M redo logs from before MySQL 8.0.3 --remove_file $newdir/ib_logfile0 --remove_file $newdir/ib_logfile1 --remove_file $newdir/ibdata1 diff --git a/mysql-test/suite/innodb/t/log_corruption_1.test b/mysql-test/suite/innodb/t/log_corruption_1.test index 3a3076d8689c..0414dd77ded3 100644 --- a/mysql-test/suite/innodb/t/log_corruption_1.test +++ b/mysql-test/suite/innodb/t/log_corruption_1.test @@ -1,7 +1,7 @@ # This test case tests if we handle the corrupted redo logs for dynamic metadata # of tables(corrupted index bit in this case) correctly. # Since we save the data and log files in advance, anything that will modify -# the redo logs( or data file) might make it fail. If it fails, we can do +# the redo logs (or data file) might make it fail. If it fails, we can do # as follow to re-generate the log_corruption_1.zip for it # 1. In CorruptedIndexPersister::write(), force it to write a bigger number, # such as 100 for the number of corrupted index diff --git a/mysql-test/suite/innodb/t/log_file_name.test b/mysql-test/suite/innodb/t/log_file_name.test deleted file mode 100644 index 72b78bccf90d..000000000000 --- a/mysql-test/suite/innodb/t/log_file_name.test +++ /dev/null @@ -1,292 +0,0 @@ ---echo # Test tablespace discovery during crash recovery ---echo # including the detection of duplicate tablespaces. - ---source include/have_innodb_max_16k.inc - ---source include/have_debug.inc ---source include/not_valgrind.inc - -let MYSQLD_DATADIR= `select @@datadir`; -let SEARCH_FILE= $MYSQLTEST_VARDIR/tmp/my_restart.err; -let $mysqld=$MYSQLD_CMD --core-file --console --log-error-verbosity=3 > $SEARCH_FILE 2>&1; - ---echo # Clear old log file ---source include/shutdown_mysqld.inc ---move_file $MYSQLTEST_VARDIR/log/mysqld.1.err $MYSQLTEST_VARDIR/tmp/mysqld_00.log 25 ---source include/start_mysqld.inc - --- echo # Do up some DDL and DML to recover -SET GLOBAL innodb_file_per_table=ON; -SET GLOBAL innodb_master_thread_disabled_debug=1; - -CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE t3(a INT PRIMARY KEY) ENGINE=InnoDB; - -# We want to force full recovery -SET GLOBAL innodb_checkpoint_disabled=1; - -BEGIN; -INSERT INTO t3 VALUES (2008),(08),(25); -INSERT INTO t1 VALUES (1964),(12),(25); -RENAME TABLE t1 TO t2; -UPDATE t2 SET a=2000 where a=1964; -COMMIT; - ---source include/kill_mysqld.inc - ---move_file $MYSQLTEST_VARDIR/log/mysqld.1.err $MYSQLTEST_VARDIR/tmp/mysqld_01.log - ---echo # Fault 0 (no real fault): Orphan file with duplicate space_id. ---copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t0.ibd - ---echo # Fault 1: Two dirty files with the same space_id. ---copy_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t1.ibd - ---echo # Attempt to start mysqld. Recovery will fail ---error 1,42 ---exec $mysqld - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_02.log - ---echo # Remove orphaned file from fault 0 ---remove_file $MYSQLD_DATADIR/test/t1.ibd - ---echo # Should startup fine ---source include/start_mysqld.inc - -DROP TABLE t2; -DROP TABLE t3; - -CREATE TABLE t1(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE t3(a INT PRIMARY KEY) ENGINE=InnoDB; - -# We want to force full recovery -SET GLOBAL innodb_checkpoint_disabled=1; - -BEGIN; -INSERT INTO t3 VALUES (2008),(08),(25); -INSERT INTO t1 VALUES (1964),(12),(25); -RENAME TABLE t1 TO t2; -UPDATE t2 SET a=2000 where a=1964; -COMMIT; - ---source include/kill_mysqld.inc - ---move_file $MYSQLTEST_VARDIR/log/mysqld.1.err $MYSQLTEST_VARDIR/tmp/mysqld_03.log - ---echo # Fault 2: Wrong space_id in a dirty file, and a missing file. ---move_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t1.ibd - ---echo # Attempt to start mysqld. Recovery will fail ---error 1,42 ---exec $mysqld - -let SEARCH_PATTERN= InnoDB: Tablespace file '.*t1.ibd' ID mismatch, expected \d+ but found \d+; ---source include/search_pattern.inc - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_04.log - ---echo # Fault 3: Wrong space_id in dirty file(s) ---move_file $MYSQLD_DATADIR/test/t1.ibd $MYSQLD_DATADIR/test/t3.ibd ---echo # Swap t2.ibd and t3.ibd. ---move_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t.ibd ---move_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t3.ibd ---move_file $MYSQLD_DATADIR/test/t.ibd $MYSQLD_DATADIR/test/t2.ibd - ---echo # Attempt to start mysqld. Recovery will fail ---error 1,42 ---exec $mysqld - -let SEARCH_PATTERN= InnoDB: Tablespace file '.*t2.ibd' ID mismatch, expected \d+ but found \d+; ---source include/search_pattern.inc - -let SEARCH_PATTERN= InnoDB: Tablespace file '.*t3.ibd' ID mismatch, expected \d+ but found \d+; ---source include/search_pattern.inc - -let SEARCH_PATTERN= InnoDB: Some files associated with the tablespace \d+ were not found: '.*t2.ibd'; ---source include/search_pattern.inc - -let SEARCH_PATTERN= InnoDB: Some files associated with the tablespace \d+ were not found: '.*t3.ibd'; ---source include/search_pattern.inc - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_05.log - ---echo # Swap back t3.ibd, but hide t2.ibd. ---move_file $MYSQLD_DATADIR/test/t3.ibd $MYSQLD_DATADIR/test/t.ibd ---move_file $MYSQLD_DATADIR/test/t2.ibd $MYSQLD_DATADIR/test/t3.ibd - ---echo # Fault 4: Missing data file - ---echo # Attempt to start mysqld. Recovery will fail ---error 1,42 ---exec $mysqld - -# The error message is the same as above. This is the additional message only. -let SEARCH_PATTERN= InnoDB: Some files associated with the tablespace \d+ were not found: '.*t2.ibd'; ---source include/search_pattern.inc - -# There should be one such message, for t3.ibd -let SEARCH_PATTERN= InnoDB: Some files associated with the tablespace \d+ were not found; ---source include/search_pattern.inc - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_06.log - ---echo # Fault 5: Empty data file - -# Create an empty t2.ibd ---exec echo "" > $MYSQLD_DATADIR/test/t2.ibd - ---echo # Attempt to start mysqld. Recovery will fail ---error 1,42 ---exec $mysqld - -let SEARCH_PATTERN= \[ERROR\] \[[^]]*\] InnoDB: Datafile .*t2.*\. Cannot determine the space ID from the first 64 pages; ---source include/search_pattern.inc - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_07.log - ---echo # Restore t2.ibd ---remove_file $MYSQLD_DATADIR/test/t2.ibd ---move_file $MYSQLD_DATADIR/test/t.ibd $MYSQLD_DATADIR/test/t2.ibd - ---source include/start_mysqld.inc - -SELECT * FROM t2; -SELECT * FROM t3; -SHOW TABLES; -DROP TABLE t2,t3; - -# The orphan file from fault 0 would not allow to create the table ---error ER_TABLESPACE_EXISTS -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -# Check if file was not removed ---error ER_TABLESPACE_EXISTS -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -# Remove the orphan file and try again. ---remove_file $MYSQLD_DATADIR/test/t0.ibd -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -DROP TABLE t0; - -# The existing directory shall not allow to create the table ---mkdir $MYSQLD_DATADIR/test/t0.ibd ---error ER_TABLESPACE_EXISTS -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -# Check if directory was not removed ---error ER_TABLESPACE_EXISTS -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; ---rmdir $MYSQLD_DATADIR/test/t0.ibd -# Remove the blocking dir and try again. -CREATE TABLE t0(a INT PRIMARY KEY) ENGINE=InnoDB; -DROP TABLE t0; - -# We want to force full recovery -SET GLOBAL innodb_master_thread_disabled_debug=1; -SET GLOBAL innodb_checkpoint_disabled=1; - -CREATE TABLE u1(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE u2(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE u3(a INT PRIMARY KEY) ENGINE=InnoDB; -CREATE TABLE u4(a INT PRIMARY KEY) ENGINE=InnoDB; -INSERT INTO u4 VALUES(1); -RENAME TABLE u4 TO u5; -RENAME TABLE u5 TO u6; -INSERT INTO u6 VALUES(2); - ---source include/kill_mysqld.inc - ---move_file $MYSQLTEST_VARDIR/log/mysqld.1.err $MYSQLTEST_VARDIR/tmp/mysqld_08.log - ---echo # Fault 6: Check file exists use case during RENAME ---echo # Copy u6.ibd -> u4.ibd ---copy_file $MYSQLD_DATADIR/test/u6.ibd $MYSQLD_DATADIR/test/u4.ibd - ---echo # We now have the original file and the final file (u4 -> u5 -> u6) ---echo # Recovery should should rename u4 -> u5 and abort on u5 -> u6 - ---echo # Attempt to start mysqld. Recovery will fail ---error 1,42 ---exec $mysqld - -let SEARCH_PATTERN= InnoDB: Attempted to open a previously opened tablespace. Previous tablespace .*u5 at filepath: .*u5.ibd uses space ID: \d+. Cannot open filepath: .*u6.ibd which uses the same space ID; ---source include/search_pattern.inc - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_09.log - ---echo # Fix the problem by removing u5.ibd ---remove_file $MYSQLD_DATADIR/test/u5.ibd - ---echo # Fault 7: Check all zero data file - ---echo # Remove u1.ibd, u2.ibd & u3.ibd ---remove_file $MYSQLD_DATADIR/test/u1.ibd ---remove_file $MYSQLD_DATADIR/test/u2.ibd ---remove_file $MYSQLD_DATADIR/test/u3.ibd - ---echo # Make the header page of u1.ibd consists of zero bytes -perl; -die unless open(FILE, ">$ENV{MYSQLD_DATADIR}/test/u1.ibd"); -print FILE "\0" x 16384; -close(FILE); -EOF - ---echo # Write "" to u2.ibd ---exec echo "" > $MYSQLD_DATADIR/test/u2.ibd - ---echo # Attempt to start mysqld. Recovery will fail ---error 1,42 ---exec $mysqld - -let SEARCH_PATTERN= \[ERROR\] \[[^]]*\] InnoDB: Datafile .*u1.*\. Cannot determine the space ID from the first 64 pages; ---source include/search_pattern.inc - -let SEARCH_PATTERN= \[ERROR\] \[[^]]*\] InnoDB: Cannot read first page of .*u2.ibd.*; ---source include/search_pattern.inc - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_10.log - ---echo # Fault 8: Missing or wrong data file - ---echo # Remove empty u2.ibd ---remove_file $MYSQLD_DATADIR/test/u2.ibd - ---echo # Attempt to start mysqld. Recovery will fail ---error 1,42 ---exec $mysqld - -let SEARCH_PATTERN= \[ERROR\] \[[^]]*\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd; ---source include/search_pattern.inc - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_11.log - ---echo # Remove empty u6.ibd ---remove_file $MYSQLD_DATADIR/test/u6.ibd - ---echo # Attempt to start mysqld with force-recovery=1. Recovery will fail ---error 1,42 ---exec $mysqld --innodb-force-recovery=1 --innodb-nonexistent-option - -let SEARCH_PATTERN= \[ERROR\] \[[^]]*\] unknown option '--innodb-nonexistent-option'; ---source include/search_pattern.inc - -let SEARCH_PATTERN= \[ERROR\] \[[^]]*\] InnoDB: Header page consists of zero bytes in datafile: .*u1.ibd; ---source include/search_pattern.inc - -let SEARCH_PATTERN= InnoDB: At LSN: \d+: unable to open file .*u[1-5].ibd for tablespace; ---source include/search_pattern.inc - ---move_file $SEARCH_FILE $MYSQLTEST_VARDIR/tmp/mysqld_12.log - -# Avoid assertion failure on DROP TABLE due to all-zero file - ---remove_file $MYSQLD_DATADIR/test/u1.ibd - ---source include/start_mysqld.inc - -DROP TABLE u1,u2,u3,u6; - ---echo # List of files: ---list_files $MYSQLD_DATADIR/test - -SHOW TABLES; -SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLES -WHERE NAME NOT LIKE 'SYS_%' AND NAME NOT LIKE 'sys/%' AND NAME NOT LIKE 'mysql/%'; diff --git a/mysql-test/suite/innodb/t/log_file_name_discover.test b/mysql-test/suite/innodb/t/log_file_name_discover.test index bce02d8c69b5..0eaa2b863c52 100644 --- a/mysql-test/suite/innodb/t/log_file_name_discover.test +++ b/mysql-test/suite/innodb/t/log_file_name_discover.test @@ -9,54 +9,37 @@ let MYSQLD_DATADIR= `select @@datadir`; SET GLOBAL innodb_file_per_table=ON; +CREATE DATABASE discover; +use discover; + # Stop the once a second checkpoint SET GLOBAL innodb_master_thread_disabled_debug=1; # Check if we can recover from a crash when a full checkpoint happens -# between the CREATE2 redo logging and the tablespace page init. +# between the CREATE redo logging and the tablespace page init. SET SESSION debug="+d,fil_ibd_create_log"; --source include/expect_crash.inc --error 2013 -CREATE TABLE t1(c INT) engine=InnoDB; +CREATE TABLE discover.t1(c INT) engine=InnoDB; --source include/start_mysqld.inc -# TODO: We have to use this because the two DDs get out of sync. -call mtr.add_suppression("InnoDB: Table .*test.*t1.*does not exist in the InnoDB internal data dictionary though MySQL is trying to drop it"); -# TODO: The physical is not remvoed either, related to above -DROP TABLE IF EXISTS t1; - -# Test --innodb-scan-directories -CREATE TABLE t2(c INT) ENGINE=InnoDB; +CREATE TABLE discover.t2(c INT) ENGINE=InnoDB; BEGIN; INSERT INTO t2 VALUES(1); INSERT INTO t2 SELECT * FROM t2; INSERT INTO t2 SELECT * FROM t2; INSERT INTO t2 SELECT * FROM t2; -INSERT INTO t2 SELECT * FROM t2; -INSERT INTO t2 SELECT * FROM t2; -INSERT INTO t2 SELECT * FROM t2; -INSERT INTO t2 SELECT * FROM t2; --source include/kill_mysqld.inc - ---echo "Removing tablespaces.open.* files" -# Force recovery from scanned directories ---remove_file $MYSQLD_DATADIR/tablespaces.open.1 ---remove_file $MYSQLD_DATADIR/tablespaces.open.2 - ---echo "Restarting with --innodb-scan-directories=." ---let $restart_parameters = restart: --innodb-scan-directories="."; --source include/start_mysqld.inc # Should be empty, transaction should be rolled back and the tablespace # should be found in the right place. SELECT * FROM t2; -DROP TABLE t2; - -call mtr.add_suppression("InnoDB: No space ID to filename mapping file found"); +DROP DATABASE discover; diff --git a/mysql-test/suite/innodb/t/log_file_size_checkpoint.test b/mysql-test/suite/innodb/t/log_file_size_checkpoint.test index e2b23cc47970..dc473f57c6e8 100644 --- a/mysql-test/suite/innodb/t/log_file_size_checkpoint.test +++ b/mysql-test/suite/innodb/t/log_file_size_checkpoint.test @@ -1,7 +1,7 @@ --source include/no_valgrind_without_big.inc let $n=250; -let $t=veryLongTableNameToCreateMLOG_FILE_NAMErecords; +let $t=very_long_table_name; --disable_query_log call mtr.add_suppression("InnoDB: Resizing redo log"); diff --git a/mysql-test/suite/innodb/t/missing_tablespaces.test b/mysql-test/suite/innodb/t/missing_tablespaces.test index cc72b54d823e..5d13b8d80b30 100644 --- a/mysql-test/suite/innodb/t/missing_tablespaces.test +++ b/mysql-test/suite/innodb/t/missing_tablespaces.test @@ -26,6 +26,7 @@ call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot open datafile f call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Could not find a valid tablespace file for"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace .* because it could not be opened"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot calculate statistics for table `\.\.*`\.`\.\.*` because the \.ibd file is missing"); +call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace .*, name '.*', file '.*' is missing!"); --enable_query_log --error ER_TABLESPACE_MISSING diff --git a/mysql-test/suite/innodb/t/readahead.test b/mysql-test/suite/innodb/t/readahead.test index d64dead699ca..539b1dcd5f94 100644 --- a/mysql-test/suite/innodb/t/readahead.test +++ b/mysql-test/suite/innodb/t/readahead.test @@ -16,5 +16,8 @@ EOF --source include/start_mysqld.inc SET GLOBAL innodb_random_read_ahead = 1; +call mtr.add_suppression("\\[ERROR\\].*InnoDB: '.*t1.ibd' read failed! - attempted to read 4 bytes, read only 0 bytes at offset 16418"); +call mtr.add_suppression("\\[Warning\\].*InnoDB: Ignoring '.*t1.ibd' invalid tablespace ID in the header"); + DROP TABLE t1; SET GLOBAL innodb_random_read_ahead = default; diff --git a/mysql-test/suite/innodb/t/sdi.test b/mysql-test/suite/innodb/t/sdi.test index d5f87f1a201a..b1195a176693 100644 --- a/mysql-test/suite/innodb/t/sdi.test +++ b/mysql-test/suite/innodb/t/sdi.test @@ -442,6 +442,9 @@ call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the sy call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot open datafile for read-only:"); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot find space for ts1 in tablespace memory cache"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot calculate statistics for table .* because the \.ibd file is missing"); +call mtr.add_suppression("InnoDB: Tablespace .*, name 'ts1', file 'ts1.ibd' is missing!"); +call mtr.add_suppression("InnoDB: Cannot find tablespace for 'ts1' in tablespace memory cache"); + --enable_query_log let $MYSQLD_DATADIR= `select @@datadir`; diff --git a/mysql-test/suite/innodb/t/table_encrypt_1.test b/mysql-test/suite/innodb/t/table_encrypt_1.test index fa8742c247e8..eb3974dd73d0 100644 --- a/mysql-test/suite/innodb/t/table_encrypt_1.test +++ b/mysql-test/suite/innodb/t/table_encrypt_1.test @@ -8,14 +8,15 @@ #let $KEYRING_PLUGIN = 'keyring_file.so'; --disable_query_log -call mtr.add_suppression("\\[Error\\] \\[[^]]*\\] InnoDB: Encryption can't find master key, please check the keyring plugin is loaded."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Failed to find tablespace for table `\.\.*`\.`\.\.*` in the cache."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number 2 in a file operation."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the system cannot find the path specified."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Could not find a valid tablespace file for"); +call mtr.add_suppression("\\[Error\\].*InnoDB: Encryption can't find master key, please check the keyring plugin is loaded."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Failed to find tablespace for table `\.\.*`\.`\.\.*` in the cache."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Operating system error number 2 in a file operation."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: The error means the system cannot find the path specified."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Could not find a valid tablespace file for"); call mtr.add_suppression("ibd can't be decrypted , please confirm the keyfile is match and keyring plugin is loaded."); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace .* because it could not be opened"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); +call mtr.add_suppression("\\[Warning\\].*InnoDB: Ignoring tablespace .* because it could not be opened"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Tablespace [0-9]+, name 'test.*t1', unable to open file '.*test.*t1.ibd' - Data structure corruption"); --enable_query_log # Create a table with encryption, should fail since keyring is not diff --git a/mysql-test/suite/innodb/t/table_encrypt_5.test b/mysql-test/suite/innodb/t/table_encrypt_5.test index 256a065fb0d7..6e811827d868 100644 --- a/mysql-test/suite/innodb/t/table_encrypt_5.test +++ b/mysql-test/suite/innodb/t/table_encrypt_5.test @@ -30,9 +30,10 @@ call mtr.add_suppression("InnoDB: Cannot open table tde_db/t_encrypt.* from the call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Table tde_db/t_encrypt.* in the InnoDB data dictionary has tablespace id .*, but tablespace with that id or name does not exist"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Please refer to .* for how to resolve the issue"); call mtr.add_suppression("Error while loading keyring content. The keyring might be malformed"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Plugin keyring_file reported: 'Could not create keyring directory"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Plugin keyring_file reported: 'keyring_file initialization failure."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Plugin keyring_file reported: 'File .*keyring' not found .*"); +call mtr.add_suppression("\\[ERROR\\].*Plugin keyring_file reported: 'Could not create keyring directory"); +call mtr.add_suppression("\\[ERROR\\].*Plugin keyring_file reported: 'keyring_file initialization failure."); +call mtr.add_suppression("\\[ERROR\\].*Plugin keyring_file reported: 'File .*keyring' not found .*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Tablespace [0-9]+, name 'tde_db.*t_encrypt.*', unable to open file '.*tde_db.*t_encrypt.*.ibd' - Data structure corruption"); -- enable_query_log let $innodb_file_per_table = `SELECT @@innodb_file_per_table`; diff --git a/mysql-test/suite/innodb/t/tablespace_per_table_not_windows.test b/mysql-test/suite/innodb/t/tablespace_per_table_not_windows.test index 7867e89ff63c..8cc78d5fb1e6 100644 --- a/mysql-test/suite/innodb/t/tablespace_per_table_not_windows.test +++ b/mysql-test/suite/innodb/t/tablespace_per_table_not_windows.test @@ -10,13 +10,16 @@ SET default_storage_engine=InnoDB; LET $MYSQLD_DATADIR = `select @@datadir`; ---echo # Suppress errors about too long filenames in error log. -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number .* in a file operation."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Error number .* means '.*'"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot create file .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'create' returned OS error .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'delete' returned OS error .*"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'stat' returned OS error .*"); +--disable_query_log +# Suppress errors about too long filenames in error log. +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Operating system error number .* in a file operation."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Error number .* means '.*'"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Cannot create file .*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: File .* 'create' returned OS error .*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: File .* 'delete' returned OS error .*"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: File .* 'stat' returned OS error .*"); +call mtr.add_suppression("\\[WARNING\\].*InnoDB: '.*' name too long, can't delete!"); +--enable_query_log --echo # --echo # MySQL limits each database and tablename identifier to 64 characters diff --git a/mysql-test/suite/innodb/t/tablespace_portability.test b/mysql-test/suite/innodb/t/tablespace_portability.test index 1225bfce5e78..183f623e7c6e 100644 --- a/mysql-test/suite/innodb/t/tablespace_portability.test +++ b/mysql-test/suite/innodb/t/tablespace_portability.test @@ -15,6 +15,16 @@ SET DEFAULT_STORAGE_ENGINE=InnoDB; +--disable_query_log +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +LET $restart_parameters = restart:--innodb-directories=$MYSQL_TMP_DIR; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--source include/restart_mysqld.inc +--enable_query_log + +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +SHOW VARIABLES LIKE 'innodb_directories'; + --echo # --echo # Try to create a tablespace in a read-only directory. --echo # @@ -30,9 +40,10 @@ SHOW WARNINGS; --rmdir $MYSQL_TMP_DIR/read_only_dir --disable_query_log -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means mysqld does not have the access rights to the directory."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot create file"); -call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Converting backslash to forward slash in ADD DATAFILE"); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: The error means mysqld does not have the access rights to the directory."); +call mtr.add_suppression("\\[ERROR\\].*InnoDB: Cannot create file"); +call mtr.add_suppression("\\[Warning\\].*InnoDB: Converting backslash to forward slash in ADD DATAFILE"); +call mtr.add_suppression("\\[Warning\\].*InnoDB: '.*' permission error, can't delete!"); --enable_query_log --source suite/innodb/include/tablespace_filenames.inc diff --git a/mysql-test/suite/innodb/t/tablespace_rename.test b/mysql-test/suite/innodb/t/tablespace_rename.test index 066dcedd8e1c..e4a86df3cead 100644 --- a/mysql-test/suite/innodb/t/tablespace_rename.test +++ b/mysql-test/suite/innodb/t/tablespace_rename.test @@ -5,19 +5,11 @@ # Crash between each of the points listed below : # - RENAME TABLE t to t1; -- gets translated into the following steps # -# 1. LSN = WRITE LOG "MLOG_FILE_OPEN t1.ibd;" +# 1. WRITE_LOG "MLOG_FILE_RENAME t, t1"; # -# 2. ADD_ENTRY_TO_TABLESPACE_ID_MAPPING (SPACE_ID, t1.ibd, LSN); +# 2. RENAME FILE t.ibd TO t1.ibd; # -# 3. WRITE MAPPING DATA TO FILE 'tablespaces.open.X' IN COMPRESSED FORMAT; -# -# 4. FSYNC 'tablespaces.open.X'; -# -# 5. WRITE_LOG "MLOG_FILE_RENAME2 t, t1"; -# -# 6. RENAME FILE t.ibd TO t1.ibd; -# -# 7. COMPLETED +# 3. COMPLETED # --source include/not_valgrind.inc @@ -46,7 +38,7 @@ SELECT * FROM t1; SHOW TABLES; --echo # ---echo # Test crash before writing MLOG_FILE_OPEN t2. +--echo # Test crash before writing MLOG_FILE_RENAME t2. --echo # On recovery the rename never happened. --echo # @@ -60,6 +52,7 @@ RENAME TABLE t1 TO t2; --source include/start_mysqld.inc SET SESSION debug="-d,ib_crash_rename_log_1"; + SHOW TABLES; INSERT INTO t1 VALUES (2); SELECT * FROM t1; diff --git a/mysql-test/suite/innodb_undo/include/i_s_files.inc b/mysql-test/suite/innodb_undo/include/i_s_files.inc index f2365750d1cf..a29ca4e88395 100644 --- a/mysql-test/suite/innodb_undo/include/i_s_files.inc +++ b/mysql-test/suite/innodb_undo/include/i_s_files.inc @@ -8,12 +8,14 @@ CREATE TABLE t1 (a serial, b text, c text); --replace_result ./ MYSQLD_DATADIR/ $MYSQLD_DATADIR/ MYSQLD_DATADIR/ $MYSQLD_DATADIR MYSQLD_DATADIR/ SELECT "==I_S.FILES==", FILE_ID, TABLESPACE_NAME, FILE_NAME, FILE_TYPE, STATUS, ENGINE, EXTENT_SIZE, MAXIMUM_SIZE, AUTOEXTEND_SIZE -FROM information_schema.files WHERE file_id = 0; +FROM information_schema.files WHERE file_id = 0 +ORDER BY FILE_ID; --replace_result ./ MYSQLD_DATADIR/ $MYSQLD_DATADIR/ MYSQLD_DATADIR/ $MYSQLD_DATADIR MYSQLD_DATADIR/ SELECT "==I_S.FILES==", FILE_ID, TABLESPACE_NAME, FILE_NAME, FILE_TYPE, STATUS, ENGINE, EXTENT_SIZE, MAXIMUM_SIZE -FROM information_schema.files WHERE tablespace_name RLIKE 'innodb_undo_00[1-2]'; +FROM information_schema.files WHERE tablespace_name RLIKE 'innodb_undo_00[1-2]' +ORDER BY FILE_ID; --replace_result ./ MYSQLD_DATADIR/ $MYSQLD_DATADIR/ MYSQLD_DATADIR/ $MYSQLD_DATADIR MYSQLD_DATADIR/ SELECT "==I_S.FILES==", TABLESPACE_NAME, FILE_NAME, FILE_TYPE, STATUS, diff --git a/mysql-test/suite/innodb_undo/r/i_s_files_16k.result b/mysql-test/suite/innodb_undo/r/i_s_files_16k.result index 36b4241b16cf..610668d51796 100644 --- a/mysql-test/suite/innodb_undo/r/i_s_files_16k.result +++ b/mysql-test/suite/innodb_undo/r/i_s_files_16k.result @@ -18,18 +18,18 @@ EXTENT_SIZE 1048576 MAXIMUM_SIZE NULL AUTOEXTEND_SIZE 67108864 ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967279 -TABLESPACE_NAME innodb_undo_001 -FILE_NAME MYSQLD_DATADIR/undo_001 +FILE_ID 4294967278 +TABLESPACE_NAME innodb_undo_002 +FILE_NAME MYSQLD_DATADIR/undo_002 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB EXTENT_SIZE 1048576 MAXIMUM_SIZE NULL ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967278 -TABLESPACE_NAME innodb_undo_002 -FILE_NAME MYSQLD_DATADIR/undo_002 +FILE_ID 4294967279 +TABLESPACE_NAME innodb_undo_001 +FILE_NAME MYSQLD_DATADIR/undo_001 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB diff --git a/mysql-test/suite/innodb_undo/r/i_s_files_32k.result b/mysql-test/suite/innodb_undo/r/i_s_files_32k.result index 3d7e3fb8ef2d..028b12986ed8 100644 --- a/mysql-test/suite/innodb_undo/r/i_s_files_32k.result +++ b/mysql-test/suite/innodb_undo/r/i_s_files_32k.result @@ -18,18 +18,18 @@ EXTENT_SIZE 2097152 MAXIMUM_SIZE NULL AUTOEXTEND_SIZE 67108864 ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967279 -TABLESPACE_NAME innodb_undo_001 -FILE_NAME MYSQLD_DATADIR/undo_001 +FILE_ID 4294967278 +TABLESPACE_NAME innodb_undo_002 +FILE_NAME MYSQLD_DATADIR/undo_002 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB EXTENT_SIZE 2097152 MAXIMUM_SIZE NULL ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967278 -TABLESPACE_NAME innodb_undo_002 -FILE_NAME MYSQLD_DATADIR/undo_002 +FILE_ID 4294967279 +TABLESPACE_NAME innodb_undo_001 +FILE_NAME MYSQLD_DATADIR/undo_001 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB diff --git a/mysql-test/suite/innodb_undo/r/i_s_files_4k.result b/mysql-test/suite/innodb_undo/r/i_s_files_4k.result index a7efa3e945a7..48b168774626 100644 --- a/mysql-test/suite/innodb_undo/r/i_s_files_4k.result +++ b/mysql-test/suite/innodb_undo/r/i_s_files_4k.result @@ -18,18 +18,18 @@ EXTENT_SIZE 1048576 MAXIMUM_SIZE NULL AUTOEXTEND_SIZE 67108864 ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967279 -TABLESPACE_NAME innodb_undo_001 -FILE_NAME MYSQLD_DATADIR/undo_001 +FILE_ID 4294967278 +TABLESPACE_NAME innodb_undo_002 +FILE_NAME MYSQLD_DATADIR/undo_002 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB EXTENT_SIZE 1048576 MAXIMUM_SIZE NULL ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967278 -TABLESPACE_NAME innodb_undo_002 -FILE_NAME MYSQLD_DATADIR/undo_002 +FILE_ID 4294967279 +TABLESPACE_NAME innodb_undo_001 +FILE_NAME MYSQLD_DATADIR/undo_001 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB diff --git a/mysql-test/suite/innodb_undo/r/i_s_files_64k.result b/mysql-test/suite/innodb_undo/r/i_s_files_64k.result index 05b5f7c03030..88a5db0bb8fa 100644 --- a/mysql-test/suite/innodb_undo/r/i_s_files_64k.result +++ b/mysql-test/suite/innodb_undo/r/i_s_files_64k.result @@ -18,18 +18,18 @@ EXTENT_SIZE 4194304 MAXIMUM_SIZE NULL AUTOEXTEND_SIZE 67108864 ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967279 -TABLESPACE_NAME innodb_undo_001 -FILE_NAME MYSQLD_DATADIR/undo_001 +FILE_ID 4294967278 +TABLESPACE_NAME innodb_undo_002 +FILE_NAME MYSQLD_DATADIR/undo_002 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB EXTENT_SIZE 4194304 MAXIMUM_SIZE NULL ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967278 -TABLESPACE_NAME innodb_undo_002 -FILE_NAME MYSQLD_DATADIR/undo_002 +FILE_ID 4294967279 +TABLESPACE_NAME innodb_undo_001 +FILE_NAME MYSQLD_DATADIR/undo_001 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB diff --git a/mysql-test/suite/innodb_undo/r/i_s_files_8k.result b/mysql-test/suite/innodb_undo/r/i_s_files_8k.result index cbb84fd5fda0..e61598d73ee0 100644 --- a/mysql-test/suite/innodb_undo/r/i_s_files_8k.result +++ b/mysql-test/suite/innodb_undo/r/i_s_files_8k.result @@ -18,18 +18,18 @@ EXTENT_SIZE 1048576 MAXIMUM_SIZE NULL AUTOEXTEND_SIZE 67108864 ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967279 -TABLESPACE_NAME innodb_undo_001 -FILE_NAME MYSQLD_DATADIR/undo_001 +FILE_ID 4294967278 +TABLESPACE_NAME innodb_undo_002 +FILE_NAME MYSQLD_DATADIR/undo_002 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB EXTENT_SIZE 1048576 MAXIMUM_SIZE NULL ==I_S.FILES== ==I_S.FILES== -FILE_ID 4294967278 -TABLESPACE_NAME innodb_undo_002 -FILE_NAME MYSQLD_DATADIR/undo_002 +FILE_ID 4294967279 +TABLESPACE_NAME innodb_undo_001 +FILE_NAME MYSQLD_DATADIR/undo_001 FILE_TYPE UNDO LOG STATUS NORMAL ENGINE InnoDB diff --git a/mysql-test/suite/innodb_undo/t/truncate_recover.test b/mysql-test/suite/innodb_undo/t/truncate_recover.test index 0627bb746703..4aadcc914f12 100644 --- a/mysql-test/suite/innodb_undo/t/truncate_recover.test +++ b/mysql-test/suite/innodb_undo/t/truncate_recover.test @@ -74,6 +74,7 @@ let SEARCH_PATTERN = ib_undo_trunc_before_ddl_log_end; --disable_query_log call mtr.add_suppression("InnoDB: Header page consists of zero bytes in datafile"); call mtr.add_suppression("InnoDB: Datafile '.*' is corrupted. Cannot determine the space ID from the first 64 pages"); +call mtr.add_suppression("InnoDB: Ignoring.*undo_002.*invalid tablespace ID in the header"); --enable_query_log # # diff --git a/mysql-test/suite/innodb_zip/r/restart.result b/mysql-test/suite/innodb_zip/r/restart.result index 9adca637a511..a88def413b36 100644 --- a/mysql-test/suite/innodb_zip/r/restart.result +++ b/mysql-test/suite/innodb_zip/r/restart.result @@ -3,6 +3,8 @@ SET default_storage_engine=InnoDB; # A series of tests to make sure tables are opened after restart. # Bug#13357607 Compressed file-per-table tablespaces fail to open # +CREATE DATABASE move_test; +USE move_test; set global innodb_file_per_table=on; # # Create and insert records into a REDUNDANT row formatted table. @@ -99,6 +101,7 @@ count(*) # # Create and insert records into a table that uses a remote DATA DIRECTORY. # +# CREATE TABLE t5_restart(c1 DOUBLE AUTO_INCREMENT KEY, c2 CHAR(10), c3 VARCHAR(100), c4 DATE, c5 TEXT) ROW_FORMAT=DYNAMIC ENGINE=InnoDB DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'; INSERT INTO t5_restart VALUES (1000000000, 'MySQL', 'InnoDB', '2011-11-11', 'Read this after reboot'); @@ -120,16 +123,14 @@ SELECT count(*) FROM t5_restart; count(*) 16 # -# Create and insert records into a partitioned table that uses -# a remote DATA DIRECTORY for each partition. -# +# Create and insert records into a partitioned table that uses-echo # a remote DATA DIRECTORY for each partition. CREATE TABLE t6_restart( c1 INT AUTO_INCREMENT KEY, c2 CHAR(10), c3 VARCHAR(100), c4 DATE, c5 TEXT) ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=2 ENGINE=InnoDB PARTITION BY HASH(c1) ( -PARTITION p0 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir', -PARTITION p1 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir', -PARTITION p2 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir'); +PARTITION p0 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir', +PARTITION p1 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir', +PARTITION p2 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'); INSERT INTO t6_restart VALUES (0, 'MySQL', 'InnoDB', '2011-11-11', 'Read this after reboot'); INSERT INTO t6_restart (SELECT 0, c2, c3, c4, c5 FROM t6_restart); INSERT INTO t6_restart (SELECT 0, c2, c3, c4, c5 FROM t6_restart); @@ -161,11 +162,11 @@ c1 INT AUTO_INCREMENT KEY, c2 CHAR(10), c3 VARCHAR(100), c4 DATE, c5 TEXT) ROW_FORMAT=DYNAMIC ENGINE=InnoDB PARTITION BY RANGE(c1) SUBPARTITION BY HASH(c1) ( PARTITION p0 VALUES LESS THAN (10) ( -SUBPARTITION s0 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir', -SUBPARTITION s1 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir'), +SUBPARTITION s0 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir', +SUBPARTITION s1 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir'), PARTITION p1 VALUES LESS THAN MAXVALUE ( -SUBPARTITION s2 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir', -SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir')); +SUBPARTITION s2 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir', +SUBPARTITION s3 DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir')); INSERT INTO t7_restart VALUES (0, 'MySQL', 'InnoDB', '2011-11-11', 'Read this after reboot'); INSERT INTO t7_restart (SELECT 0, c2, c3, c4, c5 FROM t7_restart); INSERT INTO t7_restart (SELECT 0, c2, c3, c4, c5 FROM t7_restart); @@ -241,61 +242,61 @@ count(*) # === information_schema.innodb_tables and innodb_tablespaces === Table Name Tablespace Table Flags Columns Row Format Zip Size Space Type -test/t1_restart test/t1_restart 0 8 Redundant 0 Single -test/t2_restart test/t2_restart 1 8 Compact 0 Single -test/t3_restart test/t3_restart 37 8 Compressed 2048 Single -test/t4_restart test/t4_restart 33 8 Dynamic 0 Single -test/t5_restart test/t5_restart 97 8 Dynamic 0 Single -test/t6_restart#p#p0 test/t6_restart#p#p0 101 8 Compressed 2048 Single -test/t6_restart#p#p1 test/t6_restart#p#p1 101 8 Compressed 2048 Single -test/t6_restart#p#p2 test/t6_restart#p#p2 101 8 Compressed 2048 Single -test/t7_restart#p#p0#sp#s0 test/t7_restart#p#p0#sp#s0 97 8 Dynamic 0 Single -test/t7_restart#p#p0#sp#s1 test/t7_restart#p#p0#sp#s1 97 8 Dynamic 0 Single -test/t7_restart#p#p1#sp#s2 test/t7_restart#p#p1#sp#s2 97 8 Dynamic 0 Single -test/t7_restart#p#p1#sp#s3 test/t7_restart#p#p1#sp#s3 97 8 Dynamic 0 Single -test/t8_restart s1_restart 129 8 Compact 0 General -test/t9_restart s1_restart 161 8 Dynamic 0 General +move_test/t1_restart move_test/t1_restart 0 8 Redundant 0 Single +move_test/t2_restart move_test/t2_restart 1 8 Compact 0 Single +move_test/t3_restart move_test/t3_restart 37 8 Compressed 2048 Single +move_test/t4_restart move_test/t4_restart 33 8 Dynamic 0 Single +move_test/t5_restart move_test/t5_restart 97 8 Dynamic 0 Single +move_test/t6_restart#p#p0 move_test/t6_restart#p#p0 101 8 Compressed 2048 Single +move_test/t6_restart#p#p1 move_test/t6_restart#p#p1 101 8 Compressed 2048 Single +move_test/t6_restart#p#p2 move_test/t6_restart#p#p2 101 8 Compressed 2048 Single +move_test/t7_restart#p#p0#sp#s0 move_test/t7_restart#p#p0#sp#s0 97 8 Dynamic 0 Single +move_test/t7_restart#p#p0#sp#s1 move_test/t7_restart#p#p0#sp#s1 97 8 Dynamic 0 Single +move_test/t7_restart#p#p1#sp#s2 move_test/t7_restart#p#p1#sp#s2 97 8 Dynamic 0 Single +move_test/t7_restart#p#p1#sp#s3 move_test/t7_restart#p#p1#sp#s3 97 8 Dynamic 0 Single +move_test/t8_restart s1_restart 129 8 Compact 0 General +move_test/t9_restart s1_restart 161 8 Dynamic 0 General === information_schema.innodb_tablespaces and innodb_datafiles === Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/t1_restart Single DEFAULT 0 Compact or Redundant MYSQLD_DATADIR/test/t1_restart.ibd -test/t2_restart Single DEFAULT 0 Compact or Redundant MYSQLD_DATADIR/test/t2_restart.ibd -test/t3_restart Single DEFAULT 2048 Compressed MYSQLD_DATADIR/test/t3_restart.ibd -test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/test/t4_restart.ibd -test/t5_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd -test/t6_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd -test/t6_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd -test/t6_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd -test/t7_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd -test/t7_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd -test/t7_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd -test/t7_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd +move_test/t1_restart Single DEFAULT 0 Compact or Redundant MYSQLD_DATADIR/move_test/t1_restart.ibd +move_test/t2_restart Single DEFAULT 0 Compact or Redundant MYSQLD_DATADIR/move_test/t2_restart.ibd +move_test/t3_restart Single DEFAULT 2048 Compressed MYSQLD_DATADIR/move_test/t3_restart.ibd +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t5_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t5_restart.ibd +move_test/t6_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p0.ibd +move_test/t6_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p1.ibd +move_test/t6_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p2.ibd +move_test/t7_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s0.ibd +move_test/t7_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s1.ibd +move_test/t7_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s2.ibd +move_test/t7_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s3.ibd s1_restart General DEFAULT 0 Any s1_restart.ibd === information_schema.files === Space_Name File_Type Engine Status Tablespace_Name Path -test/t1_restart TABLESPACE InnoDB NORMAL test/t1_restart MYSQLD_DATADIR/test/t1_restart.ibd -test/t2_restart TABLESPACE InnoDB NORMAL test/t2_restart MYSQLD_DATADIR/test/t2_restart.ibd -test/t3_restart TABLESPACE InnoDB NORMAL test/t3_restart MYSQLD_DATADIR/test/t3_restart.ibd -test/t4_restart TABLESPACE InnoDB NORMAL test/t4_restart MYSQLD_DATADIR/test/t4_restart.ibd -test/t5_restart TABLESPACE InnoDB NORMAL test/t5_restart MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd -test/t6_restart#p#p0 TABLESPACE InnoDB NORMAL test/t6_restart#p#p0 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd -test/t6_restart#p#p1 TABLESPACE InnoDB NORMAL test/t6_restart#p#p1 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd -test/t6_restart#p#p2 TABLESPACE InnoDB NORMAL test/t6_restart#p#p2 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd -test/t7_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL test/t7_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd -test/t7_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL test/t7_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd -test/t7_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL test/t7_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd -test/t7_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL test/t7_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd +move_test/t1_restart TABLESPACE InnoDB NORMAL move_test/t1_restart MYSQLD_DATADIR/move_test/t1_restart.ibd +move_test/t2_restart TABLESPACE InnoDB NORMAL move_test/t2_restart MYSQLD_DATADIR/move_test/t2_restart.ibd +move_test/t3_restart TABLESPACE InnoDB NORMAL move_test/t3_restart MYSQLD_DATADIR/move_test/t3_restart.ibd +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t5_restart TABLESPACE InnoDB NORMAL move_test/t5_restart MYSQL_TMP_DIR/alt_dir/move_test/t5_restart.ibd +move_test/t6_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p0 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p0.ibd +move_test/t6_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p1 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p1.ibd +move_test/t6_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p2 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p2.ibd +move_test/t7_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s0.ibd +move_test/t7_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s1.ibd +move_test/t7_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s2.ibd +move_test/t7_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s3.ibd s1_restart TABLESPACE InnoDB NORMAL s1_restart MYSQLD_DATADIR/s1_restart.ibd # # Shutdown the server and list the tablespace OS files # ----- MYSQL_DATA_DIR/test +---- MYSQL_DATA_DIR/move_test t1_restart.ibd t2_restart.ibd t3_restart.ibd t4_restart.ibd ----- MYSQL_TMP_DIR/alt_dir -test ----- MYSQL_TMP_DIR/alt_dir/test +---- ALT_DIR +move_test +---- ALT_DIR/move_test t5_restart.ibd t6_restart#p#p0.ibd t6_restart#p#p1.ibd @@ -306,8 +307,7 @@ t7_restart#p#p1#sp#s2.ibd t7_restart#p#p1#sp#s3.ibd # # Start the server and show that tables are still visible and accessible. -# -# restart +# restart:--innodb-directories=MYSQL_TMP_DIR/alt_dir SHOW VARIABLES LIKE 'innodb_file_per_table'; Variable_name Value innodb_file_per_table ON @@ -454,49 +454,49 @@ count(*) # === information_schema.innodb_tables and innodb_tablespaces === Table Name Tablespace Table Flags Columns Row Format Zip Size Space Type -test/t1_restart test/t1_restart 0 8 Redundant 0 Single -test/t2_restart test/t2_restart 1 8 Compact 0 Single -test/t3_restart test/t3_restart 37 8 Compressed 2048 Single -test/t4_restart test/t4_restart 33 8 Dynamic 0 Single -test/t5_restart test/t5_restart 97 8 Dynamic 0 Single -test/t6_restart#p#p0 test/t6_restart#p#p0 101 8 Compressed 2048 Single -test/t6_restart#p#p1 test/t6_restart#p#p1 101 8 Compressed 2048 Single -test/t6_restart#p#p2 test/t6_restart#p#p2 101 8 Compressed 2048 Single -test/t7_restart#p#p0#sp#s0 test/t7_restart#p#p0#sp#s0 97 8 Dynamic 0 Single -test/t7_restart#p#p0#sp#s1 test/t7_restart#p#p0#sp#s1 97 8 Dynamic 0 Single -test/t7_restart#p#p1#sp#s2 test/t7_restart#p#p1#sp#s2 97 8 Dynamic 0 Single -test/t7_restart#p#p1#sp#s3 test/t7_restart#p#p1#sp#s3 97 8 Dynamic 0 Single -test/t8_restart s1_restart 129 8 Compact 0 General -test/t9_restart s1_restart 161 8 Dynamic 0 General +move_test/t1_restart move_test/t1_restart 0 8 Redundant 0 Single +move_test/t2_restart move_test/t2_restart 1 8 Compact 0 Single +move_test/t3_restart move_test/t3_restart 37 8 Compressed 2048 Single +move_test/t4_restart move_test/t4_restart 33 8 Dynamic 0 Single +move_test/t5_restart move_test/t5_restart 97 8 Dynamic 0 Single +move_test/t6_restart#p#p0 move_test/t6_restart#p#p0 101 8 Compressed 2048 Single +move_test/t6_restart#p#p1 move_test/t6_restart#p#p1 101 8 Compressed 2048 Single +move_test/t6_restart#p#p2 move_test/t6_restart#p#p2 101 8 Compressed 2048 Single +move_test/t7_restart#p#p0#sp#s0 move_test/t7_restart#p#p0#sp#s0 97 8 Dynamic 0 Single +move_test/t7_restart#p#p0#sp#s1 move_test/t7_restart#p#p0#sp#s1 97 8 Dynamic 0 Single +move_test/t7_restart#p#p1#sp#s2 move_test/t7_restart#p#p1#sp#s2 97 8 Dynamic 0 Single +move_test/t7_restart#p#p1#sp#s3 move_test/t7_restart#p#p1#sp#s3 97 8 Dynamic 0 Single +move_test/t8_restart s1_restart 129 8 Compact 0 General +move_test/t9_restart s1_restart 161 8 Dynamic 0 General === information_schema.innodb_tablespaces and innodb_datafiles === Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/t1_restart Single DEFAULT 0 Compact or Redundant MYSQLD_DATADIR/test/t1_restart.ibd -test/t2_restart Single DEFAULT 0 Compact or Redundant MYSQLD_DATADIR/test/t2_restart.ibd -test/t3_restart Single DEFAULT 2048 Compressed MYSQLD_DATADIR/test/t3_restart.ibd -test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/test/t4_restart.ibd -test/t5_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd -test/t6_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd -test/t6_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd -test/t6_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd -test/t7_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd -test/t7_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd -test/t7_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd -test/t7_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd +move_test/t1_restart Single DEFAULT 0 Compact or Redundant MYSQLD_DATADIR/move_test/t1_restart.ibd +move_test/t2_restart Single DEFAULT 0 Compact or Redundant MYSQLD_DATADIR/move_test/t2_restart.ibd +move_test/t3_restart Single DEFAULT 2048 Compressed MYSQLD_DATADIR/move_test/t3_restart.ibd +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t5_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t5_restart.ibd +move_test/t6_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p0.ibd +move_test/t6_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p1.ibd +move_test/t6_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p2.ibd +move_test/t7_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s0.ibd +move_test/t7_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s1.ibd +move_test/t7_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s2.ibd +move_test/t7_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s3.ibd s1_restart General DEFAULT 0 Any s1_restart.ibd === information_schema.files === Space_Name File_Type Engine Status Tablespace_Name Path -test/t1_restart TABLESPACE InnoDB NORMAL test/t1_restart MYSQLD_DATADIR/test/t1_restart.ibd -test/t2_restart TABLESPACE InnoDB NORMAL test/t2_restart MYSQLD_DATADIR/test/t2_restart.ibd -test/t3_restart TABLESPACE InnoDB NORMAL test/t3_restart MYSQLD_DATADIR/test/t3_restart.ibd -test/t4_restart TABLESPACE InnoDB NORMAL test/t4_restart MYSQLD_DATADIR/test/t4_restart.ibd -test/t5_restart TABLESPACE InnoDB NORMAL test/t5_restart MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd -test/t6_restart#p#p0 TABLESPACE InnoDB NORMAL test/t6_restart#p#p0 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd -test/t6_restart#p#p1 TABLESPACE InnoDB NORMAL test/t6_restart#p#p1 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd -test/t6_restart#p#p2 TABLESPACE InnoDB NORMAL test/t6_restart#p#p2 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd -test/t7_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL test/t7_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd -test/t7_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL test/t7_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd -test/t7_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL test/t7_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd -test/t7_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL test/t7_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd +move_test/t1_restart TABLESPACE InnoDB NORMAL move_test/t1_restart MYSQLD_DATADIR/move_test/t1_restart.ibd +move_test/t2_restart TABLESPACE InnoDB NORMAL move_test/t2_restart MYSQLD_DATADIR/move_test/t2_restart.ibd +move_test/t3_restart TABLESPACE InnoDB NORMAL move_test/t3_restart MYSQLD_DATADIR/move_test/t3_restart.ibd +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t5_restart TABLESPACE InnoDB NORMAL move_test/t5_restart MYSQL_TMP_DIR/alt_dir/move_test/t5_restart.ibd +move_test/t6_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p0 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p0.ibd +move_test/t6_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p1 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p1.ibd +move_test/t6_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p2 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p2.ibd +move_test/t7_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s0.ibd +move_test/t7_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s1.ibd +move_test/t7_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s2.ibd +move_test/t7_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s3.ibd s1_restart TABLESPACE InnoDB NORMAL s1_restart MYSQLD_DATADIR/s1_restart.ibd DROP TABLE t1_restart; DROP TABLE t2_restart; @@ -512,26 +512,26 @@ ALTER TABLE t6_restart TRUNCATE PARTITION p2; ALTER TABLE t7_restart TRUNCATE PARTITION p1; === information_schema.innodb_tablespaces and innodb_datafiles === Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/test/t4_restart.ibd -test/t6_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd -test/t6_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd -test/t7_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd -test/t7_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd -test/t5_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd -test/t6_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd -test/t7_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd -test/t7_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t6_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p0.ibd +move_test/t6_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p1.ibd +move_test/t7_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s0.ibd +move_test/t7_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s1.ibd +move_test/t5_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t5_restart.ibd +move_test/t6_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p2.ibd +move_test/t7_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s2.ibd +move_test/t7_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s3.ibd === information_schema.files === Space_Name File_Type Engine Status Tablespace_Name Path -test/t4_restart TABLESPACE InnoDB NORMAL test/t4_restart MYSQLD_DATADIR/test/t4_restart.ibd -test/t6_restart#p#p0 TABLESPACE InnoDB NORMAL test/t6_restart#p#p0 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd -test/t6_restart#p#p1 TABLESPACE InnoDB NORMAL test/t6_restart#p#p1 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd -test/t7_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL test/t7_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd -test/t7_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL test/t7_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd -test/t5_restart TABLESPACE InnoDB NORMAL test/t5_restart MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd -test/t6_restart#p#p2 TABLESPACE InnoDB NORMAL test/t6_restart#p#p2 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd -test/t7_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL test/t7_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd -test/t7_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL test/t7_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t6_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p0 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p0.ibd +move_test/t6_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p1 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p1.ibd +move_test/t7_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s0.ibd +move_test/t7_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s1.ibd +move_test/t5_restart TABLESPACE InnoDB NORMAL move_test/t5_restart MYSQL_TMP_DIR/alt_dir/move_test/t5_restart.ibd +move_test/t6_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p2 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p2.ibd +move_test/t7_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s2.ibd +move_test/t7_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s3.ibd INSERT INTO t5_restart VALUES (1000000000, 'MySQL', 'InnoDB', '2011-11-11', 'Read this after reboot'); INSERT INTO t5_restart (SELECT 0, c2, c3, c4, c5 FROM t5_restart); INSERT INTO t5_restart (SELECT 0, c2, c3, c4, c5 FROM t5_restart); @@ -590,9 +590,9 @@ SUBPARTITION BY HASH (`c1`) # # Shutdown the server and make a backup of a tablespace # ----- MYSQL_DATA_DIR/test +---- MYSQL_DATA_DIR/move_test t4_restart.ibd ----- MYSQL_TMP_DIR/alt_dir/test +---- ALT_DIR/move_test t5_restart.ibd t5_restart.ibd.bak t6_restart#p#p0.ibd @@ -605,32 +605,32 @@ t7_restart#p#p1#sp#s3.ibd # # Start the server and show the tablespaces. # -# restart +# restart:--innodb-directories=MYSQL_TMP_DIR/alt_dir SHOW VARIABLES LIKE 'innodb_file_per_table'; Variable_name Value innodb_file_per_table ON === information_schema.innodb_tablespaces and innodb_datafiles === Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/test/t4_restart.ibd -test/t6_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd -test/t6_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd -test/t7_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd -test/t7_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd -test/t5_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd -test/t6_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd -test/t7_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd -test/t7_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t6_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p0.ibd +move_test/t6_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p1.ibd +move_test/t7_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s0.ibd +move_test/t7_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s1.ibd +move_test/t5_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t5_restart.ibd +move_test/t6_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p2.ibd +move_test/t7_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s2.ibd +move_test/t7_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s3.ibd === information_schema.files === Space_Name File_Type Engine Status Tablespace_Name Path -test/t4_restart TABLESPACE InnoDB NORMAL test/t4_restart MYSQLD_DATADIR/test/t4_restart.ibd -test/t6_restart#p#p0 TABLESPACE InnoDB NORMAL test/t6_restart#p#p0 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p0.ibd -test/t6_restart#p#p1 TABLESPACE InnoDB NORMAL test/t6_restart#p#p1 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p1.ibd -test/t7_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL test/t7_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s0.ibd -test/t7_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL test/t7_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p0#sp#s1.ibd -test/t5_restart TABLESPACE InnoDB NORMAL test/t5_restart MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd -test/t6_restart#p#p2 TABLESPACE InnoDB NORMAL test/t6_restart#p#p2 MYSQL_TMP_DIR/alt_dir/test/t6_restart#p#p2.ibd -test/t7_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL test/t7_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s2.ibd -test/t7_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL test/t7_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/test/t7_restart#p#p1#sp#s3.ibd +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t6_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p0 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p0.ibd +move_test/t6_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p1 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p1.ibd +move_test/t7_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s0.ibd +move_test/t7_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p0#sp#s1.ibd +move_test/t5_restart TABLESPACE InnoDB NORMAL move_test/t5_restart MYSQL_TMP_DIR/alt_dir/move_test/t5_restart.ibd +move_test/t6_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t6_restart#p#p2 MYSQL_TMP_DIR/alt_dir/move_test/t6_restart#p#p2.ibd +move_test/t7_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s2.ibd +move_test/t7_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t7_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/move_test/t7_restart#p#p1#sp#s3.ibd SELECT count(*) FROM t5_restart; count(*) 8 @@ -685,9 +685,9 @@ SUBPARTITION BY HASH (`c1`) # # Try to rename a tablespace to a file that already exists # ----- MYSQL_DATA_DIR/test +---- MYSQL_DATA_DIR/move_test t4_restart.ibd ----- MYSQL_TMP_DIR/alt_dir/test +---- ALT_DIR/move_test t5_restart.ibd t6_restart#p#p0.ibd t6_restart#p#p1.ibd @@ -704,26 +704,26 @@ RENAME TABLE t6_restart TO t66_restart; RENAME TABLE t7_restart TO t77_restart; === information_schema.innodb_tablespaces and innodb_datafiles === Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/test/t4_restart.ibd -test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd -test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd -test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd -test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd -test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd -test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd -test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd -test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s3.ibd === information_schema.files === Space_Name File_Type Engine Status Tablespace_Name Path -test/t4_restart TABLESPACE InnoDB NORMAL test/t4_restart MYSQLD_DATADIR/test/t4_restart.ibd -test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd -test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd -test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd -test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd -test/t55_restart TABLESPACE InnoDB NORMAL test/t55_restart MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd -test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd -test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd -test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart TABLESPACE InnoDB NORMAL move_test/t55_restart MYSQL_TMP_DIR/alt_dir/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s3.ibd INSERT INTO t55_restart (SELECT 0, c2, c3, c4, c5 FROM t55_restart); SELECT count(*) FROM t55_restart; count(*) @@ -778,9 +778,9 @@ SUBPARTITION BY HASH (`c1`) PARTITION p1 VALUES LESS THAN MAXVALUE (SUBPARTITION s2 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir' ENGINE = InnoDB, SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir' ENGINE = InnoDB)) */ ----- MYSQL_DATA_DIR/test +---- MYSQL_DATA_DIR/move_test t4_restart.ibd ----- MYSQL_TMP_DIR/alt_dir/test +---- ALT_DIR/move_test t55_restart.ibd t66_restart#p#p0.ibd t66_restart#p#p1.ibd @@ -792,32 +792,32 @@ t77_restart#p#p1#sp#s3.ibd # # Restart the server # -# restart +# restart:--innodb-directories=MYSQL_TMP_DIR/alt_dir SHOW VARIABLES LIKE 'innodb_file_per_table'; Variable_name Value innodb_file_per_table ON === information_schema.innodb_tablespaces and innodb_datafiles === Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/test/t4_restart.ibd -test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd -test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd -test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd -test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd -test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd -test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd -test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd -test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s3.ibd === information_schema.files === Space_Name File_Type Engine Status Tablespace_Name Path -test/t4_restart TABLESPACE InnoDB NORMAL test/t4_restart MYSQLD_DATADIR/test/t4_restart.ibd -test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd -test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd -test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd -test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd -test/t55_restart TABLESPACE InnoDB NORMAL test/t55_restart MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd -test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd -test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd -test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart TABLESPACE InnoDB NORMAL move_test/t55_restart MYSQL_TMP_DIR/alt_dir/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s3.ibd INSERT INTO t55_restart (SELECT 0, c2, c3, c4, c5 FROM t55_restart); SELECT count(*) FROM t55_restart; count(*) @@ -875,9 +875,9 @@ SUBPARTITION BY HASH (`c1`) # # Shutdown the server # ----- MYSQL_DATA_DIR/test +---- MYSQL_DATA_DIR/move_test t4_restart.ibd ----- MYSQL_TMP_DIR/alt_dir/test +---- ALT_DIR/move_test t55_restart.ibd t66_restart#p#p0.ibd t66_restart#p#p1.ibd @@ -889,29 +889,29 @@ t77_restart#p#p1#sp#s3.ibd # # Start the server and check tablespaces. # -# restart +# restart:--innodb-directories=MYSQL_TMP_DIR/alt_dir === information_schema.innodb_tablespaces and innodb_datafiles === Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/test/t4_restart.ibd -test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd -test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd -test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd -test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd -test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd -test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd -test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd -test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s3.ibd === information_schema.files === Space_Name File_Type Engine Status Tablespace_Name Path -test/t4_restart TABLESPACE InnoDB NORMAL test/t4_restart MYSQLD_DATADIR/test/t4_restart.ibd -test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd -test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd -test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd -test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd -test/t55_restart TABLESPACE InnoDB NORMAL test/t55_restart MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd -test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd -test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd -test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart TABLESPACE InnoDB NORMAL move_test/t55_restart MYSQL_TMP_DIR/alt_dir/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s3.ibd INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); SELECT count(*) FROM t4_restart; count(*) @@ -986,29 +986,29 @@ SUBPARTITION BY HASH (`c1`) # # Start the server and check tablespaces. # -# restart +# restart:--innodb-directories=MYSQL_TMP_DIR/alt_dir === information_schema.innodb_tablespaces and innodb_datafiles === Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path -test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/test/t4_restart.ibd -test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd -test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd -test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd -test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd -test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd -test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd -test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd -test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s3.ibd === information_schema.files === Space_Name File_Type Engine Status Tablespace_Name Path -test/t4_restart TABLESPACE InnoDB NORMAL test/t4_restart MYSQLD_DATADIR/test/t4_restart.ibd -test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p0.ibd -test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p1.ibd -test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s0.ibd -test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p0#sp#s1.ibd -test/t55_restart TABLESPACE InnoDB NORMAL test/t55_restart MYSQL_TMP_DIR/alt_dir/test/t55_restart.ibd -test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir/test/t66_restart#p#p2.ibd -test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s2.ibd -test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/test/t77_restart#p#p1#sp#s3.ibd +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart TABLESPACE InnoDB NORMAL move_test/t55_restart MYSQL_TMP_DIR/alt_dir/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir/move_test/t77_restart#p#p1#sp#s3.ibd INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); SELECT count(*) FROM t4_restart; count(*) @@ -1078,9 +1078,127 @@ SUBPARTITION BY HASH (`c1`) (SUBPARTITION s2 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir' ENGINE = InnoDB, SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir' ENGINE = InnoDB)) */ # +# Shutdown the server +# Move all the files form ALT_DIR to ALT_DIR_MOVED +# +# Moving files +t55_restart.ibd +t66_restart#p#p0.ibd +t66_restart#p#p1.ibd +t66_restart#p#p2.ibd +t77_restart#p#p0#sp#s0.ibd +t77_restart#p#p0#sp#s1.ibd +t77_restart#p#p1#sp#s2.ibd +t77_restart#p#p1#sp#s3.ibd +# +# Start the server and check tablespaces. +# +# restart:--innodb-directories=MYSQL_TMP_DIR/alt_dir_moved +INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); +SELECT count(*) FROM t4_restart; +count(*) +256 +SHOW CREATE TABLE t4_restart; +Table Create Table +t4_restart CREATE TABLE `t4_restart` ( + `c1` double NOT NULL AUTO_INCREMENT, + `c2` char(10) DEFAULT NULL, + `c3` varchar(100) DEFAULT NULL, + `c4` date DEFAULT NULL, + `c5` text, + PRIMARY KEY (`c1`) +) ENGINE=InnoDB AUTO_INCREMENT=1000000387 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC +INSERT INTO t55_restart (SELECT 0, c2, c3, c4, c5 FROM t55_restart); +SELECT count(*) FROM t55_restart; +count(*) +256 +SHOW CREATE TABLE t55_restart; +Table Create Table +t55_restart CREATE TABLE `t55_restart` ( + `c1` double NOT NULL AUTO_INCREMENT, + `c2` char(10) DEFAULT NULL, + `c3` varchar(100) DEFAULT NULL, + `c4` date DEFAULT NULL, + `c5` text, + PRIMARY KEY (`c1`) +) ENGINE=InnoDB AUTO_INCREMENT=1000000384 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC DATA DIRECTORY='MYSQL_TMP_DIR/alt_dir_moved/' +INSERT INTO t66_restart (SELECT 0, c2, c3, c4, c5 FROM t66_restart); +SELECT count(*) FROM t66_restart; +count(*) +672 +SHOW CREATE TABLE t66_restart; +Table Create Table +t66_restart CREATE TABLE `t66_restart` ( + `c1` int(11) NOT NULL AUTO_INCREMENT, + `c2` char(10) DEFAULT NULL, + `c3` varchar(100) DEFAULT NULL, + `c4` date DEFAULT NULL, + `c5` text, + PRIMARY KEY (`c1`) +) ENGINE=InnoDB AUTO_INCREMENT=684 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=2 +/*!50100 PARTITION BY HASH (`c1`) +(PARTITION p0 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir_moved' ENGINE = InnoDB, + PARTITION p1 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir_moved' ENGINE = InnoDB, + PARTITION p2 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir_moved' ENGINE = InnoDB) */ +INSERT INTO t77_restart (SELECT 0, c2, c3, c4, c5 FROM t77_restart); +SELECT count(*) FROM t77_restart; +count(*) +288 +SHOW CREATE TABLE t77_restart; +Table Create Table +t77_restart CREATE TABLE `t77_restart` ( + `c1` int(11) NOT NULL AUTO_INCREMENT, + `c2` char(10) DEFAULT NULL, + `c3` varchar(100) DEFAULT NULL, + `c4` date DEFAULT NULL, + `c5` text, + PRIMARY KEY (`c1`) +) ENGINE=InnoDB AUTO_INCREMENT=312 DEFAULT CHARSET=utf8mb4 ROW_FORMAT=DYNAMIC +/*!50100 PARTITION BY RANGE (`c1`) +SUBPARTITION BY HASH (`c1`) +(PARTITION p0 VALUES LESS THAN (10) + (SUBPARTITION s0 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir_moved' ENGINE = InnoDB, + SUBPARTITION s1 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir_moved' ENGINE = InnoDB), + PARTITION p1 VALUES LESS THAN MAXVALUE + (SUBPARTITION s2 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir_moved' ENGINE = InnoDB, + SUBPARTITION s3 DATA DIRECTORY = 'MYSQL_TMP_DIR/alt_dir_moved' ENGINE = InnoDB)) */ +# +# Show these tables in information_schema with the updated path +# +=== information_schema.innodb_tables and innodb_tablespaces === +Table Name Tablespace Table Flags Columns Row Format Zip Size Space Type +move_test/t4_restart move_test/t4_restart 33 8 Dynamic 0 Single +move_test/t55_restart move_test/t55_restart 97 8 Dynamic 0 Single +move_test/t66_restart#p#p0 move_test/t66_restart#p#p0 101 8 Compressed 2048 Single +move_test/t66_restart#p#p1 move_test/t66_restart#p#p1 101 8 Compressed 2048 Single +move_test/t66_restart#p#p2 move_test/t66_restart#p#p2 101 8 Compressed 2048 Single +move_test/t77_restart#p#p0#sp#s0 move_test/t77_restart#p#p0#sp#s0 97 8 Dynamic 0 Single +move_test/t77_restart#p#p0#sp#s1 move_test/t77_restart#p#p0#sp#s1 97 8 Dynamic 0 Single +move_test/t77_restart#p#p1#sp#s2 move_test/t77_restart#p#p1#sp#s2 97 8 Dynamic 0 Single +move_test/t77_restart#p#p1#sp#s3 move_test/t77_restart#p#p1#sp#s3 97 8 Dynamic 0 Single +=== information_schema.innodb_tablespaces and innodb_datafiles === +Space_Name Space_Type Page_Size Zip_Size Formats_Permitted Path +move_test/t4_restart Single DEFAULT 0 Dynamic MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir_moved/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir_moved/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir_moved/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir_moved/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir_moved/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 Single DEFAULT 2048 Compressed MYSQL_TMP_DIR/alt_dir_moved/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir_moved/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 Single DEFAULT 0 Dynamic MYSQL_TMP_DIR/alt_dir_moved/move_test/t77_restart#p#p1#sp#s3.ibd +=== information_schema.files === +Space_Name File_Type Engine Status Tablespace_Name Path +move_test/t4_restart TABLESPACE InnoDB NORMAL move_test/t4_restart MYSQLD_DATADIR/move_test/t4_restart.ibd +move_test/t66_restart#p#p0 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p0 MYSQL_TMP_DIR/alt_dir_moved/move_test/t66_restart#p#p0.ibd +move_test/t66_restart#p#p1 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p1 MYSQL_TMP_DIR/alt_dir_moved/move_test/t66_restart#p#p1.ibd +move_test/t77_restart#p#p0#sp#s0 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s0 MYSQL_TMP_DIR/alt_dir_moved/move_test/t77_restart#p#p0#sp#s0.ibd +move_test/t77_restart#p#p0#sp#s1 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p0#sp#s1 MYSQL_TMP_DIR/alt_dir_moved/move_test/t77_restart#p#p0#sp#s1.ibd +move_test/t55_restart TABLESPACE InnoDB NORMAL move_test/t55_restart MYSQL_TMP_DIR/alt_dir_moved/move_test/t55_restart.ibd +move_test/t66_restart#p#p2 TABLESPACE InnoDB NORMAL move_test/t66_restart#p#p2 MYSQL_TMP_DIR/alt_dir_moved/move_test/t66_restart#p#p2.ibd +move_test/t77_restart#p#p1#sp#s2 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s2 MYSQL_TMP_DIR/alt_dir_moved/move_test/t77_restart#p#p1#sp#s2.ibd +move_test/t77_restart#p#p1#sp#s3 TABLESPACE InnoDB NORMAL move_test/t77_restart#p#p1#sp#s3 MYSQL_TMP_DIR/alt_dir_moved/move_test/t77_restart#p#p1#sp#s3.ibd +# # Cleanup # -DROP TABLE t4_restart; -DROP TABLE t55_restart; -DROP TABLE t66_restart; -DROP TABLE t77_restart; +DROP DATABASE move_test; diff --git a/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result b/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result index 584e3f64e45a..4cc96282a8e6 100644 --- a/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result +++ b/mysql-test/suite/innodb_zip/r/wl5522_debug_zip.result @@ -29,6 +29,8 @@ SELECT * FROM test_wl5522.t1; ERROR HY000: Tablespace has been discarded for table 't1' ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE; ERROR HY000: Lost connection to MySQL server during query +unlink: t1.ibd +unlink: t1.cfg SET SESSION debug="+d,ib_import_before_checkpoint_crash"; SELECT COUNT(*) FROM test_wl5522.t1; ERROR HY000: Tablespace has been discarded for table 't1' diff --git a/mysql-test/suite/innodb_zip/t/restart.test b/mysql-test/suite/innodb_zip/t/restart.test index 1bb4ca82be5e..27ad5763c57c 100644 --- a/mysql-test/suite/innodb_zip/t/restart.test +++ b/mysql-test/suite/innodb_zip/t/restart.test @@ -10,13 +10,14 @@ LET $INNODB_PAGE_SIZE = `select @@innodb_page_size`; --disable_query_log # This error is expected in the error log for this test. -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Error number 17 means 'File exists'"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number (17|80) in a file operation."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The link file: .* already exists."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Cannot open datafile for read-only:"); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number 2 in a file operation."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the system cannot find the path specified."); -call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Error number 17 means 'File exists'"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Operating system error number (17|80) in a file operation."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: The link file: .* already exists."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot open datafile for read-only:"); +call mtr.add_suppression("\\[ERROR\\] InnoDB: Operating system error number 2 in a file operation."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: The error means the system cannot find the path specified."); +call mtr.add_suppression("\\[ERROR\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them."); +call mtr.add_suppression("\\[Warning\\] InnoDB: Tablespace .*, name 'innodb_file_per_table.*', file '.*alt_dir/move_test/.*\.ibd' has been moved to '.*alt_dir_moved/move_test/.*\.ibd'"); --enable_query_log --echo # @@ -29,12 +30,22 @@ call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: If you are installing # --disable_query_log -let $MYSQL_DATA_DIR= `select @@datadir`; -let $data_directory = DATA DIRECTORY='$MYSQL_TMP_DIR/alt_dir'; -let $innodb_file_per_table_orig=`select @@innodb_file_per_table`; +LET $SCHEMA_NAME = move_test; +LET $MYSQL_DATA_DIR = `select @@datadir`; +LET $ALT_DIR = $MYSQL_TMP_DIR/alt_dir; +LET $ALT_DATA_DIR = DATA DIRECTORY='$ALT_DIR'; +LET $ALT_DIR_MOVED = $MYSQL_TMP_DIR/alt_dir_moved; + +LET $innodb_file_per_table_orig=`select @@innodb_file_per_table`; + +LET $restart_parameters = restart:--innodb-directories=$ALT_DIR; + --enable_query_log +eval CREATE DATABASE $SCHEMA_NAME; +eval USE $SCHEMA_NAME; + set global innodb_file_per_table=on; --echo # @@ -47,6 +58,7 @@ INSERT INTO t1_restart (SELECT 0, c2, c3, c4, c5 FROM t1_restart); INSERT INTO t1_restart (SELECT 0, c2, c3, c4, c5 FROM t1_restart); INSERT INTO t1_restart (SELECT 0, c2, c3, c4, c5 FROM t1_restart); INSERT INTO t1_restart (SELECT 0, c2, c3, c4, c5 FROM t1_restart); +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t1_restart; SELECT count(*) FROM t1_restart; @@ -60,6 +72,7 @@ INSERT INTO t2_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t2_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t2_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t2_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t2_restart; SELECT count(*) FROM t2_restart; @@ -73,6 +86,7 @@ INSERT INTO t3_restart (SELECT 0, c2, c3, c4, c5 FROM t3_restart); INSERT INTO t3_restart (SELECT 0, c2, c3, c4, c5 FROM t3_restart); INSERT INTO t3_restart (SELECT 0, c2, c3, c4, c5 FROM t3_restart); INSERT INTO t3_restart (SELECT 0, c2, c3, c4, c5 FROM t3_restart); +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t3_restart; SELECT count(*) FROM t3_restart; @@ -86,15 +100,17 @@ INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t4_restart; SELECT count(*) FROM t4_restart; --echo # --echo # Create and insert records into a table that uses a remote DATA DIRECTORY. --echo # +--echo # --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR eval CREATE TABLE t5_restart(c1 DOUBLE AUTO_INCREMENT KEY, c2 CHAR(10), c3 VARCHAR(100), c4 DATE, c5 TEXT) - ROW_FORMAT=DYNAMIC ENGINE=InnoDB $data_directory; + ROW_FORMAT=DYNAMIC ENGINE=InnoDB $ALT_DATA_DIR; INSERT INTO t5_restart VALUES (1000000000, 'MySQL', 'InnoDB', '2011-11-11', 'Read this after reboot'); INSERT INTO t5_restart (SELECT 0, c2, c3, c4, c5 FROM t5_restart); INSERT INTO t5_restart (SELECT 0, c2, c3, c4, c5 FROM t5_restart); @@ -105,17 +121,15 @@ SHOW CREATE TABLE t5_restart; SELECT count(*) FROM t5_restart; --echo # ---echo # Create and insert records into a partitioned table that uses ---echo # a remote DATA DIRECTORY for each partition. ---echo # +--echo # Create and insert records into a partitioned table that uses-echo # a remote DATA DIRECTORY for each partition. --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR eval CREATE TABLE t6_restart( c1 INT AUTO_INCREMENT KEY, c2 CHAR(10), c3 VARCHAR(100), c4 DATE, c5 TEXT) ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=2 ENGINE=InnoDB PARTITION BY HASH(c1) ( - PARTITION p0 DATA DIRECTORY = '$MYSQL_TMP_DIR/alt_dir', - PARTITION p1 DATA DIRECTORY = '$MYSQL_TMP_DIR/alt_dir', - PARTITION p2 DATA DIRECTORY = '$MYSQL_TMP_DIR/alt_dir'); + PARTITION p0 $ALT_DATA_DIR, + PARTITION p1 $ALT_DATA_DIR, + PARTITION p2 $ALT_DATA_DIR); INSERT INTO t6_restart VALUES (0, 'MySQL', 'InnoDB', '2011-11-11', 'Read this after reboot'); INSERT INTO t6_restart (SELECT 0, c2, c3, c4, c5 FROM t6_restart); INSERT INTO t6_restart (SELECT 0, c2, c3, c4, c5 FROM t6_restart); @@ -135,11 +149,11 @@ eval CREATE TABLE t7_restart( ROW_FORMAT=DYNAMIC ENGINE=InnoDB PARTITION BY RANGE(c1) SUBPARTITION BY HASH(c1) ( PARTITION p0 VALUES LESS THAN (10) ( - SUBPARTITION s0 DATA DIRECTORY = '$MYSQL_TMP_DIR/alt_dir', - SUBPARTITION s1 DATA DIRECTORY = '$MYSQL_TMP_DIR/alt_dir'), + SUBPARTITION s0 $ALT_DATA_DIR, + SUBPARTITION s1 $ALT_DATA_DIR), PARTITION p1 VALUES LESS THAN MAXVALUE ( - SUBPARTITION s2 DATA DIRECTORY = '$MYSQL_TMP_DIR/alt_dir', - SUBPARTITION s3 DATA DIRECTORY = '$MYSQL_TMP_DIR/alt_dir')); + SUBPARTITION s2 $ALT_DATA_DIR, + SUBPARTITION s3 $ALT_DATA_DIR)); INSERT INTO t7_restart VALUES (0, 'MySQL', 'InnoDB', '2011-11-11', 'Read this after reboot'); INSERT INTO t7_restart (SELECT 0, c2, c3, c4, c5 FROM t7_restart); INSERT INTO t7_restart (SELECT 0, c2, c3, c4, c5 FROM t7_restart); @@ -160,6 +174,7 @@ INSERT INTO t8_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t8_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t8_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t8_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t8_restart; SELECT count(*) FROM t8_restart; CREATE TABLE t9_restart(c1 DOUBLE AUTO_INCREMENT KEY, c2 CHAR(10), c3 VARCHAR(100), c4 DATE, c5 TEXT) @@ -169,6 +184,7 @@ INSERT INTO t9_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t9_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t9_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); INSERT INTO t9_restart (SELECT 0, c2, c3, c4, c5 FROM t2_restart); +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t9_restart; SELECT count(*) FROM t9_restart; @@ -183,24 +199,28 @@ SELECT count(*) FROM t9_restart; --echo # --source include/shutdown_mysqld.inc ---echo ---- MYSQL_DATA_DIR/test +--echo ---- MYSQL_DATA_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_DATA_DIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir ---list_files $MYSQL_TMP_DIR/alt_dir ---echo ---- MYSQL_TMP_DIR/alt_dir/test +--list_files $MYSQL_DATA_DIR$SCHEMA_NAME +--echo ---- ALT_DIR +--list_files $ALT_DIR +--echo ---- ALT_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_TMP_DIR/alt_dir/test +--list_files $ALT_DIR/$SCHEMA_NAME --echo # --echo # Start the server and show that tables are still visible and accessible. ---echo # +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR --source include/start_mysqld.inc SHOW VARIABLES LIKE 'innodb_file_per_table'; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t1_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t2_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t3_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t4_restart; --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t5_restart; @@ -208,7 +228,9 @@ SHOW CREATE TABLE t5_restart; SHOW CREATE TABLE t6_restart; --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t7_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t8_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t9_restart; INSERT INTO t1_restart (SELECT 0, c2, c3, c4, c5 FROM t1_restart); @@ -277,18 +299,19 @@ SHOW CREATE TABLE t7_restart; --echo # --source include/shutdown_mysqld.inc ---copy_file $MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd $MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd.bak +--copy_file $ALT_DIR/$SCHEMA_NAME/t5_restart.ibd $ALT_DIR/$SCHEMA_NAME/t5_restart.ibd.bak ---echo ---- MYSQL_DATA_DIR/test +--echo ---- MYSQL_DATA_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_DATA_DIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test +--list_files $MYSQL_DATA_DIR/$SCHEMA_NAME +--echo ---- ALT_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_TMP_DIR/alt_dir/test +--list_files $ALT_DIR/$SCHEMA_NAME --echo # --echo # Start the server and show the tablespaces. --echo # +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR --source include/start_mysqld.inc SHOW VARIABLES LIKE 'innodb_file_per_table'; @@ -311,14 +334,14 @@ SHOW CREATE TABLE t7_restart; --echo # Try to rename a tablespace to a file that already exists --echo # ---remove_file $MYSQL_TMP_DIR/alt_dir/test/t5_restart.ibd.bak +--remove_file $ALT_DIR/$SCHEMA_NAME/t5_restart.ibd.bak ---echo ---- MYSQL_DATA_DIR/test +--echo ---- MYSQL_DATA_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_DATA_DIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test +--list_files $MYSQL_DATA_DIR/$SCHEMA_NAME +--echo ---- ALT_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_TMP_DIR/alt_dir/test +--list_files $ALT_DIR/$SCHEMA_NAME --echo # --echo # Rename file table and tablespace @@ -345,16 +368,17 @@ SELECT count(*) FROM t77_restart; --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t77_restart; ---echo ---- MYSQL_DATA_DIR/test +--echo ---- MYSQL_DATA_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_DATA_DIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test +--list_files $MYSQL_DATA_DIR/$SCHEMA_NAME +--echo ---- ALT_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_TMP_DIR/alt_dir/test +--list_files $ALT_DIR/$SCHEMA_NAME --echo # --echo # Restart the server --echo # +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR --source include/restart_mysqld.inc SHOW VARIABLES LIKE 'innodb_file_per_table'; @@ -380,16 +404,17 @@ SHOW CREATE TABLE t77_restart; --echo # --source include/shutdown_mysqld.inc ---echo ---- MYSQL_DATA_DIR/test +--echo ---- MYSQL_DATA_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_DATA_DIR/test ---echo ---- MYSQL_TMP_DIR/alt_dir/test +--list_files $MYSQL_DATA_DIR/$SCHEMA_NAME +--echo ---- ALT_DIR/$SCHEMA_NAME --replace_result #P# #p# #SP# #sp# ---list_files $MYSQL_TMP_DIR/alt_dir/test +--list_files $ALT_DIR/$SCHEMA_NAME --echo # --echo # Start the server and check tablespaces. --echo # +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR --source include/start_mysqld.inc --source suite/innodb/include/show_i_s_tablespaces.inc @@ -423,12 +448,14 @@ SHOW CREATE TABLE t77_restart; --echo # --echo # Start the server and check tablespaces. --echo # --- source include/start_mysqld.inc +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--source include/start_mysqld.inc --source suite/innodb/include/show_i_s_tablespaces.inc INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); SELECT count(*) FROM t4_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t4_restart; INSERT INTO t55_restart (SELECT 0, c2, c3, c4, c5 FROM t55_restart); @@ -446,19 +473,67 @@ SELECT count(*) FROM t77_restart; --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR SHOW CREATE TABLE t77_restart; +--echo # +--echo # Shutdown the server +--echo # Move all the files form ALT_DIR to ALT_DIR_MOVED +--echo # +--source include/shutdown_mysqld.inc + +--echo # Moving files +--mkdir $ALT_DIR_MOVED +--mkdir $ALT_DIR_MOVED/$SCHEMA_NAME +--copy_files_wildcard $ALT_DIR/$SCHEMA_NAME $ALT_DIR_MOVED/$SCHEMA_NAME * +--remove_files_wildcard $ALT_DIR/$SCHEMA_NAME * +--rmdir $ALT_DIR/$SCHEMA_NAME +--replace_result #P# #p# #SP# #sp# +--list_files $ALT_DIR_MOVED/$SCHEMA_NAME + +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +LET $restart_parameters = restart:--innodb-directories="$ALT_DIR_MOVED"; + +--echo # +--echo # Start the server and check tablespaces. +--echo # +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--source include/start_mysqld.inc + +INSERT INTO t4_restart (SELECT 0, c2, c3, c4, c5 FROM t4_restart); +SELECT count(*) FROM t4_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +SHOW CREATE TABLE t4_restart; + +INSERT INTO t55_restart (SELECT 0, c2, c3, c4, c5 FROM t55_restart); +SELECT count(*) FROM t55_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +SHOW CREATE TABLE t55_restart; + +INSERT INTO t66_restart (SELECT 0, c2, c3, c4, c5 FROM t66_restart); +SELECT count(*) FROM t66_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +SHOW CREATE TABLE t66_restart; + +INSERT INTO t77_restart (SELECT 0, c2, c3, c4, c5 FROM t77_restart); +SELECT count(*) FROM t77_restart; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +SHOW CREATE TABLE t77_restart; + +--echo # +--echo # Show these tables in information_schema with the updated path +--echo # +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +--source suite/innodb/include/show_i_s_tables.inc +--source suite/innodb/include/show_i_s_tablespaces.inc --echo # --echo # Cleanup --echo # -DROP TABLE t4_restart; -DROP TABLE t55_restart; -DROP TABLE t66_restart; -DROP TABLE t77_restart; +eval DROP DATABASE $SCHEMA_NAME; + +--disable_query_log ---rmdir $MYSQL_TMP_DIR/alt_dir/test ---rmdir $MYSQL_TMP_DIR/alt_dir +--rmdir $ALT_DIR_MOVED/$SCHEMA_NAME +--rmdir $ALT_DIR_MOVED --- disable_query_log eval set global innodb_file_per_table=$innodb_file_per_table_orig; --- enable_query_log +--enable_query_log diff --git a/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test b/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test index 0dcb923304a5..67d9f02d3afa 100644 --- a/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test +++ b/mysql-test/suite/innodb_zip/t/wl5522_debug_zip.test @@ -78,6 +78,11 @@ SELECT * FROM test_wl5522.t1; --error 2013 ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE; +perl; +require 'include/innodb-util.inc'; +ib_unlink_tablespace("test_wl5522", "t1"); +EOF + --enable_reconnect --source include/wait_until_connected_again.inc --disable_reconnect @@ -719,6 +724,7 @@ call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Monitor ibuf_merges is already enabled"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Monitor ibuf_merges_insert is already enabled"); call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace for table .* is set as discarded."); +call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace .*, name '.*', file '.*' is missing"); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Trying to import a tablespace, but could not open the tablespace file"); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number 32 in a file operation."); call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means that another program is using InnoDB's files."); diff --git a/mysql-test/suite/sys_vars/r/innodb_scan_directories_basic.result b/mysql-test/suite/sys_vars/r/innodb_scan_directories_basic.result deleted file mode 100644 index eddf55eaeca8..000000000000 --- a/mysql-test/suite/sys_vars/r/innodb_scan_directories_basic.result +++ /dev/null @@ -1,22 +0,0 @@ -Valid values are semi-colon separated strings -select @@global.innodb_scan_directories; -@@global.innodb_scan_directories - -select @@session.innodb_scan_directories; -ERROR HY000: Variable 'innodb_scan_directories' is a GLOBAL variable -show global variables like 'innodb_scan_directories'; -Variable_name Value -innodb_scan_directories -show session variables like 'innodb_scan_directories'; -Variable_name Value -innodb_scan_directories -select * from performance_schema.global_variables where variable_name='innodb_scan_directories'; -VARIABLE_NAME VARIABLE_VALUE -innodb_scan_directories -select * from performance_schema.session_variables where variable_name='innodb_scan_directories'; -VARIABLE_NAME VARIABLE_VALUE -innodb_scan_directories -set global innodb_scan_directories='a;b;c'; -ERROR HY000: Variable 'innodb_scan_directories' is a read only variable -set session innodb_scan_directories=1; -ERROR HY000: Variable 'innodb_scan_directories' is a read only variable diff --git a/mysql-test/suite/sys_vars/t/innodb_scan_directories_basic.test b/mysql-test/suite/sys_vars/t/innodb_scan_directories_basic.test deleted file mode 100644 index 6e333c0af876..000000000000 --- a/mysql-test/suite/sys_vars/t/innodb_scan_directories_basic.test +++ /dev/null @@ -1,20 +0,0 @@ -# Can only be set from the command line. -# show the global and session values; - ---echo Valid values are semi-colon separated strings -select @@global.innodb_scan_directories; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -select @@session.innodb_scan_directories; -show global variables like 'innodb_scan_directories'; -show session variables like 'innodb_scan_directories'; ---disable_warnings -select * from performance_schema.global_variables where variable_name='innodb_scan_directories'; -select * from performance_schema.session_variables where variable_name='innodb_scan_directories'; ---enable_warnings - -# Show that it's read-only ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -set global innodb_scan_directories='a;b;c'; ---error ER_INCORRECT_GLOBAL_LOCAL_VAR -set session innodb_scan_directories=1; - diff --git a/mysql-test/t/innodb_recovery_with_upper_case_names.test b/mysql-test/t/innodb_recovery_with_upper_case_names.test index 5f0d5fa7683e..4c1e9486df01 100644 --- a/mysql-test/t/innodb_recovery_with_upper_case_names.test +++ b/mysql-test/t/innodb_recovery_with_upper_case_names.test @@ -59,3 +59,6 @@ DROP TABLE mytable; use test; DROP DATABASE MYDB; +--disable_query_log +call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace .*, name '.*', file '.*' is missing"); +--enable_query_log diff --git a/mysql-test/t/partition_innodb.test b/mysql-test/t/partition_innodb.test index ca3b08feac5b..01a2df650531 100644 --- a/mysql-test/t/partition_innodb.test +++ b/mysql-test/t/partition_innodb.test @@ -1129,6 +1129,10 @@ DROP TABLE t1; #Create one partition in general tablespace and other in different path +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +let $restart_parameters = restart: --innodb-directories=$MYSQL_TMP_DIR; +--source include/restart_mysqld.inc + --replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR eval CREATE TABLESPACE ts1 ADD DATAFILE '$MYSQL_TMP_DIR/ts1.ibd' ENGINE=Innodb; diff --git a/mysql-test/t/schema.test b/mysql-test/t/schema.test index 60b0fe893a33..fa96a28e08ac 100644 --- a/mysql-test/t/schema.test +++ b/mysql-test/t/schema.test @@ -644,6 +644,9 @@ SET DEBUG_SYNC= 'now SIGNAL continue_alter'; --connection con3 --reap +--disable_query_log +call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Tablespace .*, name '.*', file '.*' is missing!"); +--enable_query_log --echo # --echo # Bug#26043994: CREATE DATABASE/DIRECTORY INCONSISTENCY diff --git a/mysql-test/t/tablespace.test b/mysql-test/t/tablespace.test index 7b9c74c141da..f507933854dd 100644 --- a/mysql-test/t/tablespace.test +++ b/mysql-test/t/tablespace.test @@ -505,6 +505,10 @@ SET AUTOCOMMIT=ON; --echo # Test ALTER TABLESPACE RENAME with partition tables CREATE TABLESPACE ts1 ADD DATAFILE 'ts1.ibd' Engine=InnoDB; +--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR +let $restart_parameters = restart: --innodb-directories=$MYSQL_TMP_DIR; +--source include/restart_mysqld.inc + --disable_query_log --eval CREATE TABLESPACE ts2 ADD DATAFILE '$MYSQL_TMP_DIR/ts2.ibd' ENGINE=InnoDB; --enable_query_log diff --git a/storage/innobase/buf/buf0dblwr.cc b/storage/innobase/buf/buf0dblwr.cc index 83a184eb7fc5..2669f2475298 100644 --- a/storage/innobase/buf/buf0dblwr.cc +++ b/storage/innobase/buf/buf0dblwr.cc @@ -659,11 +659,13 @@ buf_dblwr_recover_page( /* Write the good page from the doublewrite buffer to the intended position. */ - fil_io(write_request, true, + err = fil_io(write_request, true, page_id, page_size, 0, page_size.physical(), const_cast(page), NULL); + ut_a(err == DB_SUCCESS); + ib::info() << "Recovered page " << page_id @@ -969,15 +971,19 @@ buf_dblwr_write_block_to_datafile( type |= IORequest::DO_NOT_WAKE; } + dberr_t err; IORequest request(type); if (bpage->zip.data != NULL) { ut_ad(bpage->size.is_compressed()); - fil_io(request, sync, bpage->id, bpage->size, 0, - bpage->size.physical(), - (void*) bpage->zip.data, - (void*) bpage); + err = fil_io( + request, sync, bpage->id, bpage->size, 0, + bpage->size.physical(), (void*) bpage->zip.data, + (void*) bpage); + + ut_a(err == DB_SUCCESS); + } else { ut_ad(!bpage->size.is_compressed()); @@ -990,9 +996,11 @@ buf_dblwr_write_block_to_datafile( ut_a(buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE); buf_dblwr_check_page_lsn(block->frame); - fil_io(request, - sync, bpage->id, bpage->size, 0, bpage->size.physical(), - block->frame, block); + err = fil_io( + request, sync, bpage->id, bpage->size, 0, + bpage->size.physical(), block->frame, block); + + ut_a(err == DB_SUCCESS); } } @@ -1006,9 +1014,10 @@ void buf_dblwr_flush_buffered_writes(void) /*=================================*/ { + ulint len; + dberr_t err; byte* write_buf; ulint first_free; - ulint len; if (!srv_use_doublewrite_buf || buf_dblwr == NULL) { /* Sync the writes to the disk. */ @@ -1092,9 +1101,12 @@ buf_dblwr_flush_buffered_writes(void) len = ut_min(TRX_SYS_DOUBLEWRITE_BLOCK_SIZE, buf_dblwr->first_free) * UNIV_PAGE_SIZE; - fil_io(IORequestWrite, true, - page_id_t(TRX_SYS_SPACE, buf_dblwr->block1), univ_page_size, - 0, len, (void*) write_buf, NULL); + err = fil_io( + IORequestWrite, true, + page_id_t(TRX_SYS_SPACE, buf_dblwr->block1), univ_page_size, + 0, len, (void*) write_buf, NULL); + + ut_a(err == DB_SUCCESS); if (buf_dblwr->first_free <= TRX_SYS_DOUBLEWRITE_BLOCK_SIZE) { /* No unwritten pages in the second block. */ @@ -1108,9 +1120,12 @@ buf_dblwr_flush_buffered_writes(void) write_buf = buf_dblwr->write_buf + TRX_SYS_DOUBLEWRITE_BLOCK_SIZE * UNIV_PAGE_SIZE; - fil_io(IORequestWrite, true, - page_id_t(TRX_SYS_SPACE, buf_dblwr->block2), univ_page_size, - 0, len, (void*) write_buf, NULL); + err = fil_io( + IORequestWrite, true, + page_id_t(TRX_SYS_SPACE, buf_dblwr->block2), univ_page_size, + 0, len, (void*) write_buf, NULL); + + ut_a(err == DB_SUCCESS); flush: /* increment the doublewrite flushed pages counter */ @@ -1241,10 +1256,11 @@ buf_dblwr_write_single_page( buf_page_t* bpage, /*!< in: buffer block to write */ bool sync) /*!< in: true if sync IO requested */ { + page_no_t i; + dberr_t err; ulint n_slots; page_no_t size; page_no_t offset; - page_no_t i; ut_a(buf_page_in_file(bpage)); ut_a(srv_use_doublewrite_buf); @@ -1330,22 +1346,23 @@ buf_dblwr_write_single_page( + bpage->size.physical(), 0x0, univ_page_size.physical() - bpage->size.physical()); - fil_io(IORequestWrite, true, - page_id_t(TRX_SYS_SPACE, offset), univ_page_size, 0, - univ_page_size.physical(), - (void*) (buf_dblwr->write_buf - + univ_page_size.physical() * i), - NULL); + err = fil_io( + IORequestWrite, true, page_id_t(TRX_SYS_SPACE, offset), + univ_page_size, 0, univ_page_size.physical(), + (void*) (buf_dblwr->write_buf + + univ_page_size.physical() * i), NULL); } else { /* It is a regular page. Write it directly to the doublewrite buffer */ - fil_io(IORequestWrite, true, - page_id_t(TRX_SYS_SPACE, offset), univ_page_size, 0, - univ_page_size.physical(), - (void*) ((buf_block_t*) bpage)->frame, - NULL); + + err = fil_io( + IORequestWrite, true, page_id_t(TRX_SYS_SPACE, offset), + univ_page_size, 0, univ_page_size.physical(), + (void*) ((buf_block_t*) bpage)->frame, NULL); } + ut_a(err == DB_SUCCESS); + /* Now flush the doublewrite buffer data to disk */ fil_flush(TRX_SYS_SPACE); diff --git a/storage/innobase/buf/buf0dump.cc b/storage/innobase/buf/buf0dump.cc index bc74131f5482..2771efef8caf 100644 --- a/storage/innobase/buf/buf0dump.cc +++ b/storage/innobase/buf/buf0dump.cc @@ -195,7 +195,7 @@ get_buf_dump_dir() /* The dump file should be created in the default data directory if innodb_data_home_dir is set as an empty string. */ if (strcmp(srv_data_home, "") == 0) { - dump_dir = fil_path_to_mysql_datadir; + dump_dir = MySQL_datadir_path; } else { dump_dir = srv_data_home; } diff --git a/storage/innobase/buf/buf0flu.cc b/storage/innobase/buf/buf0flu.cc index deeab1d722da..f5fb0f588262 100644 --- a/storage/innobase/buf/buf0flu.cc +++ b/storage/innobase/buf/buf0flu.cc @@ -1133,11 +1133,15 @@ buf_flush_write_block_low( ulint type = IORequest::WRITE | IORequest::DO_NOT_WAKE; + dberr_t err; IORequest request(type); - fil_io(request, - sync, bpage->id, bpage->size, 0, bpage->size.physical(), - frame, bpage); + err = fil_io( + request, + sync, bpage->id, bpage->size, 0, bpage->size.physical(), + frame, bpage); + + ut_a(err == DB_SUCCESS); } else if (flush_type == BUF_FLUSH_SINGLE_PAGE) { buf_dblwr_write_single_page(bpage, sync); @@ -3301,16 +3305,17 @@ buf_flush_page_coordinator_thread(size_t n_page_cleaners) if (curr_time > next_loop_time + 3000) { if (warn_count == 0) { - ib::info() << "page_cleaner: 1000ms" - " intended loop took " - << 1000 + curr_time - - next_loop_time - << "ms. The settings might not" - " be optimal. (flushed=" + ulint us; + + us = 1000 + curr_time - next_loop_time; + + ib::info() + << "Page cleaner took " + << us << "ms to flush" << n_flushed_last - << " and evicted=" - << n_evicted - << ", during the time.)"; + << " and evict " + << n_evicted << " pages"; + if (warn_interval > 300) { warn_interval = 600; } else { diff --git a/storage/innobase/buf/buf0rea.cc b/storage/innobase/buf/buf0rea.cc index 5376035f4b69..dd99f07dadda 100644 --- a/storage/innobase/buf/buf0rea.cc +++ b/storage/innobase/buf/buf0rea.cc @@ -779,10 +779,11 @@ buf_read_ibuf_merge_pages( /** Issues read requests for pages which recovery wants to read in. @param[in] sync true if the caller wants this function to wait -for the highest address page to get read in, before this function returns + for the highest address page to get read in, + before this function returns @param[in] space_id tablespace id @param[in] page_nos array of page numbers to read, with the -highest page number the last in the array + highest page number the last in the array @param[in] n_stored number of page numbers in the array */ void buf_read_recv_pages( @@ -792,8 +793,6 @@ buf_read_recv_pages( ulint n_stored) { ulint count; - dberr_t err; - ulint i; fil_space_t* space = fil_space_get(space_id); if (space == NULL) { @@ -835,14 +834,15 @@ buf_read_recv_pages( const page_size_t page_size(space->flags); - for (i = 0; i < n_stored; i++) { - buf_pool_t* buf_pool; + for (ulint i = 0; i < n_stored; i++) { + buf_pool_t* buf_pool; const page_id_t cur_page_id(space_id, page_nos[i]); count = 0; buf_pool = buf_pool_get(cur_page_id); os_rmb; + while (buf_pool->n_pend_reads >= recv_n_pool_free_frames / 2) { os_aio_simulated_wake_handler_threads(); @@ -860,6 +860,8 @@ buf_read_recv_pages( } } + dberr_t err; + if ((i + 1 == n_stored) && sync) { buf_read_page_low( &err, true, diff --git a/storage/innobase/clone/clone0apply.cc b/storage/innobase/clone/clone0apply.cc index c66bb57801f0..0221965df6e6 100644 --- a/storage/innobase/clone/clone0apply.cc +++ b/storage/innobase/clone/clone0apply.cc @@ -99,32 +99,43 @@ Clone_Snapshot::add_file_from_desc( ptr += dir_len; name_len += dir_len; - const char* name_ptr; + std::string name; char name_buf[MAX_LOG_FILE_NAME]; if (m_snapshot_state == CLONE_SNAPSHOT_FILE_COPY) { - name_ptr = file_desc->m_file_name; + name.assign(file_desc->m_file_name); /* For absolute path, we must ensure that the file is not present. This would always fail for local clone. */ - if (is_absolute_path(file_desc->m_file_name)) { + if (Fil_path::is_absolute_path(name)) { - os_file_type_t type; - bool exists = false; + auto type = Fil_path::get_file_type(name); - os_file_status(file_desc->m_file_name, &exists, &type); + if (type != OS_FILE_TYPE_MISSING) { - if (exists) { + if (type == OS_FILE_TYPE_FILE) { - my_error(ER_FILE_EXISTS_ERROR, MYF(0), - file_desc->m_file_name); - return(DB_TABLESPACE_EXISTS); + my_error(ER_FILE_EXISTS_ERROR, MYF(0), + name.c_str()); + + return(DB_TABLESPACE_EXISTS); + } else { + + /* Either the stat() call failed or + the name is a directory/block device, + or permission error etc. */ + + my_error(ER_INTERNAL_ERROR, MYF(0), + name.c_str()); + + return(DB_ERROR); + } } - } else if (name_ptr[0] == '.' && name_ptr[1] == OS_PATH_SEPARATOR) { + } else if (Fil_path::has_prefix(name, Fil_path::DOT_SLASH)) { - name_ptr += 2; + name.erase(0, 2); } } else { @@ -134,13 +145,14 @@ Clone_Snapshot::add_file_from_desc( snprintf(name_buf, MAX_LOG_FILE_NAME, "%s%u", ib_logfile_basename, idx); - name_ptr = const_cast(name_buf); + name.assign(name_buf); } - strcpy(ptr, name_ptr); + strcpy(ptr, name.c_str()); + + name_len += name.length(); - name_len += strlen(name_ptr); - name_len++; + ++name_len; file_meta->m_file_name_len = name_len; @@ -337,7 +349,7 @@ Clone_Handle::receive_data( success = os_file_seek(nullptr, file_hdl, offset); if (!success) { - my_error(ER_ERROR_ON_READ, MYF(0), file_meta->m_file_name, errno, + my_error(ER_ERROR_ON_READ, MYF(0),file_meta->m_file_name, errno, my_strerror(errbuf, sizeof(errbuf), errno)); return(DB_ERROR); } diff --git a/storage/innobase/clone/clone0copy.cc b/storage/innobase/clone/clone0copy.cc index 0d8a5bcd8548..7f3caf0d3f01 100644 --- a/storage/innobase/clone/clone0copy.cc +++ b/storage/innobase/clone/clone0copy.cc @@ -29,25 +29,6 @@ Innodb copy snapshot data #include "buf0dump.h" #include "dict0dict.h" -/** Callback to add a tablespace file node to current snapshot -@param[in] node file node -@param[in] context snapshot -@return error code */ -static dberr_t -add_node_callback( - fil_node_t* node, - void* context) -{ - dberr_t err; - - Clone_Snapshot* snapshot; - snapshot = static_cast(context); - - err = snapshot->add_node(node); - - return(err); -} - /** Callback to add an archived redo file to current snapshot @param[in] file_name file name @param[in] file_size file size in bytes @@ -204,18 +185,14 @@ Clone_Snapshot::init_file_copy() return(err); } - /* Collect data file metadata from tablespace nodes. */ - void* context; - bool include_log; - - context = static_cast(this); - - /* For blocking clone, include redo files in file list. */ - include_log = (m_snapshot_type == HA_CLONE_BLOCKING); + /* Do not include redo files in file list. */ + bool include_log = (m_snapshot_type == HA_CLONE_BLOCKING); /* Iterate all tablespace files and add persistent data files. */ - err = fil_iterate_tablespace_files(include_log, - context, add_node_callback); + err = Fil_iterator::for_each_file(include_log, [&] (fil_node_t* file) + { + return(add_node(file)); + }); if (err != DB_SUCCESS) { @@ -783,14 +760,16 @@ Clone_Handle::send_file_metadata( ut_ad(file_desc.m_state == CLONE_SNAPSHOT_FILE_COPY); ut_ad(file_meta->m_file_index == 0); - file_desc.m_file_meta.m_file_name = SRV_BUF_DUMP_FILENAME_DEFAULT; + file_desc.m_file_meta.m_file_name = + SRV_BUF_DUMP_FILENAME_DEFAULT; + file_desc.m_file_meta.m_file_name_len = static_cast( strlen(SRV_BUF_DUMP_FILENAME_DEFAULT)) + 1; } else if (!fsp_is_ibd_tablespace( static_cast(file_meta->m_space_id)) - && is_absolute_path(file_meta->m_file_name)) { + && Fil_path::is_absolute_path(file_meta->m_file_name)) { /* For system tablespace, remove absolute path. */ ut_ad(file_desc.m_state == CLONE_SNAPSHOT_FILE_COPY); diff --git a/storage/innobase/dict/dict0crea.cc b/storage/innobase/dict/dict0crea.cc index 716440e5f46c..b02dd592b199 100644 --- a/storage/innobase/dict/dict0crea.cc +++ b/storage/innobase/dict/dict0crea.cc @@ -58,16 +58,14 @@ dict_build_table_def( dict_table_t* table, trx_t* trx) { - dberr_t err = DB_SUCCESS; - char db_buf[NAME_LEN + 1]; char tbl_buf[NAME_LEN + 1]; dd_parse_tbl_name(table->name.m_name, db_buf, tbl_buf, nullptr, nullptr, nullptr); - bool is_dd_table = dd::get_dictionary()->is_dd_table_name( - db_buf, tbl_buf); + bool is_dd_table = dd::get_dictionary()->is_dd_table_name( + db_buf, tbl_buf); /** In-memory counter used for assigning table_id of data dictionary table. This counter is only used @@ -77,13 +75,15 @@ dict_build_table_def( if (is_dd_table) { table->id = dd_table_id++; table->is_dd_table = true; - ut_ad(strcmp(tbl_buf, innodb_dd_table[table->id - 1].name) == 0); + + ut_ad(strcmp(tbl_buf, innodb_dd_table[table->id - 1].name) + == 0); } else { dict_table_assign_new_id(table, trx); } - err = dict_build_tablespace_for_table(table, trx); + dberr_t err = dict_build_tablespace_for_table(table, trx); return(err); } @@ -114,7 +114,7 @@ dict_build_tablespace( } tablespace->set_space_id(space); - Datafile* datafile = tablespace->first_datafile(); + Datafile* datafile = tablespace->first_datafile(); /* If file already exists we cannot write delete space to ddl log. */ os_file_type_t type; @@ -222,24 +222,21 @@ dict_build_tablespace_for_table( table->space = space; /* Determine the tablespace flags. */ - bool has_data_dir = DICT_TF_HAS_DATA_DIR(table->flags); bool is_encrypted = dict_table_is_encrypted(table); - ulint fsp_flags = dict_tf_to_fsp_flags(table->flags, - is_encrypted); - std::string tablespace_name; - /* Determine the full filepath */ - if (has_data_dir) { - ut_ad(table->data_dir_path); - filepath = fil_make_filepath( - table->data_dir_path, - table->name.m_name, IBD, true); + ulint fsp_flags = dict_tf_to_fsp_flags( + table->flags, is_encrypted); + if (DICT_TF_HAS_DATA_DIR(table->flags)) { + std::string path; + + path = dict_table_get_datadir(table); + + filepath = Fil_path::make( + path, table->name.m_name, IBD, true); } else { - /* Make the tablespace file in the default dir - using the table name */ - filepath = fil_make_filepath( - NULL, table->name.m_name, IBD, false); + filepath = Fil_path::make_ibd_from_table_name( + table->name.m_name); } /* If file already exists we cannot write delete space to ddl log. */ @@ -266,8 +263,9 @@ dict_build_tablespace_for_table( - page 3 will contain the root of the clustered index of the table we create here. */ - dd_filename_to_spacename(table->name.m_name, - &tablespace_name); + std::string tablespace_name; + + dd_filename_to_spacename(table->name.m_name, &tablespace_name); err = fil_ibd_create( space, tablespace_name.c_str(), filepath, fsp_flags, diff --git a/storage/innobase/dict/dict0dd.cc b/storage/innobase/dict/dict0dd.cc index c11527457698..6ce995a4746d 100644 --- a/storage/innobase/dict/dict0dd.cc +++ b/storage/innobase/dict/dict0dd.cc @@ -34,7 +34,6 @@ Data dictionary interface */ #include "dict0priv.h" #include
#include "dict0mem.h" -#include "mysql_com.h" #ifndef UNIV_HOTBACKUP # include "dict0stats.h" #endif /* !UNIV_HOTBACKUP */ @@ -811,7 +810,8 @@ dd_table_discard_tablespace( #ifdef UNIV_DEBUG btrsea_sync_check check(false); ut_ad(!sync_check_iterate(check)); -#endif +#endif /* UNIV_DEBUG */ + ut_ad(!srv_is_being_shutdown); if (table_def->se_private_id() != dd::INVALID_OBJECT_ID) { @@ -841,10 +841,13 @@ dd_table_discard_tablespace( dd::Properties& p = table_def->se_private_data(); p.set_bool(dd_table_key_strings[DD_TABLE_DISCARD], discard); + using Client = dd::cache::Dictionary_client; + using Releaser = dd::cache::Dictionary_client::Auto_releaser; + /* Get Tablespace object */ - dd::Tablespace* dd_space = nullptr; - dd::cache::Dictionary_client* client = dd::get_dd_client(thd); - dd::cache::Dictionary_client::Auto_releaser releaser(client); + dd::Tablespace* dd_space = nullptr; + Client* client = dd::get_dd_client(thd); + Releaser releaser{client}; dd::Object_id dd_space_id = (*table_def->indexes()->begin())->tablespace_id(); @@ -853,8 +856,9 @@ dd_table_discard_tablespace( dd_filename_to_spacename(table->name.m_name, &space_name); - if (dd::acquire_exclusive_tablespace_mdl(thd, space_name.c_str(), - false)) { + if (dd::acquire_exclusive_tablespace_mdl( + thd, space_name.c_str(), false)) { + ut_a(false); } @@ -1043,38 +1047,36 @@ dd_table_close( #ifndef UNIV_HOTBACKUP /** Update filename of dd::Tablespace -@param[in] dd_space_id dd tablespace id -@param[in] new_space_name new tablespace name -@param[in] new_path new data file path -@retval true if fail. */ -bool +@param[in] dd_space_id DD tablespace id +@param[in] new_space_name New tablespace name +@param[in] new_path New data file path +@retval DB_SUCCESS on success. */ +dberr_t dd_rename_tablespace( dd::Object_id dd_space_id, const char* new_space_name, const char* new_path) { - dd::Tablespace* dd_space = nullptr; - dd::Tablespace* new_space = nullptr; - bool ret = false; THD* thd = current_thd; std::string tablespace_name; - DBUG_ENTER("dd_tablespace_update_for_rename"); + DBUG_ENTER("dd_rename_tablespace"); #ifdef UNIV_DEBUG btrsea_sync_check check(false); ut_ad(!sync_check_iterate(check)); -#endif +#endif /* UNIV_DEBUG */ ut_ad(!srv_is_being_shutdown); dd::cache::Dictionary_client* client = dd::get_dd_client(thd); dd::cache::Dictionary_client::Auto_releaser releaser(client); - /* Get the dd tablespace */ + dd::Tablespace* dd_space = nullptr; + /* Get the dd tablespace */ if (client->acquire_uncached_uncommitted( dd_space_id, &dd_space)) { ut_ad(false); - DBUG_RETURN(true); + DBUG_RETURN(DB_ERROR); } ut_a(dd_space != nullptr); @@ -1082,23 +1084,24 @@ dd_rename_tablespace( if (dd::acquire_exclusive_tablespace_mdl( thd, dd_space->name().c_str(), false)) { ut_ad(false); - DBUG_RETURN(true); + DBUG_RETURN(DB_ERROR); } - dd_filename_to_spacename(new_space_name, - &tablespace_name); + dd_filename_to_spacename(new_space_name, &tablespace_name); if (dd::acquire_exclusive_tablespace_mdl( thd, tablespace_name.c_str(), false)) { ut_ad(false); - DBUG_RETURN(true); + DBUG_RETURN(DB_ERROR); } + dd::Tablespace* new_space = nullptr; + /* Acquire the new dd tablespace for modification */ if (client->acquire_for_modification( dd_space_id, &new_space)) { ut_ad(false); - DBUG_RETURN(true); + DBUG_RETURN(DB_ERROR); } ut_ad(new_space->files().size() == 1); @@ -1106,9 +1109,12 @@ dd_rename_tablespace( new_space->set_name(tablespace_name.c_str()); if (new_path != nullptr) { + dd::Tablespace_file* dd_file = const_cast< dd::Tablespace_file*>(*(new_space->files().begin())); + dd_file->set_filename(new_path); + } else { #ifdef UNIV_DEBUG const dd::Properties& p = dd_space->se_private_data(); @@ -1120,14 +1126,10 @@ dd_rename_tablespace( #endif /* UNIV_DEBUG */ } - bool fail = client->update(new_space); + bool fail = client->update(new_space); + ut_ad(!fail); - if (fail) { - ut_ad(false); - ret = true; - } - - DBUG_RETURN(ret); + DBUG_RETURN(fail ? DB_ERROR : DB_SUCCESS); } /** Validate the table format options. @@ -2695,6 +2697,7 @@ dd_filename_to_spacename( filename_to_tablename((char*) space_name, orig_tablespace, (NAME_LEN + 1)); tablespace_name->append(orig_tablespace); + return; } @@ -2703,12 +2706,12 @@ dd_filename_to_spacename( tablespace_name->append(tbl_buf); if (part_buf[0] != '\0') { - tablespace_name->append(part_sep); + tablespace_name->append(PART_SEPARATOR); tablespace_name->append(part_buf); } if (sub_buf[0] != '\0') { - tablespace_name->append(sub_sep); + tablespace_name->append(SUB_PART_SEPARATOR); tablespace_name->append(sub_buf); } @@ -2806,11 +2809,11 @@ dd_create_implicit_tablespace( bool discarded, dd::Object_id& dd_space_id) { - fil_space_t* space = fil_space_get(space_id); - ulint flags = space->flags; std::string space_name; - dd_filename_to_spacename(tablespace_name, - &space_name); + fil_space_t* space = fil_space_get(space_id); + ulint flags = space->flags; + + dd_filename_to_spacename(tablespace_name, &space_name); bool fail = create_dd_tablespace( dd_client, thd, space_name.c_str(), space_id, @@ -3279,44 +3282,8 @@ dd_table_get_space_name( DBUG_RETURN(space_name); } -/** Using the table->heap, copy the null-terminated filepath into -table->data_dir_path and replace the 'databasename/tablename.ibd' -portion with 'tablename'. -This allows SHOW CREATE TABLE to return the correct DATA DIRECTORY path. -Make this data directory path only if it has not yet been saved. -@param[in,out] table table obj -@param[in] filepath filepath of tablespace */ -static -void -dd_save_data_dir_path( - dict_table_t* table, - char* filepath) -{ - ut_ad(mutex_own(&dict_sys->mutex)); - ut_ad(DICT_TF_HAS_DATA_DIR(table->flags)); - - ut_ad(!table->data_dir_path); - ut_ad(filepath); - - /* Be sure this filepath is not the default filepath. */ - char* default_filepath = fil_make_filepath( - NULL, table->name.m_name, IBD, false); - if (default_filepath) { - if (0 != strcmp(filepath, default_filepath)) { - ulint pathlen = strlen(filepath); - ut_a(pathlen < OS_FILE_MAX_PATH); - ut_a(0 == strcmp(filepath + pathlen - 4, DOT_IBD)); - - table->data_dir_path = mem_heap_strdup( - table->heap, filepath); - os_file_make_data_dir_path(table->data_dir_path); - } - - ut_free(default_filepath); - } -} - -/** Get the first filepath from mysql.tablespace_datafiles for a given space_id. +/** Get the first filepath from mysql.tablespace_datafiles +for a given space_id. @tparam Table dd::Table or dd::Partition @param[in,out] heap heap for store file name. @param[in] table dict table @@ -3333,8 +3300,8 @@ dd_get_first_path( char* filepath = nullptr; dd::Tablespace* dd_space = nullptr; THD* thd = current_thd; - MDL_ticket* mdl = nullptr; - dd::Object_id dd_space_id; + MDL_ticket* mdl = nullptr; + dd::Object_id dd_space_id; ut_ad(!srv_is_being_shutdown); ut_ad(!mutex_own(&dict_sys->mutex)); @@ -3351,13 +3318,13 @@ dd_get_first_path( table->name.m_name, db_buf, tbl_buf, nullptr, nullptr, nullptr) || dd_mdl_acquire(thd, &mdl, db_buf, tbl_buf)) { - return(filepath); + return(nullptr); } if (client->acquire(db_buf, tbl_buf, &table_def) || table_def == nullptr) { dd_mdl_release(thd, &mdl); - return(filepath); + return(nullptr); } dd_space_id = dd_first_index(table_def)->tablespace_id(); @@ -3372,17 +3339,22 @@ dd_get_first_path( ut_a(false); } - ut_a(dd_space != nullptr); - dd::Tablespace_file* dd_file = const_cast< - dd::Tablespace_file*>(*(dd_space->files().begin())); + if (dd_space != nullptr) { + dd::Tablespace_file* dd_file = const_cast< + dd::Tablespace_file*>(*(dd_space->files().begin())); - filepath = mem_heap_strdup(heap, dd_file->filename().c_str()); + filepath = mem_heap_strdup(heap, dd_file->filename().c_str()); + + return(filepath); + } - return(filepath); + return(nullptr); } -/** Make sure the data_dir_path is saved in dict_table_t if DATA DIRECTORY -was used. Try to read it from the fil_system first, then from NEW DD. +/** Make sure the data_dir_path is saved in dict_table_t if this is a +remote single file tablespace. This allows DATA DIRECTORY to be +displayed correctly for SHOW CREATE TABLE. Try to read the filepath +from the fil_system first, then from the DD. @tparam Table dd::Table or dd::Partition @param[in,out] table Table object @param[in] dd_table DD table object @@ -3396,40 +3368,40 @@ dd_get_and_save_data_dir_path( { mem_heap_t* heap = NULL; - if (DICT_TF_HAS_DATA_DIR(table->flags) - && (!table->data_dir_path)) { - char* path = fil_space_get_first_path(table->space); + if (!DICT_TF_HAS_DATA_DIR(table->flags) + || table->data_dir_path != nullptr) { + return; + } - if (!dict_mutex_own) { - dict_mutex_enter_for_mysql(); - } + char* path = fil_space_get_first_path(table->space); - if (path == NULL) { - heap = mem_heap_create(1000); + if (path == nullptr) { + heap = mem_heap_create(1000); + if (dict_mutex_own) { dict_mutex_exit_for_mysql(); - path = dd_get_first_path(heap, table, dd_table); + } + path = dd_get_first_path(heap, table, dd_table); + if (dict_mutex_own) { dict_mutex_enter_for_mysql(); } + } - if (path != NULL) { - dd_save_data_dir_path(table, path); - } + if (!dict_mutex_own) { + dict_mutex_enter_for_mysql(); + } - if (table->data_dir_path == NULL) { - /* Since we did not set the table data_dir_path, - unset the flag. */ - table->flags &= ~DICT_TF_MASK_DATA_DIR; - } + if (path != nullptr) { + dict_save_data_dir_path(table, path); + } - if (!dict_mutex_own) { - dict_mutex_exit_for_mysql(); - } + if (!dict_mutex_own) { + dict_mutex_exit_for_mysql(); + } - if (heap) { - mem_heap_free(heap); - } else { - ut_free(path); - } + if (heap != nullptr) { + mem_heap_free(heap); + } else { + ut_free(path); } } @@ -3452,28 +3424,18 @@ dd_get_meta_data_filename( char* filename, ulint max_len) { - ulint len; - char* path; - /* Make sure the data_dir_path is set. */ dd_get_and_save_data_dir_path(table, dd_table, false); - if (DICT_TF_HAS_DATA_DIR(table->flags)) { - ut_a(table->data_dir_path); + std::string path = dict_table_get_datadir(table); - path = fil_make_filepath( - table->data_dir_path, table->name.m_name, CFG, true); - } else { - path = fil_make_filepath(NULL, table->name.m_name, CFG, false); - } + auto filepath = Fil_path::make(path, table->name.m_name, CFG, true); - ut_a(path); - len = ut_strlen(path); - ut_a(max_len >= len); + ut_a(max_len >= strlen(filepath) + 1); - strcpy(filename, path); + strcpy(filename, filepath); - ut_free(path); + ut_free(filepath); } /** Opens a tablespace for dd_load_table_one() @@ -3495,7 +3457,8 @@ dd_load_tablespace( ut_ad(!table->is_temporary()); ut_ad(mutex_own(&dict_sys->mutex)); - /* The system and temporary tablespaces are preloaded and always available. */ + /* The system and temporary tablespaces are preloaded and + always available. */ if (fsp_is_system_or_temp_tablespace(table->space)) { return; } @@ -3548,9 +3511,10 @@ dd_load_tablespace( } /* The tablespace may already be open. */ - if (fil_space_for_table_exists_in_mem( + if (fil_space_exists_in_mem( table->space, space_name, false, true, heap, table->id)) { + ut_free(shared_space_name); return; } @@ -3562,32 +3526,24 @@ dd_load_tablespace( << table->space; } - /* Use the remote filepath if needed. This parameter is optional - in the call to fil_ibd_open(). If not supplied, it will be built - from the space_name. */ - char* filepath = nullptr; - if (DICT_TF_HAS_DATA_DIR(table->flags)) { - /* This will set table->data_dir_path from either - fil_system */ - dd_get_and_save_data_dir_path(table, dd_table, true); - - if (table->data_dir_path) { - filepath = fil_make_filepath( - table->data_dir_path, - table->name.m_name, IBD, true); - } - - } - else if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { + /* Try to get the filepath if this space_id is already open. + If the filepath is not found, fil_ibd_open() will make a default + filepath from the tablespace name */ + char* filepath = fil_space_get_first_path(table->space); + if (filepath == nullptr) { + /* boot_tablespaces() made sure that the scanned filepath + is in the DD even if the datafile was moved. So let's use + that path to open this tablespace. */ mutex_exit(&dict_sys->mutex); - filepath = dd_get_first_path(heap, table, dd_table); + char* filepath = dd_get_first_path(heap, table, dd_table); mutex_enter(&dict_sys->mutex); + if (filepath == nullptr) { - ib::warn() - << "Could not find the filepath for table " - << table->name - << ", space ID " << table->space; + ib::warn() << "Could not find the filepath" + << " for table " << table->name + << ", space ID " << table->space + << " in the data dictionary."; } else { alloc_from_heap = true; } @@ -3603,7 +3559,11 @@ dd_load_tablespace( true, FIL_TYPE_TABLESPACE, table->space, fsp_flags, space_name, tbl_name, filepath, true, false); - if (err != DB_SUCCESS) { + if (err == DB_SUCCESS) { + /* This will set the DATA DIRECTORY for SHOW CREATE TABLE. */ + dd_get_and_save_data_dir_path(table, dd_table, true); + + } else { /* We failed to find a sensible tablespace file */ table->ibd_file_missing = TRUE; } @@ -3628,23 +3588,23 @@ dd_space_get_name( dict_table_t* table, Table* dd_table) { - char* space_name = nullptr; - dd::Tablespace* dd_space = nullptr; - THD* thd = current_thd; - MDL_ticket* mdl = nullptr; dd::Object_id dd_space_id; + THD* thd = current_thd; + dd::Tablespace* dd_space = nullptr; ut_ad(!srv_is_being_shutdown); ut_ad(!mutex_own(&dict_sys->mutex)); - dd::cache::Dictionary_client* client = dd::get_dd_client(thd); - dd::cache::Dictionary_client::Auto_releaser releaser(client); + dd::cache::Dictionary_client* client = dd::get_dd_client(thd); + dd::cache::Dictionary_client::Auto_releaser releaser(client); if (dd_table == nullptr) { char db_buf[MAX_DATABASE_NAME_LEN + 1]; char tbl_buf[MAX_TABLE_NAME_LEN + 1]; const dd::Table* table_def = nullptr; + MDL_ticket* mdl = nullptr; + if (!dd_parse_tbl_name( table->name.m_name, db_buf, tbl_buf, nullptr, nullptr, nullptr) @@ -3653,7 +3613,8 @@ dd_space_get_name( } if (client->acquire(db_buf, tbl_buf, &table_def) - || table_def == nullptr) { + || table_def == nullptr) { + dd_mdl_release(thd, &mdl); return(nullptr); } @@ -3672,9 +3633,7 @@ dd_space_get_name( ut_a(dd_space != nullptr); - space_name = mem_heap_strdup(heap, dd_space->name().c_str()); - - return(space_name); + return(mem_heap_strdup(heap, dd_space->name().c_str())); } /** Make sure the tablespace name is saved in dict_table_t if the table @@ -5479,7 +5438,7 @@ dd_rename_fts_table( if (dd_rename_tablespace(table->dd_space_id, table->name.m_name, - new_path)) { + new_path) != DB_SUCCESS) { ut_a(false); } @@ -5563,7 +5522,9 @@ dd_get_referenced_table( { char* ref; const char* db_name; - bool is_part = (strstr(name, part_sep) != nullptr); + bool is_part; + + is_part = (strstr(name, PART_SEPARATOR) != nullptr); *table = nullptr; @@ -5596,7 +5557,8 @@ dd_get_referenced_table( } memcpy(ref, db_name, database_name_len); ref[database_name_len] = '/'; - memcpy(ref + database_name_len + 1, table_name, table_name_len + 1); + memcpy(ref + database_name_len + 1, + table_name, table_name_len + 1); } else { #ifndef _WIN32 @@ -5698,7 +5660,7 @@ dd_tablespace_update_cache(THD* thd) ut_ad(space->flags == flags); - fil_space_update_name(space, space_name, false); + fil_space_update_name(space, space_name); } else { fil_type_t purpose = fsp_is_system_temporary(id) diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 83d1a4d72ed4..de138edfb7e4 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -1666,14 +1666,20 @@ dict_table_rename_in_cache( HASH_SEARCH(name_hash, dict_sys->table_hash, fold, dict_table_t*, table2, ut_ad(table2->cached), (ut_strcmp(table2->name.m_name, new_name) == 0)); + DBUG_EXECUTE_IF("dict_table_rename_in_cache_failure", if (table2 == NULL) { table2 = (dict_table_t*) -1; } ); - if (table2) { - ib::error() << "Cannot rename table '" << old_name + + if (table2 != nullptr) { + + ib::error() + << "Cannot rename table '" << old_name << "' to '" << new_name << "' since the" - " dictionary cache already contains '" << new_name << "'."; + " dictionary cache already contains '" + << new_name << "'."; + return(DB_ERROR); } @@ -1689,22 +1695,28 @@ dict_table_rename_in_cache( /* Make sure the data_dir_path is set. */ dd_get_and_save_data_dir_path(table, NULL, true); - if (DICT_TF_HAS_DATA_DIR(table->flags)) { - ut_a(table->data_dir_path); + std::string path = dict_table_get_datadir(table); - filepath = fil_make_filepath( - table->data_dir_path, table->name.m_name, - IBD, true); - } else { - filepath = fil_make_filepath( - NULL, table->name.m_name, IBD, false); - } + filepath = Fil_path::make(path, table->name.m_name, IBD, true); if (filepath == NULL) { return(DB_OUT_OF_MEMORY); } - fil_delete_tablespace(table->space, BUF_REMOVE_ALL_NO_WRITE); + err = fil_delete_tablespace( + table->space, BUF_REMOVE_ALL_NO_WRITE); + + ut_a(err == DB_SUCCESS + || err == DB_TABLESPACE_NOT_FOUND + || err == DB_IO_ERROR); + + if (err == DB_IO_ERROR) { + + ib::info() + << "IO error while deleting: " << table->space + << " during rename of '" << old_name << "' to" + << " '" << new_name << "'"; + } /* Delete any temp file hanging around. */ if (os_file_status(filepath, &exists, &ftype) @@ -1724,11 +1736,14 @@ dict_table_rename_in_cache( ut_ad(!table->is_temporary()); if (DICT_TF_HAS_DATA_DIR(table->flags)) { - new_path = os_file_make_new_pathname( - old_path, new_name); + std::string new_ibd; + + new_ibd = Fil_path::make_new_ibd(old_path, new_name); + + new_path = mem_strdup(new_ibd.c_str()); + } else { - new_path = fil_make_filepath( - NULL, new_name, IBD, false); + new_path = Fil_path::make_ibd_from_table_name(new_name); } /* New filepath must not exist. */ @@ -1746,7 +1761,8 @@ dict_table_rename_in_cache( dd_filename_to_spacename(new_name, &new_tablespace_name); bool success = fil_rename_tablespace( - table->space, old_path, new_tablespace_name.c_str(), new_path); + table->space, old_path, new_tablespace_name.c_str(), + new_path); clone_mark_active(); @@ -2202,8 +2218,9 @@ dict_partitioned_table_remove_from_cache( if ((strncmp(name, prev_table->name.m_name, name_len) == 0) - && (strncmp(prev_table->name.m_name + name_len, - part_sep, strlen(part_sep)) == 0)) { + && strncmp(prev_table->name.m_name + name_len, + PART_SEPARATOR, + PART_SEPARATOR_LEN) == 0) { btr_drop_ahi_for_table(prev_table); dict_table_remove_from_cache(prev_table); @@ -8152,4 +8169,20 @@ dd_sdi_acquire_shared_mdl( return(DB_SUCCESS); } + +/** Get the tablespace data directory if set, otherwise empty string. +@return the data directory */ +std::string +dict_table_get_datadir(const dict_table_t* table) +{ + std::string path; + + if (DICT_TF_HAS_DATA_DIR(table->flags) + && table->data_dir_path != nullptr) { + + path.assign(table->data_dir_path); + } + + return(path); +} #endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/dict/dict0load.cc b/storage/innobase/dict/dict0load.cc index e645f2236fc1..33673158d27c 100644 --- a/storage/innobase/dict/dict0load.cc +++ b/storage/innobase/dict/dict0load.cc @@ -1144,7 +1144,7 @@ dict_get_first_path( /* The dictionary may have been written on another OS. */ - os_normalize_path(filepath); + Fil_path::normalize(filepath); } } } @@ -1427,7 +1427,7 @@ dict_check_sys_tablespaces( if (fsp_is_system_or_temp_tablespace(space_id) || fsp_is_undo_tablespace(space_id) || !fsp_is_shared_tablespace(fsp_flags) - || fil_space_for_table_exists_in_mem( + || fil_space_exists_in_mem( space_id, space_name, false, true, NULL, 0)) { continue; } @@ -1665,7 +1665,7 @@ dict_check_sys_tables( whether it is a shared tablespace or a single table tablespace, look to see if it is already in the tablespace cache. */ - if (fil_space_for_table_exists_in_mem( + if (fil_space_exists_in_mem( space_id, space_name, false, true, NULL, 0)) { ut_free(table_name.m_name); ut_free(space_name_from_dict); @@ -2210,34 +2210,48 @@ dict_load_table_low( return(NULL); } +/** Using the table->heap, copy the null-terminated filepath into +table->data_dir_path. The data directory path is derived from the +filepath by stripping the the table->name.m_name component suffix. +If the filepath is not of the correct form (".../db/table.ibd"), +then table->data_dir_path will remain nullptr. +@param[in,out] table table instance +@param[in] filepath filepath of tablespace */ void dict_save_data_dir_path( -/*====================*/ - dict_table_t* table, /*!< in/out: table */ - char* filepath) /*!< in: filepath of tablespace */ + dict_table_t* table, + char* filepath) { ut_ad(mutex_own(&dict_sys->mutex)); - ut_a(DICT_TF_HAS_DATA_DIR(table->flags)); - - ut_a(!table->data_dir_path); - ut_a(filepath); - - /* Be sure this filepath is not the default filepath. */ - char* default_filepath = fil_make_filepath( - NULL, table->name.m_name, IBD, false); - if (default_filepath) { - if (0 != strcmp(filepath, default_filepath)) { - ulint pathlen = strlen(filepath); - ut_a(pathlen < OS_FILE_MAX_PATH); - ut_a(0 == strcmp(filepath + pathlen - 4, DOT_IBD)); - - table->data_dir_path = mem_heap_strdup( - table->heap, filepath); - os_file_make_data_dir_path(table->data_dir_path); - } + ut_ad(DICT_TF_HAS_DATA_DIR(table->flags)); + ut_ad(table->data_dir_path == nullptr); + ut_a(Fil_path::has_ibd_suffix(filepath)); + + /* Ensure this filepath is not the default filepath. */ + char* default_filepath = Fil_path::make( + "", table->name.m_name, IBD); + + if (default_filepath == nullptr) { + /* Memory allocation problem. */ + return; + } + + if (strcmp(filepath, default_filepath) != 0) { + size_t pathlen = strlen(filepath); + + ut_a(pathlen < OS_FILE_MAX_PATH); + ut_a(Fil_path::has_ibd_suffix(filepath)); - ut_free(default_filepath); + char* data_dir_path = mem_heap_strdup(table->heap, filepath); + + Fil_path::make_data_dir_path(data_dir_path); + + if (strlen(data_dir_path)) { + table->data_dir_path = data_dir_path; + } } + + ut_free(default_filepath); } /** Make sure the data_dir_path is saved in dict_table_t if DATA DIRECTORY @@ -2249,34 +2263,30 @@ dict_get_and_save_data_dir_path( dict_table_t* table, bool dict_mutex_own) { - if (DICT_TF_HAS_DATA_DIR(table->flags) - && (!table->data_dir_path)) { - char* path = fil_space_get_first_path(table->space); + if (!(DICT_TF_HAS_DATA_DIR(table->flags) + && table->data_dir_path == nullptr)) { + return; + } - if (!dict_mutex_own) { - dict_mutex_enter_for_mysql(); - } + char* path = fil_space_get_first_path(table->space); - if (path == NULL) { - path = dict_get_first_path(table->space); - } + if (!dict_mutex_own) { + dict_mutex_enter_for_mysql(); + } - if (path != NULL) { - dict_save_data_dir_path(table, path); - ut_free(path); - } + if (path == nullptr) { + path = dict_get_first_path(table->space); + } - if (table->data_dir_path == NULL) { - /* Since we did not set the table data_dir_path, - unset the flag. This does not change SYS_DATAFILES - or SYS_TABLES or FSP_FLAGS on the header page of the - tablespace, but it makes dict_table_t consistent. */ - table->flags &= ~DICT_TF_MASK_DATA_DIR; - } + if (path != nullptr) { + dict_save_data_dir_path(table, path); + ut_free(path); + } - if (!dict_mutex_own) { - dict_mutex_exit_for_mysql(); - } + ut_ad(table->data_dir_path != nullptr); + + if (!dict_mutex_own) { + dict_mutex_exit_for_mysql(); } } @@ -2446,7 +2456,7 @@ dict_load_tablespace( } /* The tablespace may already be open. */ - if (fil_space_for_table_exists_in_mem( + if (fil_space_exists_in_mem( table->space, space_name, false, true, heap, table->id)) { ut_free(shared_space_name); @@ -2469,10 +2479,9 @@ dict_load_tablespace( fil_system or SYS_DATAFILES */ dict_get_and_save_data_dir_path(table, true); - if (table->data_dir_path) { - filepath = fil_make_filepath( - table->data_dir_path, - table->name.m_name, IBD, true); + if (table->data_dir_path != nullptr) { + filepath = Fil_path::make( + table->data_dir_path, table->name.m_name, IBD); } } else if (DICT_TF_HAS_SHARED_SPACE(table->flags)) { diff --git a/storage/innobase/dict/dict0stats.cc b/storage/innobase/dict/dict0stats.cc index ecf9ec47a0ec..6613f5299abe 100644 --- a/storage/innobase/dict/dict0stats.cc +++ b/storage/innobase/dict/dict0stats.cc @@ -3616,18 +3616,45 @@ dict_stats_evict_tablespaces() { ut_ad(srv_is_upgrade_mode); - space_id_t space_id_index_stats = fil_space_get_id_by_name(INDEX_STATS_NAME); - space_id_t space_id_table_stats = fil_space_get_id_by_name(TABLE_STATS_NAME); + space_id_t space_id_index_stats = + fil_space_get_id_by_name(INDEX_STATS_NAME); + + space_id_t space_id_table_stats = + fil_space_get_id_by_name(TABLE_STATS_NAME); + trx_t* trx = trx_allocate_for_background(); trx_start_internal(trx); if (space_id_index_stats != SPACE_UNKNOWN) { - fil_close_tablespace(trx, space_id_index_stats); + + dberr_t err; + + err = fil_close_tablespace(trx, space_id_index_stats); + + if (err != DB_SUCCESS) { + + ib::info() + << "dict_stats_evict_tablespace: " + << " fil_close_tablespace(" + << space_id_index_stats << ") failed! " + << ut_strerr(err); + } } if (space_id_table_stats != SPACE_UNKNOWN) { - fil_close_tablespace(trx, space_id_table_stats); + dberr_t err; + + err = fil_close_tablespace(trx, space_id_table_stats); + + if (err != DB_SUCCESS) { + + ib::info() + << "dict_stats_evict_tablespace: " + << " fil_close_tablespace(" + << space_id_index_stats << ") failed! " + << ut_strerr(err); + } } trx_commit_for_mysql(trx); diff --git a/storage/innobase/dict/dict0upgrade.cc b/storage/innobase/dict/dict0upgrade.cc index f799c6144bf4..3920527d3dca 100644 --- a/storage/innobase/dict/dict0upgrade.cc +++ b/storage/innobase/dict/dict0upgrade.cc @@ -679,7 +679,7 @@ static bool dd_upgrade_partitions(THD* thd, const char* norm_name, /* Set table id */ part_obj->set_se_private_id(part_table->id); - /* Set DATADIRECTORY attribute in se_private_data */ + /* Set DATA_DIRECTORY attribute in se_private_data */ if (DICT_TF_HAS_DATA_DIR(part_table->flags)) { ut_ad(dict_table_is_file_per_table(part_table)); part_obj->se_private_data().set_bool( @@ -872,7 +872,7 @@ bool dd_upgrade_table(THD* thd, const char* db_name, const char* table_name, dd_table->set_se_private_id(ib_table->id); - /* Set DATADIRECTORY attribute in se_private_data */ + /* Set DATA_DIRECTORY attribute in se_private_data */ if (DICT_TF_HAS_DATA_DIR(ib_table->flags)) { ut_ad(dict_table_is_file_per_table(ib_table)); dd_table->se_private_data().set_bool( diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 7ba21f10794d..9e4d0335a186 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -16,12 +16,8 @@ this program; if not, write to the Free Software Foundation, Inc., *****************************************************************************/ -/**************************************************//** -@file fil/fil0fil.cc -The tablespace memory cache - -Created 10/25/1995 Heikki Tuuri -*******************************************************/ +/** @file fil/fil0fil.cc +The tablespace memory cache */ #include "my_config.h" @@ -49,12 +45,11 @@ Created 10/25/1995 Heikki Tuuri #include "my_inttypes.h" #include "os0file.h" #include "page0zip.h" -#ifndef UNIV_HOTBACKUP -# include "row0mysql.h" -#endif /* !UNIV_HOTBACKUP */ #include "srv0start.h" #include "clone0api.h" + #ifndef UNIV_HOTBACKUP +# include "row0mysql.h" # include "trx0purge.h" # include "buf0lru.h" # include "ibuf0ibuf.h" @@ -62,28 +57,80 @@ Created 10/25/1995 Heikki Tuuri # include "sync0sync.h" #else /* !UNIV_HOTBACKUP */ # include "srv0srv.h" -#include +# include #endif /* !UNIV_HOTBACKUP */ +#include "os0thread-create.h" + +#include "current_thd.h" +#include "ha_prototypes.h" + #include #include #include +#include +#include +#include +#include #include +using Dirs = std::vector; +using Space_id_set = std::set; + +#if defined(__SUNPRO_CC) +char* Fil_path::SEPARATOR = "\\/"; +char* Fil_path::DOT_SLASH = "./"; +char* Fil_path::DOT_DOT_SLASH = "../"; +#endif /* defined(__SUNPRO_CC) */ + +/** Used for collecting the data in boot_tablespaces() */ +namespace dd_fil { + + enum { + /** DD Object ID */ + OBJECT_ID, + + /** InnoDB tablspace ID */ + SPACE_ID, + + /** DD/InnoDB tablespace name */ + SPACE_NAME, + + /** Path in DD tablespace */ + OLD_PATH, + + /** Path where it was found during the scan. */ + NEW_PATH + }; + + using Moved = std::tuple< + dd::Object_id, space_id_t, + std::string, std::string, std::string>; + + using Tablespaces = std::vector; +} + +/* uint16_t is the index into Tablespace_dirs::m_dirs */ +using Scanned_files = std::vector>; + #ifdef UNIV_PFS_IO mysql_pfs_key_t innodb_tablespace_open_file_key; #endif /* UNIV_PFS_IO */ +/** System tablespace. */ fil_space_t* fil_space_t::s_sys_space; +/** Redo log tablespace */ +fil_space_t* fil_space_t::s_redo_space; + #ifdef UNIV_HOTBACKUP /** Directories in which remote general tablespaces have been found in the target directory during apply log operation */ -dir_set rem_gen_ts_dirs; +Dir_set rem_gen_ts_dirs; -/* true in case the apply-log operation is being performed +/** true in case the apply-log operation is being performed in the data directory */ -bool replay_in_datadir = false; +bool replay_in_datadir = false; /* Re-define mutex macros to use the Mutex class defined by the MEB source. MEB calls the routines in "fil0fil.cc" in parallel and, @@ -104,30 +151,38 @@ module. */ # define mutex_own(M) 1 # define mutex_validate(M) 1 -/* Re-define the mutex macros for the mutex protecting the critical -sections of the log subsystem using an object of the meb::Mutex -class. */ - -meb::Mutex log_mutex; +/** Process a MLOG_FILE_CREATE redo record. +@param[in] page_id Page id of the redo log record +@param[in] flags Tablespace flags +@param[in] name Tablespace filename */ +static +void +meb_tablespace_redo_create( + const page_id_t& page_id, + ulint flags, + const char* name); -# undef log_mutex_enter -# undef log_mutex_exit -# define log_mutex_enter() log_mutex.lock() -# define log_mutex_exit() log_mutex.unlock() -#endif /* UNIV_HOTBACKUP */ +/** Process a MLOG_FILE_RENAME redo record. +@param[in] page_id Page id of the redo log record +@param[in] from_name Tablespace from filename +@param[in] to_name Tablespace to filename */ +static +void +meb_tablespace_redo_rename( + const page_id_t& page_id, + const char* from_name, + const char* to_name); -/** Tries to close a file in the LRU list. The caller must hold the fil_sys -mutex. This function will release the fil sys mutex temporarily. -@return true if success, false if should retry later; since i/o's -generally complete in < 100 ms, and as InnoDB writes at most 128 pages -from the buffer pool in a batch, and then immediately flushes the -files, there is a good chance that the next time we find a suitable -node from the LRU list. -@param[in] print_info if true, prints information why it - cannot close a file */ +/** Process a MLOG_FILE_DELETE redo record. +@param[in] page_id Page id of the redo log record +@param[in] name Tablespace filename */ static -bool -fil_try_to_close_file_in_LRU(bool print_info); +void +meb_tablespace_redo_delete( + const page_id_t& page_id, + const char* name); + +#endif /* UNIV_HOTBACKUP */ /* IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE @@ -137,63 +192,57 @@ The tablespace cache is responsible for providing fast read/write access to tablespaces and logs of the database. File creation and deletion is done in other modules which know more of the logic of the operation, however. -A tablespace consists of a chain of files. The size of the files does not -have to be divisible by the database block size, because we may just leave -the last incomplete block unused. When a new file is appended to the -tablespace, the maximum size of the file is also specified. At the moment, -we think that it is best to extend the file to its maximum size already at -the creation of the file, because then we can avoid dynamically extending -the file when more space is needed for the tablespace. +Only the system tablespace consists of a list of files. The size of these +files does not have to be divisible by the database block size, because +we may just leave the last incomplete block unused. When a new file is +appended to the tablespace, the maximum size of the file is also specified. +At the moment, we think that it is best to extend the file to its maximum +size already at the creation of the file, because then we can avoid dynamically +extending the file when more space is needed for the tablespace. + +Non system tablespaces contain only a single file. A block's position in the tablespace is specified with a 32-bit unsigned -integer. The files in the chain are thought to be catenated, and the block +integer. The files in the list are thought to be catenated, and the block corresponding to an address n is the nth block in the catenated file (where the first block is named the 0th block, and the incomplete block fragments at the end of files are not taken into account). A tablespace can be extended -by appending a new file at the end of the chain. +by appending a new file at the end of the list. Our tablespace concept is similar to the one of Oracle. -To acquire more speed in disk transfers, a technique called disk striping is -sometimes used. This means that logical block addresses are divided in a -round-robin fashion across several disks. Windows NT supports disk striping, -so there we do not need to support it in the database. Disk striping is -implemented in hardware in RAID disks. We conclude that it is not necessary -to implement it in the database. Oracle 7 does not support disk striping, -either. - -Another trick used at some database sites is replacing tablespace files by -raw disks, that is, the whole physical disk drive, or a partition of it, is -opened as a single file, and it is accessed through byte offsets calculated -from the start of the disk or the partition. This is recommended in some -books on database tuning to achieve more speed in i/o. Using raw disk -certainly prevents the OS from fragmenting disk space, but it is not clear -if it really adds speed. We measured on the Pentium 100 MHz + NT + NTFS file -system + EIDE Conner disk only a negligible difference in speed when reading -from a file, versus reading from a raw disk. - To have fast access to a tablespace or a log file, we put the data structures to a hash table. Each tablespace and log file is given an unique 32-bit -identifier. +identifier, its tablespace ID. Some operating systems do not support many open files at the same time, -though NT seems to tolerate at least 900 open files. Therefore, we put the -open files in an LRU-list. If we need to open another file, we may close the -file at the end of the LRU-list. When an i/o-operation is pending on a file, -the file cannot be closed. We take the file nodes with pending i/o-operations -out of the LRU-list and keep a count of pending operations. When an operation -completes, we decrement the count and return the file node to the LRU-list if -the count drops to zero. */ +Therefore, we put the open files in an LRU-list. If we need to open another +file, we may close the file at the end of the LRU-list. When an I/O-operation +is pending on a file, the file cannot be closed. We take the file nodes with +pending I/O-operations out of the LRU-list and keep a count of pending +operations. When an operation completes, we decrement the count and return +the file to the LRU-list if the count drops to zero. + +The data structure (Fil_shard) that keeps track of the tablespace ID to +fil_space_t* mapping are hashed on the tablespace ID. The tablespace name to +fil_space_t* mapping is stored in the same shard. A shard tracks the flushing +and open state of a file. When we run out open file handles, we use a ticketing +system to serialize the file open, see Fil_shard::reserve_open_slot() and +Fil_shard::release_open_slot(). + +When updating the global/shared data in Fil_system acquire the mutexes of +all shards in ascending order. The shard mutex covers the fil_space_t data +members as noted in the fil_space_t and fil_node_t definition. */ /** This tablespace name is used internally during recovery to open a general tablespace before the data dictionary are recovered and available. */ const char general_space_name[] = "innodb_general"; -/** Reference to the server data directory. Usually it is the -current working directory ".", but in the MySQL Embedded Server Library -it is an absolute path. */ -const char* fil_path_to_mysql_datadir; -Folder folder_mysql_datadir; +/** Reference to the server data directory. */ +Fil_path MySQL_datadir_path; + +/** Sentinel value to check for "NULL" Fil_path. */ +Fil_path Fil_path::s_null_path; /** Common InnoDB file extentions */ const char* dot_ext[] = { "", ".ibd", ".cfg", ".cfp" }; @@ -203,6 +252,7 @@ ulint fil_n_log_flushes = 0; /** Number of pending redo log flushes */ ulint fil_n_pending_log_flushes = 0; + /** Number of pending tablespace flushes */ ulint fil_n_pending_tablespace_flushes = 0; @@ -227,10 +277,54 @@ enum fil_load_status { FIL_LOAD_MISMATCH }; +/** File operations for tablespace */ +enum fil_operation_t { + + /** delete a single-table tablespace */ + FIL_OPERATION_DELETE, + + /** close a single-table tablespace */ + FIL_OPERATION_CLOSE +}; + /** The null file address */ fil_addr_t fil_addr_null = {FIL_NULL, 0}; -using Spaces = std::unordered_map; +/** Maximum number of threads to use for scanning data files. */ +static const size_t MAX_SCAN_THREADS = 8; + +#ifndef UNIV_HOTBACKUP +/** Maximum number of shards supported. */ +static const size_t MAX_SHARDS = 64; + +/** The redo log is in its own shard. */ +static const size_t REDO_SHARD = MAX_SHARDS - 1; + +/** Number of undo shards to reserve. */ +static const size_t UNDO_SHARDS = 4; + +/** The UNDO logs have their own shards (4). */ +static const size_t UNDO_SHARDS_START = REDO_SHARD - (UNDO_SHARDS + 1); +#else /* !UNIV_HOTBACKUP */ + +/** Maximum number of shards supported. */ +static const size_t MAX_SHARDS = 1; + +/** The redo log is in its own shard. */ +static const size_t REDO_SHARD = 0; + +/** The UNDO logs have their own shards (4). */ +static const size_t UNDO_SHARDS_START = 0; +#endif /* !UNIV_HOTBACKUP */ + +/** Maximum pages to check for valid space ID during start up. */ +static const size_t MAX_PAGES_TO_CHECK = 4; + +/** Sentinel for empty open slot. */ +static const size_t EMPTY_OPEN_SLOT = std::numeric_limits::max(); + +/** We want to store the line number from where it was called. */ +#define mutex_acquire() acquire(__LINE__) /** Hash a NUL terminated 'string' */ struct Char_Ptr_Hash { @@ -257,692 +351,1400 @@ struct Char_Ptr_Compare { } }; -/** Maximum number of tablespaces.open.* files */ -static const uint32_t MAX_TABLESPACE_OPEN_FILES = 2; +/** Tablespace files disovered during startup. */ +class Tablespace_files { +public: + using Names = std::vector>; + using Paths = std::unordered_map; -using Names = std::unordered_map< - const char*, fil_space_t*, Char_Ptr_Hash, Char_Ptr_Compare>; + /** Default constructor + @param[in] dir Directory that the files are under */ + explicit Tablespace_files(const std::string& dir); -/* File Handle for tablespaces.open.* file */ -struct Fil_Open_Hdl { - /** Path to tablespace open file */ - std::string path; - /** File handle for tablespace open state files. */ - pfs_os_file_t handle; -}; + /** Add a space ID to filename mapping. + @param[in] space_id Tablespace ID + @param[in] name File name. + @return number of files that map to the space ID */ + size_t add(space_id_t space_id, const std::string& name) + MY_ATTRIBUTE((warn_unused_result)); -/** For tracking space ID to physical filename during recovery */ -struct Fil_Open { + /** Get the file names that map to a space ID + @param[in] space_id Tablespace ID + @return the filenames that map to space id */ + Names* find(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)) + { + ut_ad(space_id != TRX_SYS_SPACE); - /** Persistent file format version number. */ - static const uint32_t VERSION_1 = 1; + if (dict_sys_t::is_reserved(space_id) + && space_id != dict_sys_t::s_space_id) { - /** Store the mapping from space ID to physical filename */ - struct Nodes { + auto it = m_undo_paths.find(space_id); + + if (it != m_undo_paths.end()) { + return(&it->second); + } - /** State of the file in memeory. */ - enum State : uint8_t { + } else { - /** Initial state */ - INIT, + auto it = m_ibd_paths.find(space_id); - /** File is open */ - OPEN, + if (it != m_ibd_paths.end()) { + return(&it->second); + } + } - /** File is closed or deleted. */ - CLOSED, + return(nullptr); + } - /** Tablespace has been dropped. */ - DELETED, + /** Remove the entry for the space ID. + @param[in] space_id Tablespace ID mapping to remove + @return true if erase successful */ + bool erase(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)) + { + ut_ad(space_id != TRX_SYS_SPACE); - /** File is missing. */ - MISSING, + if (dict_sys_t::is_reserved(space_id) + && space_id != dict_sys_t::s_space_id) { - /** Failed to open file. */ - OPEN_FAILED - }; + auto n_erased = m_undo_paths.erase(space_id); - /** For tracking state of a filename */ - struct File { + return(n_erased == 1); + } else { + auto n_erased = m_ibd_paths.erase(space_id); - /** Constructor - @param[in] name Filename - @param[in] state File state - @param[in] lsn LSN of OPEN/RENAME redo */ - File(const std::string& name, State state, lsn_t lsn) - : - m_name(name), - m_state(state), - m_open_lsn(lsn), - m_closed_lsn() { } + return(n_erased == 1); + } - /** File name */ - std::string m_name; + return(false); + } - /** State of the file*/ - State m_state; + /** Clear all the tablespace data. */ + void clear() + { + m_ibd_paths.clear(); + m_undo_paths.clear(); + } - /** Commit LSN of latest MLOG_FILE_OPEN - or MLOG_FILE_RENAME entry */ - lsn_t m_open_lsn; + /** @return m_dir */ + const Fil_path& root() const + { + return(m_dir); + } - /** System LSN when the file was closed */ - lsn_t m_closed_lsn; - }; + /** @return the directory path specified by the user. */ + const std::string& path() const + { + return(m_dir.path()); + } - using Paths = std::vector>; + /** @return the real path of the directory searched. */ + const std::string& real_path() const + { + return(m_dir.abs_path()); + } - /** Constructor */ - Nodes() - : - m_id(std::numeric_limits::max()) - { } +private: + /* Note: The file names in m_ibd_paths and m_undo_paths are relative + to m_real_path. */ - /** Constructor - @param[in] space_id Tablespace ID */ - explicit Nodes(space_id_t space_id) - : - m_id(space_id) - { } + /** Mapping from tablespace ID to data filenames */ + Paths m_ibd_paths; - /** Note the state of the tablespace filename - @param[in] path Physical path of the file opened - @param[in] state OPEN or MISSING - @param[in] lsn Commit LSN of the redo log */ - void load( - const std::string& path, - State state, - lsn_t lsn); + /** Mapping from tablespace ID to Undo files */ + Paths m_undo_paths; - /** Note that a tablespace file has been closed. - @param[in] path Filename to close. */ - void close(const std::string& path); + /** Top level directory where the above files were found. */ + const Fil_path m_dir; +}; - /** Rename a file. - @param[in] from Filename to rename - @param[in] to Filename to rename to - @retval true on success */ - bool rename(const std::string& from, const std::string& to); +/** Directories scanned during startup and the files discovered. */ +class Tablespace_dirs { +public: + using Result = std::pair; - /** Remove all nodes that are deleted */ - void purge(); + /** Constructor */ + Tablespace_dirs() : m_dirs(), m_checked() { } - /** Remove all files */ - void clear(); + /** Discover tablespaces by reading the header from .ibd files. + @param[in] in_directories Directories to scan + @return DB_SUCCESS if all goes well */ + dberr_t scan(const std::string& in_directories) + MY_ATTRIBUTE((warn_unused_result)); - /** Mark all files as deleted */ - void deleted(); - - /** Lookup a file name in the files list - @return iterator from search result */ - Paths::iterator lookup(const std::string& path) - { - return(std::find_if( - m_files.begin(), - m_files.end(), - [&](const File& elem) -> bool - { - return(elem.m_name.compare(path) == 0); - })); + /** Clear all the tablespace file data but leave the list of + scanned directories in place. */ + void clear() + { + for (auto& dir : m_dirs) { + dir.clear(); } - /** @return true if no files are open */ - bool empty() const - { - return(m_files.empty()); - } + m_checked = 0; + } - /** @return total number of files opened */ - size_t size() const - { - return(m_files.size()); + /** Erase a space ID to filename mapping. + @param[in] space_id Tablespace ID to erase + @return true if successful */ + bool erase(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)) + { + for (auto& dir : m_dirs) { + + if (dir.erase(space_id)) { + + return(true); + } } - /** @return string represenation */ - std::string to_string() const - { - std::ostringstream str; - bool comma = false; + return(false); + } - for (auto& path : m_files) { + /* Find the first matching space ID -> name mapping. + @param[in] space_id Tablespace ID + @return directory searched and pointer to names that map to the + tablespace ID */ + Result find(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)) + { + for (auto& dir : m_dirs) { - if (comma) { - str << ", "; - } else { - comma = true; - } + const auto names = dir.find(space_id); - str << "'" << path.m_name << "'"; + if (names != nullptr) { + return(Result{dir.path(), names}); } - - return(str.str()); } - /** Tablespace ID */ - space_id_t m_id; - - /** Physical files that are part of this tablespace */ - Paths m_files; - }; + return(Result{"", nullptr}); + } - /** Constructor */ - Fil_Open() - : - m_needs_flush(), - m_next() + /** @return the directory that contains path */ + const Fil_path& contains(const std::string& path) const + MY_ATTRIBUTE((warn_unused_result)) { - mutex_create(LATCH_ID_FILE_OPEN, &m_mutex); - for (Fil_Open_Hdl& hdl : m_handles) { - hdl.handle.m_file = OS_FILE_CLOSED; + Fil_path file{path}; + + for (const auto& dir : m_dirs) { + const auto& d = dir.root().abs_path(); + auto abs_path = Fil_path::get_real_path(d); + + if (dir.root().is_ancestor(file) + || abs_path.compare(file.abs_path()) == 0) { + + return(dir.root()); + } } + + return(Fil_path::null()); } - /** Copy constructor - @param[in] other Instance to copy from */ - Fil_Open(const Fil_Open& other) - : - m_spaces(other.m_spaces), - m_needs_flush(), - m_next(other.m_next) + /** Get the list of directories that InnoDB knows about. + @return the list of directories 'dir1;dir2;....;dirN' */ + std::string get_dirs() const { - mutex_create(LATCH_ID_FILE_OPEN, &m_mutex); + std::string dirs; + + ut_ad(!m_dirs.empty()); - for (uint32_t i = 0; i < MAX_TABLESPACE_OPEN_FILES; i++) { - m_handles.at(i) = other.m_handles.at(i); + for (const auto& dir : m_dirs) { + + dirs.append(dir.root()); + dirs.push_back(FIL_PATH_SEPARATOR); } + + dirs.pop_back(); + + ut_ad(!dirs.empty()); + + return(dirs); } +private: + /** Print the duplicate filenames for a tablespce ID to the log + @param[in] duplicates Duplicate tablespace IDs*/ + void print_duplicates(const Space_id_set& duplicates); + + /** first=dir path from the user, second=files found under first. */ + using Scanned = std::vector; + + /** Tokenize a path specification. Convert relative paths to + absolute paths. Check if the paths are valid and filter out + invalid or unreadable directories. Sort and filter out duplicates + from dirs. + @param[in] str Path specification to tokenize + @param[in] delimiters Delimiters */ + void tokenize_paths( + const std::string& str, + const std::string& delimiters); + + using Const_iter = Scanned_files::const_iterator; + + /** Check for duplicate tablespace IDs. + @param[in] start Start of slice + @param[in] end End of slice + @param[in] thread_id Thread ID + @param[in,out] mutex Mutex protecting the global state + @param[in,out] unique To check for duplciates + @param[in,out] duplicates Duplicate space IDs found */ + void duplicate_check( + const Const_iter& start, + const Const_iter& end, + size_t thread_id, + std::mutex* mutex, + Space_id_set* unique, + Space_id_set* duplicates); + +private: + /** Directories scanned and the files discovered under them. */ + Scanned m_dirs; + + /** Number of files checked. */ + std::atomic_size_t m_checked; +}; + +/** Determine if user has explicitly disabled fsync(). */ +#ifndef _WIN32 +# define fil_buffering_disabled(s) \ + ((s)->purpose == FIL_TYPE_TABLESPACE \ + && srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC) +#else /* _WIN32 */ +# define fil_buffering_disabled(s) (0) +#endif /* _WIN32 */ + +class Fil_shard { + + using File_list = UT_LIST_BASE_NODE_T(fil_node_t); + using Space_list = UT_LIST_BASE_NODE_T(fil_space_t); + using Spaces = std::unordered_map; + + using Names = std::unordered_map< + const char*, fil_space_t*, Char_Ptr_Hash, Char_Ptr_Compare>; + +public: + /** Constructor + @param[in] shard_id Shard ID */ + explicit Fil_shard(size_t shard_id); + /** Destructor */ - ~Fil_Open() + ~Fil_shard() { +#ifndef UNIV_HOTBACKUP + mutex_destroy(&m_mutex); +#else mutex_free(&m_mutex); +#endif /* !UNIV_HOTBACKUP */ - for (Fil_Open_Hdl& hdl : m_handles) { - if (hdl.handle.m_file != OS_FILE_CLOSED) { - os_file_close(hdl.handle); - hdl.handle.m_file = OS_FILE_CLOSED; - } - } + ut_a(UT_LIST_GET_LEN(m_LRU) == 0); + ut_a(UT_LIST_GET_LEN(m_unflushed_spaces) == 0); + } + + /** @return the shard ID */ + size_t id() const + { + return(m_id); } - /** Acquire the mutex. */ - void enter() + /** Acquire the mutex. + @param[in] line Line number from where it was called */ + void acquire(int line) const { +#ifndef UNIV_HOTBACKUP + m_mutex.enter( + srv_n_spin_wait_rounds, srv_spin_wait_delay, + __FILE__, line); +#else mutex_enter(&m_mutex); +#endif /* !UNIV_HOTBACKUP */ } /** Release the mutex. */ - void exit() + void mutex_release() const { mutex_exit(&m_mutex); } - /** Create tablespaces.open.* files */ - void create_open_files(); +# ifdef UNIV_DEBUG + /** @return true if the mutex is owned. */ + bool mutex_owned() const + { + return(mutex_own(&m_mutex)); + } +# endif /* UNIV_DEBUG */ - /** Redo log an open file request. - @param[in] space_id Tablespace ID being opened - @param[in] path Physical path of the file opened */ - void log(space_id_t space_id, const std::string& path); + /** Mutex protecting this shard. */ - /** Get the first name a tablespace ID maps to. - @param[in] space_id Tablespace ID to lookup - @return first name in the mapping or empty string if not found */ - std::string fetch(space_id_t space_id) const; +#ifndef UNIV_HOTBACKUP + mutable ib_mutex_t m_mutex; +#else + mutable meb::Mutex m_mutex; +#endif /* !UNIV_HOTBACKUP */ - /** Check if a space ID is known. + /** Fetch the fil_space_t instance that maps to space_id. @param[in] space_id Tablespace ID to lookup - @return true if the space ID is known*/ - bool lookup(space_id_t space_id) const; - - /** Rename a file for a given tablespace ID. - @param[in] space_id Tablespace ID - @param[in] from Filename to rename - @param[in] to Filename to rename to - @retval true on success */ - bool rename( - space_id_t space_id, - const std::string& from, - const std::string& to); + @return tablespace instance or nullptr if not found. */ + fil_space_t* get_space_by_id(space_id_t space_id) const + MY_ATTRIBUTE((warn_unused_result)) + { + ut_ad(m_id == REDO_SHARD || mutex_owned()); - /** Note that a tablespace file has been opened. - @param[in] space_id Tablespace ID - @param[in] path Physical path of the file opened - @param[in] lsn LSN of the mtr commit */ - void open(space_id_t space_id, const std::string& path, lsn_t lsn); + auto it = m_spaces.find(space_id); - /** Note that state of the tablespace filename - @param[in] space_id Tablespace ID - @param[in] path Physical path of the file opened - @param[in] state state of the file - @param[in] lsn Commit LSN of the redo log */ - void load( - space_id_t space_id, - const std::string& path, - Nodes::State state, - lsn_t lsn); + if (it == m_spaces.end()) { + return(nullptr); + } - /** Note that a tablespace file has been closed. - @param[in] space_id Tablespace ID - @param[in] path Physical path of the file closed */ - void close(space_id_t space_id, const std::string& path); + ut_ad(it->second->magic_n == FIL_SPACE_MAGIC_N); - /** Remove all the tablespace file names that are not required for - recovery to work. */ - void purge(); + return(it->second); + } - /** Remove all nodes */ - void clear(); + /** Fetch the fil_space_t instance that maps to the name. + @param[in] name Tablespace name to lookup + @return tablespace instance or nullptr if not found. */ + fil_space_t* get_space_by_name(const char* name) const + MY_ATTRIBUTE((warn_unused_result)) + { + ut_ad(mutex_owned()); - /** Note that a tablespace ID has been dropped/deleted. - @param[in] space_id Space id to delete */ - bool deleted(space_id_t space_id); + auto it = m_names.find(name); - /** Check if the file can be opened - @param[in] space_id Tablespace ID - @param[in] path Physical path of file to open - @param[in] lsn LSN of the redo log entry - @return true if file can be opened */ - bool can_load(space_id_t space_id, const std::string& path, lsn_t lsn); - - /** Write the state to disk */ - void to_file(); - - /** Read the state from disk - @param[in] recovery true if called from crash recovery */ - void from_file(bool recovery); - - /** Read tablespaces from data directory */ - void from_current_dir(); - - /** Convert to a string */ - std::string to_string() const; - - /** Parse the input stream and populate the data structure. - @param[in,out] is Input stream to read - @param[out] max_lsn Maximum LSN read from the file - @param[in] recovery if called from recovery - @return true on success */ - bool parse(std::istream& is, lsn_t& max_lsn, bool recovery); - - /** Write the data to the file. - @param[in] version File format version - @param[in] original_len Uncompressed data len - @param[in] compressed_len Compressed data len - @param[in] data The data to write */ - void write( - size_t version, - size_t original_len, - size_t compressed_len, - const byte* data); - - /** Absolute path of file. - @param[in] dir Parent directory (can be '.') - @param[in] filename Filename to read/write - @return absolute path name */ - static std::string get_path( - const char* dir, - const std::string& filename); - - /** Check if the name is an undo tablespace name. - @param[in] name Tablespace name - @param[in] len Tablespace name length in bytes - @return true if it is an undo tablespace name */ - static bool is_undo_tablespace_name(const char* name, ulint len); - - /** Open the tablespace for recovery. Get the tablespace filenames, - space_id must already be known. - @param[in] space_id Tablespace ID to open */ - void open_for_recovery(space_id_t space_id) const; - - using Spaces = std::unordered_map; - - /** Mutex protecting changes to the data structures and synchronizing - writes to the log and serialization. */ - ib_mutex_t m_mutex; - - /** Tablespace ID to filename mapping */ - Spaces m_spaces; + if (it == m_names.end()) { + return(nullptr); + } - /** If true then we need to sync the data to disk */ - bool m_needs_flush; + ut_ad(it->second->magic_n == FIL_SPACE_MAGIC_N); - /** Next file number from PATHS. */ - size_t m_next; + return(it->second); + } - /** Paths to tablespace open files. */ - std::array m_handles; -}; + /** Tries to close a file in the shard LRU list. + The caller must hold the Fil_shard::m_mutex. + @param[in] print_info if true, prints information + why it cannot close a file + @return true if success, false if should retry later */ + bool close_files_in_LRU(bool print_info) + MY_ATTRIBUTE((warn_unused_result)); -/** Tablespace open state files. */ -static const std::vector PATHS = { - "tablespaces.open.1", - "tablespaces.open.2", -}; + /** Remove the file node from the LRU list. + @param[in,out] file File for the tablespace */ + void remove_from_LRU(fil_node_t* file); -/** Tablespace ID to filename mapping that was recovered by scanning the -directories if --innobase-scan-directories was set. */ -static Fil_Open* fil_scanned; + /** Add the file node to the LRU list if required. + @param[in,out] file File for the tablespace */ + void file_opened(fil_node_t* file); -/** Note the state of the tablespace filename -@param[in] path Physical path of the file opened -@param[in] state OPEN or MISSING -@param[in] lsn Commit LSN of the redo log */ -void -Fil_Open::Nodes::load( - const std::string& path, - Fil_Open::Nodes::State state, - lsn_t lsn) -{ - auto it = lookup(path); + /** Open all the system files. + @param[in] max_n_open Max files that can be opened. + @param[in] n_open Current number of open files */ + void open_system_tablespaces(size_t max_n_open, size_t* n_open); - if (it == m_files.end()) { + /** Close a tablespace file. + @param[in,out] file Tablespace file to close + @param[in] LRU_close true if called from LRU close */ + void close_file(fil_node_t* file, bool LRU_close); - Paths::value_type value(path, state, lsn); + /** Close a tablespace file based on tablespace ID. + @param[in] space_id Tablespace ID + @return false if space_id was not found. */ + bool close_file(space_id_t space_id); + + /** Prepare to free a file object from a tablespace + memory cache. + @param[in,out] file Tablespace file + @param[in] space tablespace */ + void file_close_to_free(fil_node_t* file, fil_space_t* space); + + /** Close log files. + @param[in] free_all If set then free all */ + void close_log_files(bool free_all); + + /** Close all open files. */ + void close_all_files(); + + /** Detach a space object from the tablespace memory cache. + Closes the tablespace files but does not delete them. + There must not be any pending I/O's or flushes on the files. + @param[in,out] space tablespace */ + void space_detach(fil_space_t* space); + + /** Delete the instance that maps to space_id + @param[in] space_id Tablespace ID to delete */ + void space_delete(space_id_t space_id) + { + ut_ad(mutex_owned()); - m_files.push_back(value); + auto it = m_spaces.find(space_id); - } else if (state == MISSING) { + if (it != m_spaces.end()) { + m_names.erase(it->second->name); + m_spaces.erase(it); + } + } - ut_ad(recv_needed_recovery); + /** Frees a space object from the tablespace memory cache. + Closes a tablespaces' files but does not delete them. + There must not be any pending I/O's or flushes on the files. + @param[in] space_id Tablespace ID + @return fil_space_t instance on success or nullptr */ + fil_space_t* space_free(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Map the space ID and name to the tablespace instance. + @param[in] space Tablespace instance */ + void space_add(fil_space_t* space); + + /** Prepare to free a file. Remove from the unflushed list + if there are no pending flushes. + @param[in,out] file File instance to free */ + void prepare_to_free_file(fil_node_t* file); + + /** If the tablespace is on the unflushed list and there + are no pending flushes then remove from the unflushed list. + @param[in,out] space Tablespace to remove*/ + void remove_from_unflushed_list(fil_space_t* space); + + /** Updates the data structures when an I/O operation + finishes. Updates the pending I/O's field in the file + appropriately. + @param[in] file Tablespace file + @param[in] type Marks the file as modified + if type == WRITE */ + void complete_io(fil_node_t* file, const IORequest& type); + + /** Prepares a file for I/O. Opens the file if it is closed. + Updates the pending I/O's field in the file and the system + appropriately. Takes the file off the LRU list if it is in + the LRU list. + @param[in] file Tablespace file for IO + @param[in] extend true if file is being extended + @return false if the file can't be opened, otherwise true */ + bool prepare_file_for_io(fil_node_t* file, bool extend) + MY_ATTRIBUTE((warn_unused_result)); + + /** Reserves the mutex and tries to make sure we can + open at least one file while holding it. This should be called + before calling prepare_file_for_io(), because that function + may need to open a file. + @param[in] space_id Tablespace ID + @param[out] space Tablespace instance + @return true if a slot was reserved. */ + bool mutex_acquire_and_get_space( + space_id_t space_id, + fil_space_t*& space) + MY_ATTRIBUTE((warn_unused_result)); + + /** Remap the tablespace to the new name. + @param[in] space Tablespace instance with old name + @param[in] new_name New tablespace name */ + void update_space_name_map(fil_space_t* space, const char* new_name); + + /** Flush the redo log writes to disk, possibly cached by the OS. */ + void flush_file_redo(); + + /** Collect the tablespace IDs of unflushed tablespaces in space_ids. + @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG, + can be ORred */ + void flush_file_spaces(uint8_t purpose); + + /** Try to extend a tablespace if it is smaller than the specified size. + @param[in,out] space tablespace + @param[in] size desired size in pages + @return whether the tablespace is at least as big as requested */ + bool space_extend(fil_space_t* space, page_no_t size) + MY_ATTRIBUTE((warn_unused_result)); + + /** Flushes to disk possible writes cached by the OS. If the space does + not exist or is being dropped, does not do anything. + @param[in] space_id File space ID (this can be a group of + log files or a tablespace of the + database) */ + void space_flush(space_id_t space_id); + + /** Open a file of a tablespace. + The caller must own the fil_system mutex. + @param[in,out] file Tablespace file + @param[in] extend true if the file is being extended + @return false if the file can't be opened, otherwise true */ + bool open_file(fil_node_t* file, bool extend) + MY_ATTRIBUTE((warn_unused_result)); + + /** Checks if all the file nodes in a space are flushed. + The caller must hold all fil_system mutexes. + @param[in] space Tablespace to check + @return true if all are flushed */ + bool space_is_flushed(const fil_space_t* space) + MY_ATTRIBUTE((warn_unused_result)); + + /** Open each file of a tablespace if not already open. + @param[in] space_id tablespace identifier + @retval true if all file nodes were opened + @retval false on failure */ + bool space_open(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Opens the files associated with a tablespace and returns a + pointer to the fil_space_t that is in the memory cache associated + with a space id. + @param[in] space_id Get the tablespace instance or this ID + @return file_space_t pointer, nullptr if space not found */ + fil_space_t* space_load(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Check pending operations on a tablespace. + @param[in] space_id Tablespace ID + @param[in] operation File operation + @param[out] space tablespace instance in memory + @param[out] path tablespace path + @return DB_SUCCESS or DB_TABLESPACE_NOT_FOUND. */ + dberr_t space_check_pending_operations( + space_id_t space_id, + fil_operation_t operation, + fil_space_t** space, + char** path) const + MY_ATTRIBUTE((warn_unused_result)); + + /** Rename a single-table tablespace. + The tablespace must exist in the memory cache. + @param[in] space_id Tablespace ID + @param[in] old_path Old file name + @param[in] new_name New tablespace name in the schema/space + @param[in] new_path_in New file name, or nullptr if it + is located in the normal data directory + @return true if success */ + bool space_rename( + space_id_t space_id, + const char* old_path, + const char* new_name, + const char* new_path_in) + MY_ATTRIBUTE((warn_unused_result)); + + /** Deletes an IBD tablespace, either general or single-table. + The tablespace must be cached in the memory cache. This will delete the + datafile, fil_space_t & fil_node_t entries from the file_system_t cache. + @param[in] space_id Tablespace ID + @param[in] buf_remove Specify the action to take on the pages + for this table in the buffer pool. + @return DB_SUCCESS, DB_TABLESPCE_NOT_FOUND or DB_IO_ERROR */ + dberr_t space_delete( + space_id_t space_id, + buf_remove_t buf_remove) + MY_ATTRIBUTE((warn_unused_result)); + + /** Truncate the tablespace to needed size. + @param[in] space_id Tablespace ID to truncate + @param[in] size_in_pages Truncate size. + @return true if truncate was successful. */ + bool space_truncate(space_id_t space_id, page_no_t size_in_pages) + MY_ATTRIBUTE((warn_unused_result)); + + /** Create a space memory object and put it to the fil_system hash + table. The tablespace name is independent from the tablespace file-name. + Error messages are issued to the server log. + @param[in] name Tablespace name + @param[in] space_id Tablespace ID + @param[in] flags Tablespace flags + @param[in] purpose Tablespace purpose + @return pointer to created tablespace + @retval nullptr on failure (such as when the same tablespace exists) */ + fil_space_t* space_create( + const char* name, + space_id_t space_id, + ulint flags, + fil_type_t purpose) + MY_ATTRIBUTE((warn_unused_result)); + + /** Returns true if a matching tablespace exists in the InnoDB + tablespace memory cache. + @param[in] space_id Tablespace ID + @param[in] name Tablespace name used in space_create(). + @param[in] print_err Print detailed error information to the + error log if a matching tablespace is + not found from memory. + @param[in] adjust_space Whether to adjust space id on mismatch + @param[in] heap Heap memory + @param[in] table_id table id + @return true if a matching tablespace exists in the memory cache */ + bool space_check_exists( + space_id_t space_id, + const char* name, + bool print_err, + bool adjust_space, + mem_heap_t* heap, + table_id_t table_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Read or write log file data synchronously. + @param[in] type IO context + @param[in] page_id page id + @param[in] page_size page size + @param[in] byte_offset remainder of offset in bytes; in AIO + this must be divisible by the OS block + size + @param[in] len how many bytes to read or write; this + must not cross a file boundary; in AIO + this must be a block size multiple + @param[in,out] buf buffer where to store read data or + from where to write + @return error code + @retval DB_SUCCESS on success */ + dberr_t do_redo_io( + const IORequest& type, + const page_id_t& page_id, + const page_size_t& page_size, + ulint byte_offset, + ulint len, + void* buf) + MY_ATTRIBUTE((warn_unused_result)); + + /** Read or write data. This operation could be asynchronous (aio). + @param[in] type IO context + @param[in] sync whether synchronous aio is desired + @param[in] page_id page id + @param[in] page_size page size + @param[in] byte_offset remainder of offset in bytes; in AIO + this must be divisible by the OS + block size + @param[in] len how many bytes to read or write; + this must not cross a file boundary; + in AIO this must be a block size + multiple + @param[in,out] buf buffer where to store read data + or from where to write; in AIO + this must be appropriately aligned + @param[in] message message for AIO handler if !sync, + else ignored + @return error code + @retval DB_SUCCESS on success + @retval DB_TABLESPACE_DELETED if the tablespace does not exist */ + dberr_t do_io( + const IORequest& type, + bool sync, + const page_id_t& page_id, + const page_size_t& page_size, + ulint byte_offset, + ulint len, + void* buf, + void* message) + MY_ATTRIBUTE((warn_unused_result)); + + /** Iterate through all persistent tablespace files + (FIL_TYPE_TABLESPACE) returning the nodes via callback function cbk. + @param[in] include_log include log files, if true + @param[in] f Callback + @return any error returned by the callback function. */ + dberr_t iterate(bool include_log, Fil_iterator::Function& f) + MY_ATTRIBUTE((warn_unused_result)); + + /** Open an ibd tablespace and add it to the InnoDB data structures. + This is similar to fil_ibd_open() except that it is used while + processing the redo and DDL log, so the data dictionary is not + available and very little validation is done. The tablespace name + is extracted from the dbname/tablename.ibd portion of the filename, + which assumes that the file is a file-per-table tablespace. Any name + will do for now. General tablespace names will be read from the + dictionary after it has been recovered. The tablespace flags are read + at this time from the first page of the file in validate_for_recovery(). + @param[in] space_id tablespace ID + @param[in] path path/to/databasename/tablename.ibd + @param[out] space the tablespace, or nullptr on error + @return status of the operation */ + fil_load_status ibd_open_for_recovery( + space_id_t space_id, + const std::string& path, + fil_space_t*& space) + MY_ATTRIBUTE((warn_unused_result)); + + /** Attach a file to a tablespace + @param[in] name file name of a file that is not open + @param[in] size file size in entire database blocks + @param[in,out] space tablespace from fil_space_create() + @param[in] is_raw true if this is a raw device + or partition + @param[in] punch_hole true if supported for this file + @param[in] atomic_write true if the file has atomic write + enabled + @param[in] max_pages maximum number of pages in file + @return pointer to the file name + @retval nullptr if error */ + fil_node_t* create_node( + const char* name, + page_no_t size, + fil_space_t* space, + bool is_raw, + bool punch_hole, + bool atomic_write, + page_no_t max_pages = PAGE_NO_MAX) + MY_ATTRIBUTE((warn_unused_result)); - /* File has gone walkies, this state is only set during - recovery. */ +#ifdef UNIV_DEBUG + /** Validate a shard. */ + void validate() const; +#endif /* UNIV_DEBUG */ - it->m_state = state; +#ifdef UNIV_HOTBACKUP + /** Extends all tablespaces to the size stored in the space header. + During the mysqlbackup --apply-log phase we extended the spaces + on-demand so that log records could be applied, but that may have + left spaces still too small compared to the size stored in the space + header. */ + void meb_extend_tablespaces_to_stored_len(); +#endif /* UNIV_HOTBACKUP */ - } else { + /** Free a tablespace object on which fil_space_detach() was invoked. + There must not be any pending i/o's or flushes on the files. + @param[in,out] space tablespace */ + static void space_free_low(fil_space_t*& space); + + /** Wait for an empty slot to reserve for opening a file. + @return true on success. */ + static bool reserve_open_slot(size_t shard_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Release the slot reserved for opening a file. + @param[in] shard_id ID of shard relasing the slot */ + static void release_open_slot(size_t shard_id); + + /** We are going to do a rename file and want to stop new I/O + for a while. + @param[in,out] space Tablespace for which we want to + wait for IO to stop */ + static void wait_for_io_to_stop(fil_space_t* space); + +private: + + /** We keep log files and system tablespace files always open; this is + important in preventing deadlocks in this module, as a page read + completion often performs another read from the insert buffer. The + insert buffer is in tablespace TRX_SYS_SPACE, and we cannot end up + waiting in this function. + @param[in] space_id Tablespace ID to look up + @return tablespace instance */ + fil_space_t* get_reserved_space(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Prepare for truncating a single-table tablespace. + 1) Check pending operations on a tablespace; + 2) Remove all insert buffer entries for the tablespace; + @param[in] space_id Tablespace ID + @return DB_SUCCESS or error */ + dberr_t space_prepare_for_truncate(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Note that a write IO has completed. + @param[in,out] file File on which a write was + completed */ + void write_completed(fil_node_t* file); + + /** If the tablespace is not on the unflushed list, add it. + @param[in,out] space Tablespace to add */ + void add_to_unflushed_list(fil_space_t* space); + + /** Check for pending operations. + @param[in] space tablespace + @param[in] count number of attempts so far + @return 0 if no pending operations else count + 1. */ + ulint space_check_pending_operations( + fil_space_t* space, + ulint count) const + MY_ATTRIBUTE((warn_unused_result)); + + /** Check for pending IO. + @param[in] operation File operation + @param[in,out] space Tablespace to check + @param[out] file File in space list + @param[in] count number of attempts so far + @return 0 if no pending else count + 1. */ + ulint check_pending_io( + fil_operation_t operation, + fil_space_t* space, + fil_node_t** file, + ulint count) const + MY_ATTRIBUTE((warn_unused_result)); + + /** Flushes to disk possible writes cached by the OS. */ + void redo_space_flush(); + + /** First we open the file in the normal mode, no async I/O here, for + simplicity. Then do some checks, and close the file again. NOTE that we + could not use the simple file read function os_file_read() in Windows + to read from a file opened for async I/O! + @param[in,out] file Get the size of this file + @param[in] read_only_mode true if read only mode set + @return DB_SUCCESS or error */ + dberr_t get_file_size( + fil_node_t* file, + bool read_only_mode) + MY_ATTRIBUTE((warn_unused_result)); + + /** Get the AIO mode. + @param[in] req_type IO request type + @param[in] sync true if Synchronous IO + return the AIO mode */ + static AIO_mode get_AIO_mode(const IORequest& req_type, bool sync) + MY_ATTRIBUTE((warn_unused_result)); + + /** Get the file name for IO and the local offset within that file. + @param[in] req_type IO context + @param[in,out] space Tablespace for IO + @param[in,out] page_no The relative page number in the file + @param[out] file File node + @return DB_SUCCESS or error code */ + static dberr_t get_file_for_io( + const IORequest& req_type, + fil_space_t* space, + page_no_t* page_no, + fil_node_t*& file) + MY_ATTRIBUTE((warn_unused_result)); +private: + + /** Fil_shard ID */ + + const size_t m_id; + + /** Tablespace instances hashed on the space id */ - it->m_state = state; + Spaces m_spaces; - it->m_open_lsn = lsn; - it->m_closed_lsn = 0; - } -} + /** Tablespace instances hashed on the space name */ -/** Remove all nodes that were closed before the LSN. */ -void -Fil_Open::Nodes::purge() -{ - m_files.erase( - std::remove_if( - m_files.begin(), - m_files.end(), - [=](Paths::value_type& file) -> bool - { - return(file.m_state == DELETED - || file.m_state == MISSING); - }), - m_files.end()); -} + Names m_names; -/** Remove all nodes. */ -void -Fil_Open::Nodes::clear() -{ - m_files.clear(); -} + /** Base node for the LRU list of the most recently used open + files with no pending I/O's; if we start an I/O on the file, + we first remove it from this list, and return it to the start + of the list when the I/O ends; log files and the system + tablespace are not put to this list: they are opened after + the startup, and kept open until shutdown */ -/** Note that a tablespace file has been closed. -@param[in] path Filename to close */ -void -Fil_Open::Nodes::close(const std::string& path) -{ - auto it = lookup(path); + File_list m_LRU; - if (it != m_files.end()) { - it->m_closed_lsn = 0; - it->m_state = CLOSED; - } -} + /** Base node for the list of those tablespaces whose files + contain unflushed writes; those spaces have at least one file + where modification_counter > flush_counter */ -/** Mark all files as DELETED. */ -void -Fil_Open::Nodes::deleted() -{ - for (auto& file : m_files) { - file.m_state = DELETED; - } -} + Space_list m_unflushed_spaces; -/** Rename a file. -@param[in] from Filename to rename -@param[in] to Filename to rename to -@retval true on success */ -bool -Fil_Open::Nodes::rename(const std::string& from, const std::string& to) -{ - auto it = lookup(from); + /** When we write to a file we increment this by one */ - if (it == m_files.end()) { - return(false); - } + int64_t m_modification_counter; - it->m_name = to; - it->m_state = OPEN; + /** Number of files currently open */ - return(true); -} + static std::atomic_size_t s_n_open; -/** Create tablespaces.open.* files */ -void -Fil_Open::create_open_files() -{ + /** ID of shard that has reserved the open slot. */ - for (uint32_t i = 0; i < MAX_TABLESPACE_OPEN_FILES; i++) { + static std::atomic_size_t s_open_slot; - if (m_handles.at(i).handle.m_file != OS_FILE_CLOSED) { - continue; - } + // Disable copying + Fil_shard(Fil_shard&&) = delete; + Fil_shard(const Fil_shard&) = delete; + Fil_shard& operator=(const Fil_shard&) = delete; - const std::string& filename = PATHS[i]; + friend class Fil_system; +}; - std::string path_filename(srv_log_group_home_dir); +/** The tablespace memory cache; also the totality of logs (the log +data space) is stored here; below we talk about tablespaces, but also +the ib_logfiles form a 'space' and it is handled here */ +class Fil_system { +public: - /* Append PATH separator */ - if ((path_filename.length() != 0) - && (*path_filename.rbegin() != OS_PATH_SEPARATOR)) { - path_filename += OS_PATH_SEPARATOR; - } + using Fil_shards = std::vector; - path_filename.append(filename); + /** Constructor. + @param[in] n_shards Number of shards to create + @param[in] max_open Maximum number of open files */ + Fil_system(size_t n_shards, size_t max_open); - const char* path = path_filename.c_str(); - bool success; + /** Destructor */ + ~Fil_system(); - pfs_os_file_t file; + /** Fetch the file names opened for a space_id during recovery. + @param[in] space_id Tablespace ID to lookup + @return pair of top level directory scanned and names that map + to space_id or nullptr if not found for names */ + Tablespace_dirs::Result get_scanned_files(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)) + { + return(m_dirs.find(space_id)); + } - file = os_file_create(innodb_tablespace_open_file_key, path, - OS_FILE_CREATE | OS_FILE_ON_ERROR_NO_EXIT, - OS_FILE_NORMAL, - OS_BUFFERED_FILE, - srv_read_only_mode, - &success); + /** Fetch the file name opened for a space_id during recovery + from the file map. + @param[in] space_id Undo tablespace ID + @return Full path to the file name that was opened, empty string + if space ID not found. */ + std::string find(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)) + { + auto result = get_scanned_files(space_id); - if (!success) { - ulint error = os_file_get_last_error(false); - bool success_open; - - if (error == OS_FILE_ALREADY_EXISTS) { - /* Open the file now. */ - file = os_file_create( - innodb_tablespace_open_file_key, path, - OS_FILE_OPEN | OS_FILE_ON_ERROR_NO_EXIT, - OS_FILE_NORMAL, - OS_BUFFERED_FILE, - srv_read_only_mode, - &success_open); - - if (!success_open) { - /* print msg to stderr */ - os_file_get_last_error(true); - ib::fatal() << "Failed to open: " << path; - } - } else { - os_file_get_last_error(true); - ib::fatal() << "Failed to create: " << path; - } + if (result.second != nullptr) { + return(result.first + result.second->front()); } - ut_ad(file.m_file != OS_FILE_CLOSED); - ut_ad(path_filename.length() >= PATHS[0].length()); - - m_handles.at(i).handle = file; - m_handles.at(i).path.assign(path_filename); + return(""); } -} - -/** Rename a file for a given tablespace ID. -@param[in] space_id Tablespace ID -@param[in] from Filename to rename -@param[in] to Filename to rename to -@retval true on success */ -bool -Fil_Open::rename( - space_id_t space_id, - const std::string& from, - const std::string& to) -{ - auto it = m_spaces.find(space_id); - if (it == m_spaces.end()) { + /** Erase a tablespace ID and its mapping from the scanned files. + @param[in] space_id Tablespace ID to erase + @return true if successful */ + bool erase(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)) + { + return(m_dirs.erase(space_id)); + } + + /** Get the top level directory where this filename was found. + @param[in] path Path to look for. + @return the top level directory under which this file was found. */ + const std::string& get_root(const std::string& path) const + MY_ATTRIBUTE((warn_unused_result)); + + /** Update the DD if any files were moved to a new location. + Free the Tablespace_files instance. + @param[in] read_only_mode true if InnoDB is started in + read only mode. + @return DB_SUCCESS if all OK */ + dberr_t prepare_open_for_business(bool read_only_mode) + MY_ATTRIBUTE((warn_unused_result)); + + /** Flush the redo log writes to disk, possibly cached by the OS. */ + void flush_file_redo(); + + /** Flush to disk the writes in file spaces of the given type + possibly cached by the OS. + @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG, + can be ORred */ + void flush_file_spaces(uint8_t purpose); + + /** Fetch the fil_space_t instance that maps to the name. + @param[in] name Tablespace name to lookup + @return tablespace instance or nullptr if not found. */ + fil_space_t* get_space_by_name(const char* name) + MY_ATTRIBUTE((warn_unused_result)) + { + for (auto shard : m_shards) { - return(false); - } + shard->mutex_acquire(); - return(it->second.rename(from, to)); -} + auto space = shard->get_space_by_name(name); -/** Remove all the tablespace file names that are not required for -recovery to work. */ -void -Fil_Open::purge() -{ - /** Remove all spaces that don't have open files attached. */ - for (auto it = m_spaces.begin(); it != m_spaces.end(); /* No op */) { + shard->mutex_release(); - it->second.purge(); + if (space != nullptr) { - if (it->second.empty()) { - it = m_spaces.erase(it); - } else { - ++it; + return(space); + } } + + return(nullptr); } -} -/** Remove all the tablespace file names. */ -void -Fil_Open::clear() -{ - for (auto& space : m_spaces) { + /** Check a space ID against the maximum known tablespace ID. + @param[in] space_id Tablespace ID to check + @return true if it is > than maximum known tablespace ID. */ + bool is_greater_than_max_id(space_id_t space_id) const + MY_ATTRIBUTE((warn_unused_result)) + { + ut_ad(mutex_owned_all()); - space.second.clear(); + return(space_id > m_max_assigned_id); } - m_spaces.clear(); -} + /** Update the maximum known tablespace ID. + @param[in] space Tablespace instance */ + void set_maximum_space_id(const fil_space_t* space) + { + ut_ad(mutex_owned_all()); -/** Delete a tablespace ID -@param[in] space_id Tablespace ID -@retval true on success */ -bool -Fil_Open::deleted(space_id_t space_id) -{ - auto it = m_spaces.find(space_id); + if (!m_space_id_reuse_warned) { - if (it == m_spaces.end()) { + m_space_id_reuse_warned = true; - return(false); + ib::warn() + << "Allocated tablespace ID " << space->id + << " for " << space->name << ", old maximum" + << " was " << m_max_assigned_id; + } + + m_max_assigned_id = space->id; } - it->second.deleted(); + /** Update the maximim known space ID if it's smaller than max_id. + @param[in] space_id Value to set if it's greater */ + void update_maximum_space_id(space_id_t space_id) + { + mutex_acquire_all(); - m_needs_flush = true; + if (is_greater_than_max_id(space_id)){ - return(true); -} + m_max_assigned_id = space_id; + } -/** The tablespace memory cache; also the totality of logs (the log -data space) is stored here; below we talk about tablespaces, but also -the ib_logfiles form a 'space' and it is handled here */ -struct fil_system_t { + mutex_release_all(); + } + + /** Assigns a new space id for a new single-table tablespace. This + works simply by incrementing the global counter. If 4 billion ids + is not enough, we may need to recycle ids. + @param[out] space_id Set this to the new tablespace ID + @return true if assigned, false if not */ + bool assign_new_space_id(space_id_t* space_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Tries to close a file in all the LRU lists. + The caller must hold the mutex. + @param[in] print_info if true, prints information why it + cannot close a file + @return true if success, false if should retry later */ + bool close_file_in_all_LRU(bool print_info) + MY_ATTRIBUTE((warn_unused_result)); + + /** Opens all log files and system tablespace data files in + all shards. */ + void open_all_system_tablespaces(); + + /** Close all open files in a shard + @param[in,out] shard Close files of this shard */ + void close_files_in_a_shard(Fil_shard* shard); + + /** Close all open files. */ + void close_all_files(); + + /** Close all the log files in all shards. + @param[in] free_all If set then free all instances */ + void close_all_log_files(bool free_all); + + /** Iterate through all persistent tablespace files + (FIL_TYPE_TABLESPACE) returning the nodes via callback function cbk. + @param[in] include_log Include log files, if true + @param[in] f Callback + @return any error returned by the callback function. */ + dberr_t iterate(bool include_log, Fil_iterator::Function& f) + MY_ATTRIBUTE((warn_unused_result)); + + /** Rotate the tablespace keys by new master key. + @param[in,out] shard Rotate the keys in this shard + @return true if the re-encrypt succeeds */ + bool encryption_rotate_in_a_shard(Fil_shard* shard); + + /** Rotate the tablespace keys by new master key. + @return true if the re-encrypt succeeds */ + bool encryption_rotate_all() + MY_ATTRIBUTE((warn_unused_result)); + + /** Detach a space object from the tablespace memory cache. + Closes the tablespace files but does not delete them. + There must not be any pending I/O's or flushes on the files. + @param[in,out] space tablespace */ + void space_detach(fil_space_t* space); + + /** @return the maximum assigned ID so far */ + space_id_t get_max_space_id() const + { + return(m_max_assigned_id); + } + + /** Lookup the tablespace ID. + @param[in] space_id Tablespace ID to lookup + @return true if the space ID is known. */ + bool lookup_for_recovery(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** Open a tablespace that has a redo log record to apply. + @param[in] space_id Tablespace ID + @return true if the open was successful */ + bool open_for_recovery(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + + /** This function should be called after recovery has completed. + Check for tablespace files for which we did not see any + MLOG_FILE_DELETE or MLOG_FILE_RENAME record. These could not + be recovered. + @return true if there were some filenames missing for which we had to + ignore redo log records during the apply phase */ + bool check_missing_tablespaces() + MY_ATTRIBUTE((warn_unused_result)); + + /** Note that a file has been relocated. + @param[in] object_id Server DD tablespace ID + @param[in] space_id InnoDB tablespace ID + @param[in] space_name Tablespace name + @param[in] old_path Path to the old location + @param[in] new_path Path scanned from disk */ + void moved( + dd::Object_id object_id, + space_id_t space_id, + const char* space_name, + const std::string& old_path, + const std::string& new_path) + { + auto tuple = std::make_tuple( + object_id, space_id, space_name, old_path, new_path); + + m_moved.push_back(tuple); + } + + /** Check if a path is known to InnoDB. + @param[in] path Path to check + @return true if path is known to InnoDB */ + bool check_path(const std::string& path) const + { + const auto& dir = m_dirs.contains(path); + + return(dir != Fil_path::null()); + } + + /** Get the list of directories that InnoDB knows about. + @return the list of directories 'dir1;dir2;....;dirN' */ + std::string get_dirs() const + { + return(m_dirs.get_dirs()); + } + + /** Determines if a file belongs to the least-recently-used list. + @param[in] space Tablespace to check + @return true if the file belongs to fil_system->m_LRU mutex. */ + static bool space_belongs_in_LRU(const fil_space_t* space) + MY_ATTRIBUTE((warn_unused_result)); + + /** Scan the directories to build the tablespace ID to file name + mapping table. */ + dberr_t scan(const std::string& directories) + { + return(m_dirs.scan(directories)); + } + + /** Get the tablespace ID from an .ibd and/or an undo tablespace. + If the ID is == 0 on the first page then check for at least + MAX_PAGES_TO_CHECK pages with the same tablespace ID. Do a Light + weight check before trying with DataFile::find_space_id(). + @param[in] filename File name to check + @return ULINT32_UNDEFINED if not found, otherwise the space ID */ + static space_id_t get_tablespace_id( + const std::string& filename) + MY_ATTRIBUTE((warn_unused_result)); + + /** Fil_shard by space ID. + @param[in] space_id Tablespace ID + @return reference to the shard */ + Fil_shard* shard_by_id(space_id_t space_id) const + MY_ATTRIBUTE((warn_unused_result)) + { #ifndef UNIV_HOTBACKUP - ib_mutex_t mutex; /*!< The mutex protecting the cache */ + if (space_id == dict_sys_t::s_log_space_first_id) { + + return(m_shards[REDO_SHARD]); + + } else if (fsp_is_undo_tablespace(space_id)) { + + const size_t limit = space_id % UNDO_SHARDS; + + return(m_shards[UNDO_SHARDS_START + limit]); + } + + ut_ad(m_shards.size() == MAX_SHARDS); + + return(m_shards[space_id % UNDO_SHARDS_START]); #else /* !UNIV_HOTBACKUP */ - meb::Mutex mutex; + ut_ad(m_shards.size() == 1); + + return(m_shards[0]); #endif /* !UNIV_HOTBACKUP */ + } + + /** Acquire all the mutexes. */ + void mutex_acquire_all() const + { + +#ifdef UNIV_HOTBACKUP + ut_ad(m_shards.size() == 1); +#endif /* UNIV_HOTBACKUP */ + + for (auto shard : m_shards) { + shard->mutex_acquire(); + } + } + + /** Release all the mutexes. */ + void mutex_release_all() const + { + +#ifdef UNIV_HOTBACKUP + ut_ad(m_shards.size() == 1); +#endif /* UNIV_HOTBACKUP */ + + for (auto shard : m_shards) { + shard->mutex_release(); + } + } + +#ifdef UNIV_DEBUG + + /** Checks the consistency of the tablespace cache. + @return true if ok */ + bool validate() const + MY_ATTRIBUTE((warn_unused_result)); + + /** Check if all mutexes are owned + @return true if all owned. */ + bool mutex_owned_all() const + MY_ATTRIBUTE((warn_unused_result)) + { + +#ifdef UNIV_HOTBACKUP + ut_ad(m_shards.size() == 1); +#endif /* UNIV_HOTBACKUP */ + + for (const auto shard: m_shards) { + ut_ad(shard->mutex_owned()); + } - Spaces spaces; /*!< Tablespace instances hashed on - the space id */ - - Names names; /*!< Tablespace instances hashed on - the space name */ - - /** Track the mapping from tablespace ID to file name on disk. */ - Fil_Open m_open; - - UT_LIST_BASE_NODE_T(fil_node_t) LRU; - /*!< base node for the LRU list of the - most recently used open files with no - pending i/o's; if we start an i/o on - the file, we first remove it from this - list, and return it to the start of - the list when the i/o ends; - log files and the system tablespace are - not put to this list: they are opened - after the startup, and kept open until - shutdown */ - UT_LIST_BASE_NODE_T(fil_space_t) unflushed_spaces; - /*!< base node for the list of those - tablespaces whose files contain - unflushed writes; those spaces have - at least one file node where - modification_counter > flush_counter */ - ulint n_open; /*!< number of files currently open */ - ulint max_n_open; /*!< n_open is not allowed to exceed - this */ - int64_t modification_counter;/*!< when we write to a file we - increment this by one */ - space_id_t max_assigned_id;/*!< maximum space id in the existing - tables, or assigned during the time - mysqld has been up; at an InnoDB - startup we scan the data dictionary - and set here the maximum of the - space id's of the tables there */ - UT_LIST_BASE_NODE_T(fil_space_t) space_list; - /*!< list of all file spaces */ - bool space_id_reuse_warned; - /* !< true if fil_space_create() - has issued a warning about - potential space_id reuse */ + return(true); + } + +#endif /* UNIV_DEBUG */ + + /** Rename a tablespace by its name only + @param[in] old_name old tablespace name + @param[in] new_name new tablespace name + @return DB_SUCCESS on success */ + dberr_t rename_tablespace_name( + const char* old_name, + const char* new_name) + MY_ATTRIBUTE((warn_unused_result)); + + /** Free the data structures required for recovery. */ + void free_scanned_files() + { + m_dirs.clear(); + } + +#ifdef UNIV_HOTBACKUP + /** Extends all tablespaces to the size stored in the space header. + During the mysqlbackup --apply-log phase we extended the spaces + on-demand so that log records could be applied, but that may have + left spaces still too small compared to the size stored in the space + header. */ + void meb_extend_tablespaces_to_stored_len() + { + ut_ad(m_shards.size() == 1); + + /* We use a single shard for MEB. */ + auto shard = shard_by_id(SPACE_UNKNOWN); + + shard->mutex_acquire(); + + shard->meb_extend_tablespaces_to_stored_len(); + + shard->mutex_release(); + + } + + /** Process a file name passed as an input + Wrapper around meb_name_process() + @param[in,out] name absolute path of tablespace file + @param[in] space_id The tablespace ID + @param[in] deleted true if MLOG_FILE_DELETE */ + void meb_name_process( + char* name, + space_id_t space_id, + bool deleted); + +#endif /* UNIV_HOTBACKUP */ + +private: + /** Open an ibd tablespace and add it to the InnoDB data structures. + This is similar to fil_ibd_open() except that it is used while + processing the redo log, so the data dictionary is not available + and very little validation is done. The tablespace name is extracted + from the dbname/tablename.ibd portion of the filename, which assumes + that the file is a file-per-table tablespace. Any name will do for + now. General tablespace names will be read from the dictionary after + it has been recovered. The tablespace flags are read at this time + from the first page of the file in validate_for_recovery(). + @param[in] space_id tablespace ID + @param[in] path path/to/databasename/tablename.ibd + @param[out] space the tablespace, or nullptr on error + @return status of the operation */ + fil_load_status ibd_open_for_recovery( + space_id_t space_id, + const std::string& path, + fil_space_t*& space) + MY_ATTRIBUTE((warn_unused_result)); +private: + /** Fil_shards managed */ + Fil_shards m_shards; + + /** n_open is not allowed to exceed this */ + const size_t m_max_n_open; + + /** Maximum space id in the existing tables, or assigned during + the time mysqld has been up; at an InnoDB startup we scan the + data dictionary and set here the maximum of the space id's of + the tables there */ + space_id_t m_max_assigned_id; + + /** true if fil_space_create() has issued a warning about + potential space_id reuse */ + bool m_space_id_reuse_warned; + + /** List of tablespaces that have been relocated. We need to + update the DD when it is safe to do so. */ + dd_fil::Tablespaces m_moved; + + /** Tablespace directories scanned at startup */ + Tablespace_dirs m_dirs; + + // Disable copying + Fil_system(Fil_system&&) = delete; + Fil_system(const Fil_system&) = delete; + Fil_system& operator=(const Fil_system&) = delete; + + friend class Fil_shard; }; -/** The tablespace memory cache. This variable is NULL before the module is +/** The tablespace memory cache. This variable is nullptr before the module is initialized. */ -static fil_system_t* fil_system = NULL; +static Fil_system* fil_system = nullptr; -/** Determine if user has explicitly disabled fsync(). */ -#ifndef _WIN32 -# define fil_buffering_disabled(s) \ - ((s)->purpose == FIL_TYPE_TABLESPACE \ - && srv_unix_file_flush_method \ - == SRV_UNIX_O_DIRECT_NO_FSYNC) -#else /* _WIN32 */ -# define fil_buffering_disabled(s) (0) -#endif /* __WIN32 */ +/** Total number of open files. */ +std::atomic_size_t Fil_shard::s_n_open; + +/** Slot reserved for opening a file. */ +std::atomic_size_t Fil_shard::s_open_slot; + +#ifdef UNIV_HOTBACKUP +static ulint srv_data_read; +static ulint srv_data_written; +#endif /* UNIV_HOTBACKUP */ + +/** Replay a file rename operation if possible. +@param[in] page_id Space ID and first page number in the file +@param[in] old_name old file name +@param[in] new_name new file name +@return whether the operation was successfully applied (the name did not exist, +or new_name did not exist and name was successfully renamed to new_name) */ +static +bool +fil_op_replay_rename( + const page_id_t& page_id, + const std::string& old_name, + const std::string& new_name) + MY_ATTRIBUTE((warn_unused_result)); #ifdef UNIV_DEBUG /** Try fil_validate() every this many times */ -# define FIL_VALIDATE_SKIP 17 - -/******************************************************************//** -Checks the consistency of the tablespace cache some of the time. +static const size_t FIL_VALIDATE_SKIP = 17; +/** Checks the consistency of the tablespace cache some of the time. @return true if ok or the check was skipped */ static bool -fil_validate_skip(void) -/*===================*/ +fil_validate_skip() { /** The fil_validate() call skip counter. Use a signed type because of the race condition below. */ @@ -961,267 +1763,505 @@ fil_validate_skip(void) fil_validate_count = FIL_VALIDATE_SKIP; return(fil_validate()); } -#endif /* UNIV_DEBUG */ -/********************************************************************//** -Determines if a file node belongs to the least-recently-used list. -@return true if the file belongs to fil_system->LRU mutex. */ -UNIV_INLINE -bool -fil_space_belongs_in_lru( -/*=====================*/ - const fil_space_t* space) /*!< in: file space */ +/** Validate a shard */ +void +Fil_shard::validate() const { - switch (space->purpose) { - case FIL_TYPE_TEMPORARY: - case FIL_TYPE_LOG: - return(false); - case FIL_TYPE_TABLESPACE: - return(fsp_is_ibd_tablespace(space->id)); - case FIL_TYPE_IMPORT: - return(true); - } - - ut_ad(0); - return(false); -} + mutex_acquire(); -/** NOTE: you must call fil_mutex_enter_and_prepare_for_io() first! + size_t n_open = 0; -Prepares a file node for i/o. Opens the file if it is closed. Updates the -pending i/o's field in the node and the system appropriately. Takes the node -off the LRU list if it is in the LRU list. The caller must hold the fil_sys -mutex. -@param[in] node File node -@param[in] system Tablespace memory cache -@param[in] space Tablespace instance -@param[in] extend true if file is being extended -@return false if the file can't be opened, otherwise true */ -static -bool -fil_node_prepare_for_io( - fil_node_t* node, - fil_system_t* system, - fil_space_t* space, - bool extend); - -/** -Updates the data structures when an i/o operation finishes. Updates the -pending i/o's field in the node appropriately. -@param[in,out] node file node -@param[in,out] system tablespace instance -@param[in] type IO context */ -static -void -fil_node_complete_io( - fil_node_t* node, - fil_system_t* system, - const IORequest& type); + for (auto elem : m_spaces) { -/** Reads data from a space to a buffer. Remember that the possible incomplete -blocks at the end of file are ignored: they are not taken into account when -calculating the byte offset within a space. -@param[in] page_id page id -@param[in] page_size page size -@param[in] byte_offset remainder of offset in bytes; in aio this -must be divisible by the OS block size -@param[in] len how many bytes to read; this must not cross a -file boundary; in aio this must be a block size multiple -@param[in,out] buf buffer where to store data read; in aio this -must be appropriately aligned -@return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do -i/o on a tablespace which does not exist */ -UNIV_INLINE -dberr_t -fil_read( - const page_id_t& page_id, - const page_size_t& page_size, - ulint byte_offset, - ulint len, - void* buf) -{ - return(fil_io(IORequestRead, true, page_id, page_size, - byte_offset, len, buf, NULL)); -} + page_no_t size = 0; + auto space = elem.second; -/** Writes data to a space from a buffer. Remember that the possible incomplete -blocks at the end of file are ignored: they are not taken into account when -calculating the byte offset within a space. -@param[in] page_id page id -@param[in] page_size page size -@param[in] byte_offset remainder of offset in bytes; in aio this -must be divisible by the OS block size -@param[in] len how many bytes to write; this must not cross -a file boundary; in aio this must be a block size multiple -@param[in] buf buffer from which to write; in aio this must -be appropriately aligned -@return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do -i/o on a tablespace which does not exist */ -UNIV_INLINE -dberr_t -fil_write( - const page_id_t& page_id, - const page_size_t& page_size, - ulint byte_offset, - ulint len, - void* buf) -{ - ut_ad(!srv_read_only_mode); + for (const auto& file : space->files) { - return(fil_io(IORequestWrite, true, page_id, page_size, - byte_offset, len, buf, NULL)); -} + ut_a(file.is_open || !file.n_pending); -/** Returns the table space by a given id, NULL if not found. -@param[in] id Tablespace ID -@return the instance of the tablespace meta data */ -static -fil_space_t* -fil_space_get_by_id(space_id_t id) -{ - ut_ad(mutex_own(&fil_system->mutex)); + if (file.is_open) { + ++n_open; + } - auto it = fil_system->spaces.find(id); + size += file.size; + } - if (it == fil_system->spaces.end()) { - return(nullptr); + ut_a(space->size == size); } - ut_ad(it->second->magic_n == FIL_SPACE_MAGIC_N); + UT_LIST_CHECK(m_LRU); + + for (auto file = UT_LIST_GET_FIRST(m_LRU); + file != nullptr; + file = UT_LIST_GET_NEXT(LRU, file)) { - return(it->second); + ut_a(file->is_open); + ut_a(file->n_pending == 0); + ut_a(fil_system->space_belongs_in_LRU(file->space)); + } + + mutex_release(); } -/** Returns the table space by a given name, NULL if not found. -@param[in] name Tablespace name to search for. -@return nullptr if not found */ -static -fil_space_t* -fil_space_get_by_name(const char* name) +/** Checks the consistency of the tablespace cache. +@return true if ok */ +bool +Fil_system::validate() const { - ut_ad(mutex_own(&fil_system->mutex)); + for (const auto shard : m_shards) { - auto it = fil_system->names.find(name); - - if (it == fil_system->names.end()) { - return(nullptr); + shard->validate(); } - ut_ad(it->second->magic_n == FIL_SPACE_MAGIC_N); - - return(it->second); + return(true); } - -/** Look up a tablespace. -The caller should hold an InnoDB table lock or a MDL that prevents -the tablespace from being dropped during the operation, -or the caller should be in single-threaded crash recovery mode -(no user connections that could drop tablespaces). -If this is not the case, fil_space_acquire() and fil_space_release() -should be used instead. -@param[in] id tablespace ID -@return tablespace, or NULL if not found */ -fil_space_t* -fil_space_get(space_id_t id) +/** Checks the consistency of the tablespace cache. +@return true if ok */ +bool +fil_validate() { - mutex_enter(&fil_system->mutex); - fil_space_t* space = fil_space_get_by_id(id); - mutex_exit(&fil_system->mutex); - - return(space); + return(fil_system->validate()); } +#endif /* UNIV_DEBUG */ -#ifndef UNIV_HOTBACKUP -/** Returns the latch of a file space. -@param[in] id space id -@param[out] flags tablespace flags -@return latch protecting storage allocation */ -rw_lock_t* -fil_space_get_latch( - space_id_t id, - ulint* flags) +/** Constructor. +@param[in] n_shards Number of shards to create +@param[in] max_open Maximum number of open files */ +Fil_system::Fil_system(size_t n_shards, size_t max_open) + : + m_shards(), + m_max_n_open(max_open), + m_max_assigned_id(), + m_space_id_reuse_warned() { - fil_space_t* space; - - ut_ad(fil_system); - - mutex_enter(&fil_system->mutex); + ut_ad(Fil_shard::s_open_slot == 0); + Fil_shard::s_open_slot = EMPTY_OPEN_SLOT; - space = fil_space_get_by_id(id); + for (size_t i = 0; i < n_shards; ++i) { - ut_a(space); + auto shard = UT_NEW_NOKEY(Fil_shard(i)); - if (flags) { - *flags = space->flags; + m_shards.push_back(shard); } - - mutex_exit(&fil_system->mutex); - - return(&(space->latch)); } -#ifdef UNIV_DEBUG -/** Gets the type of a file space. -@param[in] id tablespace identifier -@return file type */ -fil_type_t -fil_space_get_type(space_id_t id) +/** Destructor */ +Fil_system::~Fil_system() { - fil_space_t* space; - - ut_ad(fil_system); + ut_ad(Fil_shard::s_open_slot == EMPTY_OPEN_SLOT); - mutex_enter(&fil_system->mutex); + Fil_shard::s_open_slot = 0; - space = fil_space_get_by_id(id); + for (auto shard : m_shards) { - ut_a(space); - - mutex_exit(&fil_system->mutex); + UT_DELETE(shard); + } - return(space->purpose); + m_shards.clear(); } -#endif /* UNIV_DEBUG */ -/** Note that a tablespace has been imported. -It is initially marked as FIL_TYPE_IMPORT so that no logging is -done during the import process when the space ID is stamped to each page. -Now we change it to FIL_SPACE_TABLESPACE to start redo and undo logging. -NOTE: temporary tablespaces are never imported. -@param[in] id tablespace identifier */ -void -fil_space_set_imported(space_id_t id) +/** Determines if a file belongs to the least-recently-used list. +@param[in] space Tablespace to check +@return true if the file belongs to m_LRU. */ +bool +Fil_system::space_belongs_in_LRU(const fil_space_t* space) { - ut_ad(fil_system != NULL); - - mutex_enter(&fil_system->mutex); + switch (space->purpose) { + case FIL_TYPE_TEMPORARY: + case FIL_TYPE_LOG: + return(false); + + case FIL_TYPE_TABLESPACE: + return(fsp_is_ibd_tablespace(space->id)); + + case FIL_TYPE_IMPORT: + return(true); + } + + ut_ad(0); + return(false); +} + +/** Constructor +@param[in] shard_id Shard ID */ +Fil_shard::Fil_shard(size_t shard_id) + : + m_id(shard_id), + m_spaces(), + m_names(), + m_modification_counter() +{ + mutex_create(LATCH_ID_FIL_SHARD, &m_mutex); + + UT_LIST_INIT(m_LRU, &fil_node_t::LRU); + + UT_LIST_INIT(m_unflushed_spaces, &fil_space_t::unflushed_spaces); +} + +/** Wait for an empty slot to reserve for opening a file. +@return true on success. */ +bool +Fil_shard::reserve_open_slot(size_t shard_id) +{ + size_t expected = EMPTY_OPEN_SLOT; + + return(s_open_slot.compare_exchange_weak(expected, shard_id)); +} + + +/** Release the slot reserved for opening a file. +@param[in] shard_id ID of shard relasing the slot */ +void +Fil_shard::release_open_slot(size_t shard_id) +{ + size_t expected = shard_id; + + bool success = s_open_slot.compare_exchange_weak( + expected, EMPTY_OPEN_SLOT); + + ut_a(success); +} + +/** Map the space ID and name to the tablespace instance. +@param[in] space Tablespace instance */ +void +Fil_shard::space_add(fil_space_t* space) +{ + ut_ad(mutex_owned()); + + { + auto it = m_spaces.insert( + Spaces::value_type(space->id, space)); + + ut_a(it.second); + } + + { + auto name = space->name; + + auto it = m_names.insert(Names::value_type(name, space)); + + ut_a(it.second); + } +} + +/** Add the file node to the LRU list if required. +@param[in,out] file File for the tablespace */ +void +Fil_shard::file_opened(fil_node_t* file) +{ + ut_ad(m_id == REDO_SHARD || mutex_owned()); + + if (Fil_system::space_belongs_in_LRU(file->space)) { + + /* Put the file to the LRU list */ + UT_LIST_ADD_FIRST(m_LRU, file); + } + + ++s_n_open; + + file->is_open = true; + + ++fil_n_file_opened; +} + +/** Remove the file node from the LRU list. +@param[in,out] file File for the tablespace */ +void +Fil_shard::remove_from_LRU(fil_node_t* file) +{ + ut_ad(mutex_owned()); + + if (Fil_system::space_belongs_in_LRU(file->space)) { + + ut_ad(mutex_owned()); + + ut_a(UT_LIST_GET_LEN(m_LRU) > 0); + + /* The file is in the LRU list, remove it */ + UT_LIST_REMOVE(m_LRU, file); + } +} + +/** Close a tablespace file based on tablespace ID. +@param[in] space_id Tablespace ID +@return false if space_id was not found. */ +bool +Fil_shard::close_file(space_id_t space_id) +{ + mutex_acquire(); + + auto space = get_space_by_id(space_id); + + if (space == nullptr) { + + mutex_release(); + + return(false); + } + + for (auto& file : space->files) { + + while (file.in_use > 0) { + + mutex_release(); + + os_thread_sleep(10000); + + mutex_acquire(); + } + + if (file.is_open) { + close_file(&file, false); + } + } + + mutex_release(); + + return(true); +} + +/** Remap the tablespace to the new name. +@param[in] space Tablespace instance, with old name. +@param[in] new_name New tablespace name */ +void +Fil_shard::update_space_name_map(fil_space_t* space, const char* new_name) +{ + ut_ad(mutex_owned()); + + ut_ad(m_spaces.find(space->id) != m_spaces.end()); + + m_names.erase(space->name); + + auto it = m_names.insert(Names::value_type(new_name, space)); + + ut_a(it.second); +} + +/** Check if the basename of a filepath is an undo tablespace name +@param[in] name Tablespace name +@return true if it is an undo tablespace name */ +bool +Fil_path::is_undo_tablespace_name(const std::string& name) +{ + if (name.empty()) { + return(false); + } + + std::string basename(name); + + auto pos = basename.find_last_of(SEPARATOR); + + if (pos != std::string::npos) { + basename.erase(basename.begin(), basename.begin() + pos); + } + + if (basename.length() < sizeof("u.ibu") - 1) { + return(false); + } + + const auto end = basename.end(); + + /* We had two formats: undo_000 and undo000. Check for both. */ + size_t u = (*(end - 4) == '_') ? 1 : 0; + + return(basename.length() >= sizeof("undo000") - 1 + u + && *(end - 7 - u) == 'u' + && *(end - 6 - u) == 'n' + && *(end - 5 - u) == 'd' + && *(end - 4 - u) == 'o' + && isdigit(*(end - 3)) + && isdigit(*(end - 2)) + && isdigit(*(end - 1))); + +} + +/** Add a space ID to filename mapping. +@param[in] space_id Tablespace ID +@param[in] name File name. +@return number of files that map to the space ID */ +size_t +Tablespace_files::add(space_id_t space_id, const std::string& name) +{ + ut_a(space_id != TRX_SYS_SPACE); + + Names* names; + + if (Fil_path::is_undo_tablespace_name(name)) { + + if (!dict_sys_t::is_reserved(space_id)) { + + ib::warn() + << "Tablespace '" << name << "' naming" + << " format is like an undo tablespace" + << " but its ID " << space_id << " is not" + << " in the undo tablespace range"; + } + + names = &m_undo_paths[space_id]; + + } else { + + ut_ad(Fil_path::has_ibd_suffix(name.c_str())); + + names = &m_ibd_paths[space_id]; + } + + names->push_back(name); + + return(names->size()); +} + +/** Reads data from a space to a buffer. Remember that the possible incomplete +blocks at the end of file are ignored: they are not taken into account when +calculating the byte offset within a space. +@param[in] page_id page id +@param[in] page_size page size +@param[in] byte_offset remainder of offset in bytes; in aio this +must be divisible by the OS block size +@param[in] len how many bytes to read; this must not cross a +file boundary; in aio this must be a block size multiple +@param[in,out] buf buffer where to store data read; in aio this +must be appropriately aligned +@return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do +i/o on a tablespace which does not exist */ +static +dberr_t +fil_read( + const page_id_t& page_id, + const page_size_t& page_size, + ulint byte_offset, + ulint len, + void* buf) +{ + return(fil_io(IORequestRead, true, page_id, page_size, + byte_offset, len, buf, nullptr)); +} + +/** Writes data to a space from a buffer. Remember that the possible incomplete +blocks at the end of file are ignored: they are not taken into account when +calculating the byte offset within a space. +@param[in] page_id page id +@param[in] page_size page size +@param[in] byte_offset remainder of offset in bytes; in aio this +must be divisible by the OS block size +@param[in] len how many bytes to write; this must not cross +a file boundary; in aio this must be a block size multiple +@param[in] buf buffer from which to write; in aio this must +be appropriately aligned +@return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do + I/O on a tablespace which does not exist */ +static +dberr_t +fil_write( + const page_id_t& page_id, + const page_size_t& page_size, + ulint byte_offset, + ulint len, + void* buf) +{ + ut_ad(!srv_read_only_mode); + + return(fil_io(IORequestWrite, true, page_id, page_size, + byte_offset, len, buf, nullptr)); +} + +/** Look up a tablespace. The caller should hold an InnoDB table lock or +a MDL that prevents the tablespace from being dropped during the operation, +or the caller should be in single-threaded crash recovery mode (no user +connections that could drop tablespaces). If this is not the case, +fil_space_acquire() and fil_space_release() should be used instead. +@param[in] space_id Tablespace ID +@return tablespace, or nullptr if not found */ +fil_space_t* +fil_space_get(space_id_t space_id) +{ + auto shard = fil_system->shard_by_id(space_id); + + shard->mutex_acquire(); + + fil_space_t* space = shard->get_space_by_id(space_id); + + shard->mutex_release(); + + return(space); +} + +#ifndef UNIV_HOTBACKUP +/** Returns the latch of a file space. +@param[in] space_id Tablespace ID +@return latch protecting storage allocation */ +rw_lock_t* +fil_space_get_latch(space_id_t space_id) +{ + auto shard = fil_system->shard_by_id(space_id); + + shard->mutex_acquire(); + + fil_space_t* space = shard->get_space_by_id(space_id); + + shard->mutex_release(); + + return(&space->latch); +} + +# ifdef UNIV_DEBUG +/** Gets the type of a file space. +@param[in] space_id Tablespace ID +@return file type */ +fil_type_t +fil_space_get_type(space_id_t space_id) +{ + fil_space_t* space; + + auto shard = fil_system->shard_by_id(space_id); + + shard->mutex_acquire(); + + space = shard->get_space_by_id(space_id); + + shard->mutex_release(); + + return(space->purpose); +} +# endif /* UNIV_DEBUG */ + +/** Note that a tablespace has been imported. +It is initially marked as FIL_TYPE_IMPORT so that no logging is +done during the import process when the space ID is stamped to each page. +Now we change it to FIL_SPACE_TABLESPACE to start redo and undo logging. +NOTE: temporary tablespaces are never imported. +@param[in] space_id Tablespace ID */ +void +fil_space_set_imported(space_id_t space_id) +{ + auto shard = fil_system->shard_by_id(space_id); - fil_space_t* space = fil_space_get_by_id(id); + shard->mutex_acquire(); + + fil_space_t* space = shard->get_space_by_id(space_id); ut_ad(space->purpose == FIL_TYPE_IMPORT); space->purpose = FIL_TYPE_TABLESPACE; - mutex_exit(&fil_system->mutex); + shard->mutex_release(); } #endif /* !UNIV_HOTBACKUP */ -/**********************************************************************//** -Checks if all the file nodes in a space are flushed. The caller must hold +/** Checks if all the file nodes in a space are flushed. The caller must hold the fil_system mutex. +@param[in] space Tablespace to check @return true if all are flushed */ -static bool -fil_space_is_flushed( -/*=================*/ - fil_space_t* space) /*!< in: space */ +Fil_shard::space_is_flushed(const fil_space_t* space) { - ut_ad(mutex_own(&fil_system->mutex)); + ut_ad(mutex_owned()); - for (const fil_node_t* node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { + for (const auto& file : space->files) { - if (node->modification_counter > node->flush_counter) { + if (file.modification_counter > file.flush_counter) { ut_ad(!fil_buffering_disabled(space)); return(false); @@ -1238,8 +2278,7 @@ fil_space_is_flushed( /** FusionIO atomic write control info */ #define DFS_IOCTL_ATOMIC_WRITE_SET _IOW(0x95, 2, uint) -/** -Try and enable FusionIO atomic writes. +/** Try and enable FusionIO atomic writes. @param[in] file OS file handle @return true if successful */ bool @@ -1262,58 +2301,54 @@ fil_fusionio_enable_atomic_write(pfs_os_file_t file) } #endif /* !NO_FALLOCATE && UNIV_LINUX */ -/** Append a file to the chain of files of a space. +/** Attach a file to a tablespace @param[in] name file name of a file that is not open @param[in] size file size in entire database blocks @param[in,out] space tablespace from fil_space_create() @param[in] is_raw whether this is a raw device or partition -@param[in] punch_hole true if supported for this node +@param[in] punch_hole true if supported for this file @param[in] atomic_write true if the file has atomic write enabled @param[in] max_pages maximum number of pages in file @return pointer to the file name -@retval NULL if error */ -static +@retval nullptr if error */ fil_node_t* -fil_node_create_low( +Fil_shard::create_node( const char* name, page_no_t size, fil_space_t* space, bool is_raw, bool punch_hole, bool atomic_write, - page_no_t max_pages = PAGE_NO_MAX) + page_no_t max_pages) { - fil_node_t* node; + ut_ad(name != nullptr); + ut_ad(fil_system != nullptr); - ut_ad(name != NULL); - ut_ad(fil_system != NULL); - - if (space == NULL) { - return(NULL); + if (space == nullptr) { + return(nullptr); } - node = reinterpret_cast(ut_zalloc_nokey(sizeof(*node))); + fil_node_t file; - node->name = mem_strdup(name); + memset(&file, 0x0, sizeof(file)); - ut_a(!is_raw || srv_start_raw_disk_in_use); + file.name = mem_strdup(name); - node->sync_event = os_event_create("fsync_event"); + ut_a(!is_raw || srv_start_raw_disk_in_use); - node->is_raw_disk = is_raw; + file.sync_event = os_event_create("fsync_event"); - node->size = size; + file.is_raw_disk = is_raw; - node->magic_n = FIL_NODE_MAGIC_N; + file.size = size; - node->init_size = size; - node->max_size = max_pages; + file.magic_n = FIL_NODE_MAGIC_N; - mutex_enter(&fil_system->mutex); + file.init_size = size; - space->size += size; + file.max_size = max_pages; - node->space = space; + file.space = space; os_file_stat_t stat_info; @@ -1322,12 +2357,12 @@ fil_node_create_low( #endif /* UNIV_DEBUG */ os_file_get_status( - node->name, &stat_info, false, + file.name, &stat_info, false, fsp_is_system_temporary(space->id) ? true : srv_read_only_mode); ut_ad(err == DB_SUCCESS); - node->block_size = stat_info.block_size; + file.block_size = stat_info.block_size; /* In this debugging mode, we can overcome the limitation of some OSes like Windows that support Punch Hole but have a hole size @@ -1336,38 +2371,48 @@ fil_node_create_low( turn Page Compression off. This execution mode allows compression to be tested even when full punch hole support is not available. */ DBUG_EXECUTE_IF("ignore_punch_hole", - node->block_size = ut_min( + file.block_size = ut_min( static_cast(stat_info.block_size), UNIV_PAGE_SIZE / 2); ); if (!IORequest::is_punch_hole_supported() || !punch_hole - || node->block_size >= srv_page_size) { + || file.block_size >= srv_page_size) { - fil_no_punch_hole(node); + fil_no_punch_hole(&file); } else { - node->punch_hole = punch_hole; + file.punch_hole = punch_hole; } - node->atomic_write = atomic_write; + file.atomic_write = atomic_write; + + mutex_acquire(); + + space->size += size; + + space->files.push_back(file); - UT_LIST_ADD_LAST(space->chain, node); - mutex_exit(&fil_system->mutex); + mutex_release(); - return(node); + ut_a(space->id == TRX_SYS_SPACE + || space->id == dict_sys_t::s_log_space_first_id + || space->purpose == FIL_TYPE_TEMPORARY + || space->files.size() == 1); + + return(&space->files.front()); } -/** Appends a new file to the chain of files of a space. File must be closed. +/** Attach a file to a tablespace. File must be closed. @param[in] name file name (file must be closed) -@param[in] size file size in database blocks, rounded downwards to - an integer +@param[in] size file size in database blocks, rounded + downwards to an integer @param[in,out] space space where to append @param[in] is_raw true if a raw device or a raw disk partition @param[in] atomic_write true if the file has atomic write enabled @param[in] max_pages maximum number of pages in file @return pointer to the file name -@retval NULL if error */ +@retval nullptr if error */ char* fil_node_create( const char* name, @@ -1377,418 +2422,372 @@ fil_node_create( bool atomic_write, page_no_t max_pages) { - fil_node_t* node; + auto shard = fil_system->shard_by_id(space->id); + + fil_node_t* file; - node = fil_node_create_low( + file = shard->create_node( name, size, space, is_raw, IORequest::is_punch_hole_supported(), atomic_write, max_pages); - return(node == NULL ? NULL : node->name); + return(file == nullptr ? nullptr : file->name); } -/** Open a file node of a tablespace. -The caller must own the fil_system mutex. -@param[in,out] node File node -@param[in] extend true if the file is being extended -@return false if the file can't be opened, otherwise true */ -static -bool -fil_node_open_file(fil_node_t* node, bool extend) +/** First we open the file in the normal mode, no async I/O here, for +simplicity. Then do some checks, and close the file again. NOTE that we +could not use the simple file read function os_file_read() in Windows +to read from a file opened for async I/O! +@param[in,out] file Get the size of this file +@param[in] read_only_mode true if read only mode set +@return DB_SUCCESS or error */ +dberr_t +Fil_shard::get_file_size( + fil_node_t* file, + bool read_only_mode) { - os_offset_t size_bytes; bool success; - byte* buf2; - byte* page; - space_id_t space_id; - ulint flags; - ulint min_size; - bool read_only_mode; - fil_space_t* space = node->space; - - ut_ad(mutex_own(&fil_system->mutex)); - ut_a(node->n_pending == 0); - ut_a(!node->is_open); - - /* If the file is being opened then we wait for the MLOG_FILE_OPEN - redo log to be written to the redo log. */ - while (node->in_use > 0) { - - /* We increment the reference count when extending - the file. */ - if (node->in_use == 1 && extend) { - break; - } - - mutex_exit(&fil_system->mutex); - - os_thread_sleep(100000); - - mutex_enter(&fil_system->mutex); - } - - if (node->is_open) { - - return(true); - } + fil_space_t* space = file->space; - read_only_mode = !fsp_is_system_temporary(space->id) - && srv_read_only_mode; + do { + ut_a(!file->is_open); - if (node->size == 0 - || (space->size_in_header == 0 - && space->purpose == FIL_TYPE_TABLESPACE - && node == UT_LIST_GET_FIRST(space->chain) -#ifndef UNIV_HOTBACKUP - && undo::is_active(space->id) - && srv_startup_is_before_trx_rollback_phase -#endif /* !UNIV_HOTBACKUP */ - )) { - /* We do not know the size of the file yet. First we - open the file in the normal mode, no async I/O here, - for simplicity. Then do some checks, and close the - file again. NOTE that we could not use the simple - file read function os_file_read() in Windows to read - from a file opened for async I/O! */ - -retry: - ut_a(!node->is_open); - - node->handle = os_file_create_simple_no_error_handling( - innodb_data_file_key, node->name, OS_FILE_OPEN, + file->handle = os_file_create_simple_no_error_handling( + innodb_data_file_key, file->name, OS_FILE_OPEN, OS_FILE_READ_ONLY, read_only_mode, &success); if (!success) { + /* The following call prints an error message */ - ulint err = os_file_get_last_error(true); + ulint err = os_file_get_last_error(true); if (err == EMFILE + 100) { - /* Note: This call will release the - file system mutex temporarily. */ + if (close_files_in_LRU(true)) { - if (fil_try_to_close_file_in_LRU(true)) { - goto retry; + continue; } } ib::warn() - << "Cannot open '" << node->name << "'." + << "Cannot open '" << file->name << "'." " Have you deleted .ibd files under a" " running mysqld server?"; - return(false); + return(DB_ERROR); } - size_bytes = os_file_get_size(node->handle); - ut_a(size_bytes != (os_offset_t) -1); + } while (!success); + + os_offset_t size_bytes = os_file_get_size(file->handle); + + ut_a(size_bytes != (os_offset_t) -1); #ifdef UNIV_HOTBACKUP - if (space->id == 0) { - node->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); - os_file_close(node->handle); - space->size += node->size; - } else { + if (space->id == TRX_SYS_SPACE) { + file->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); + space->size += file->size; + os_file_close(file->handle); + return(DB_SUCCESS); + } #endif /* UNIV_HOTBACKUP */ - ut_a(space->purpose != FIL_TYPE_LOG); - /* Read the first page of the tablespace */ + ut_a(space->purpose != FIL_TYPE_LOG); - buf2 = static_cast(ut_malloc_nokey(2 * UNIV_PAGE_SIZE)); + /* Read the first page of the tablespace */ - /* Align the memory for file i/o if we might have O_DIRECT - set */ - page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); - ut_ad(page == page_align(page)); + byte* buf2 = static_cast(ut_malloc_nokey(2 * UNIV_PAGE_SIZE)); - IORequest request(IORequest::READ); + /* Align memory for file I/O if we might have O_DIRECT set */ - success = os_file_read_first_page( - request, - node->handle, page, UNIV_PAGE_SIZE); + byte* page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); - space_id = fsp_header_get_space_id(page); - flags = fsp_header_get_flags(page); + ut_ad(page == page_align(page)); - /* Close the file now that we have read the space id from it */ + IORequest request(IORequest::READ); - os_file_close(node->handle); + dberr_t err = os_file_read_first_page( + request, file->handle, page, UNIV_PAGE_SIZE); - const page_size_t page_size(flags); + ut_a(err == DB_SUCCESS); - /* To determine if tablespace is from 5.7 or not, we - rely on SDI flag. For IBDs from 5.7, which are opened - during import or during upgrade, their initial size - is lesser than the initial size in 8.0 */ - bool has_sdi = FSP_FLAGS_HAS_SDI(flags); - uint8_t expected_size = has_sdi ? FIL_IBD_FILE_INITIAL_SIZE - : FIL_IBD_FILE_INITIAL_SIZE_5_7; + os_file_close(file->handle); - min_size = expected_size * page_size.physical(); + ulint flags = fsp_header_get_flags(page); + space_id_t space_id = fsp_header_get_space_id(page); - if (size_bytes < min_size) { + /* To determine if tablespace is from 5.7 or not, we + rely on SDI flag. For IBDs from 5.7, which are opened + during import or during upgrade, their initial size + is lesser than the initial size in 8.0 */ + bool has_sdi = FSP_FLAGS_HAS_SDI(flags); - ib::error() << "The size of tablespace file " - << node->name << " is only " << size_bytes - << ", should be at least " << min_size << "!"; + uint8_t expected_size = has_sdi + ? FIL_IBD_FILE_INITIAL_SIZE + : FIL_IBD_FILE_INITIAL_SIZE_5_7; - ut_error; - } + const page_size_t page_size(flags); - if (space_id != space->id) { - ib::fatal() << "Tablespace id is " << space->id - << " in the data dictionary but in file " - << node->name << " it is " << space_id << "!"; - } + ulint min_size = expected_size * page_size.physical(); - const page_size_t space_page_size(space->flags); + if (size_bytes < min_size) { - if (!page_size.equals_to(space_page_size)) { - ib::fatal() << "Tablespace file " << node->name - << " has page size " << page_size - << " (flags=" << ib::hex(flags) << ") but the" - " data dictionary expects page size " - << space_page_size << " (flags=" - << ib::hex(space->flags) << ")!"; - } + ib::error() + << "The size of tablespace file " + << file->name << " is only " << size_bytes + << ", should be at least " << min_size << "!"; - /* TODO: Remove this adjustment and enable the below assert - after dict_tf_to_fsp_flags() removal. */ - space->flags |= flags & FSP_FLAGS_MASK_SDI; - /* ut_ad(space->flags == flags); */ + ut_error; + } - if (space->flags != flags) { + if (space_id != space->id) { - ib::fatal() - << "Table flags are " - << ib::hex(space->flags) << " in the data" - " dictionary but the flags in file " - << node->name << " are " << ib::hex(flags) - << "!"; - } + ib::fatal() + << "Tablespace id is " << space->id + << " in the data dictionary but in file " + << file->name << " it is " << space_id << "!"; + } - { - page_no_t size = fsp_header_get_field( - page, FSP_SIZE); - page_no_t free_limit = fsp_header_get_field( - page, FSP_FREE_LIMIT); - ulint free_len = flst_get_len( - FSP_HEADER_OFFSET + FSP_FREE + page); - ut_ad(space->free_limit == 0 - || space->free_limit == free_limit); - ut_ad(space->free_len == 0 - || space->free_len == free_len); - space->size_in_header = size; - space->free_limit = free_limit; - space->free_len = free_len; - } - - ut_free(buf2); - - /* For encrypted tablespace, we need to check the - encrytion key and iv(initial vector) is readed. */ - if (FSP_FLAGS_GET_ENCRYPTION(flags) - && !recv_recovery_is_on()) { - if (space->encryption_type != Encryption::AES) { - ib::error() - << "Can't read encryption" - << " key from file " - << node->name << "!"; - return(false); - } - } + /* We need to adjust for compressed pages. */ + const page_size_t space_page_size(space->flags); - if (node->size == 0) { - ulint extent_size; + if (!page_size.equals_to(space_page_size)) { - extent_size = page_size.physical() * FSP_EXTENT_SIZE; + ib::fatal() + << "Tablespace file " << file->name + << " has page size " << page_size + << " (flags=" << ib::hex(flags) + << ") but the data dictionary expects" + << " page size " << space_page_size + << " (flags=" + << ib::hex(space->flags) << ")!"; + } - /* After apply-incremental, tablespaces are not extended - to a whole megabyte. Do not cut off valid data. */ -#ifndef UNIV_HOTBACKUP - /* Truncate the size to a multiple of extent size. */ - if (size_bytes >= extent_size) { - size_bytes = ut_2pow_round(size_bytes, - extent_size); - } -#endif /* !UNIV_HOTBACKUP */ + /* TODO: Remove this adjustment and enable the below + assert after dict_tf_to_fsp_flags() removal. */ + space->flags |= flags & FSP_FLAGS_MASK_SDI; + /* ut_ad(space->flags == flags); */ - node->size = static_cast( - (size_bytes / page_size.physical())); + if (space->flags != flags) { - space->size += node->size; - } -#ifdef UNIV_HOTBACKUP - } -#endif /* UNIV_HOTBACKUP */ + ib::fatal() + << "Table flags are " + << ib::hex(space->flags) + << " in the data dictionary but the" + << " flags in file " << file->name + << " are " << ib::hex(flags) << "!"; } - /* Open the file for reading and writing, in Windows normally in the - unbuffered async I/O mode, though global variables may make - os_file_create() to fall back to the normal file I/O mode. */ + { + page_no_t size = fsp_header_get_field(page, FSP_SIZE); - if (space->purpose == FIL_TYPE_LOG) { - node->handle = os_file_create( - innodb_log_file_key, node->name, OS_FILE_OPEN, - OS_FILE_AIO, OS_LOG_FILE, read_only_mode, &success); - } else if (node->is_raw_disk) { - node->handle = os_file_create( - innodb_data_file_key, node->name, OS_FILE_OPEN_RAW, - OS_FILE_AIO, OS_DATA_FILE, read_only_mode, &success); - } else { - node->handle = os_file_create( - innodb_data_file_key, node->name, OS_FILE_OPEN, - OS_FILE_AIO, OS_DATA_FILE, read_only_mode, &success); - } + page_no_t free_limit; - ut_a(success); + free_limit = fsp_header_get_field(page, FSP_FREE_LIMIT); - /* The file file is ready for IO. */ + ulint free_len; - if (space->purpose == FIL_TYPE_TABLESPACE) { + free_len = flst_get_len(FSP_HEADER_OFFSET + FSP_FREE + page); - /* To obey the latching order. Note that we have set the - node->is_open above, another thread will not attempt to - open it again after we release the mutex. */ + ut_ad(space->free_limit == 0 + || space->free_limit == free_limit); - ++node->in_use; + ut_ad(space->free_len == 0 || space->free_len == free_len); - /* At this point it is safe to release fil_system mutex. No - other thread can rename, delete or close the file because - we have set the node->in_use flag. */ + space->size_in_header = size; + space->free_limit = free_limit; - mutex_exit(&fil_system->mutex); + ut_a(free_len < std::numeric_limits::max()); - fil_system->m_open.enter(); - fil_system->m_open.log(node->space->id, node->name); - fil_system->m_open.exit(); + space->free_len = (uint32_t) free_len; + } - mutex_enter(&fil_system->mutex); + ut_free(buf2); - ut_a(node->in_use > 0); - --node->in_use; - } + /* For encrypted tablespace, we need to check the + encrytion key and iv(initial vector) is readed. */ + if (FSP_FLAGS_GET_ENCRYPTION(flags) + && !recv_recovery_is_on() + && space->encryption_type != Encryption::AES) { - if (fil_space_belongs_in_lru(space)) { + ib::error() + << "Can't read encryption key from file " + << file->name << "!"; + + return(DB_ERROR); - /* Put the node to the LRU list */ - UT_LIST_ADD_FIRST(fil_system->LRU, node); } - /* Set the open flag after writing the MLOG_FILE_OPEN log record. */ - node->is_open = true; + if (file->size == 0) { - ++fil_n_file_opened; - ++fil_system->n_open; + ulint extent_size; - return(true); + extent_size = page_size.physical() * FSP_EXTENT_SIZE; + +#ifndef UNIV_HOTBACKUP + /* Truncate the size to a multiple of extent size. */ + if (size_bytes >= extent_size) { + + size_bytes = ut_2pow_round(size_bytes, extent_size); + } +#else /* !UNIV_HOTBACKUP */ + + /* After apply-incremental, tablespaces are not + extended to a whole megabyte. Do not cut off + valid data. */ + +#endif /* !UNIV_HOTBACKUP */ + + file->size = static_cast( + size_bytes / page_size.physical()); + + space->size += file->size; + } + + return(DB_SUCCESS); } -/** Close a file node. -@param[in] lru_close true if called from LRU close -@param[in,out] node File node */ -static -void -fil_node_close_file(fil_node_t* node, bool lru_close) +/** Open a file of a tablespace. +The caller must own the shard mutex. +@param[in,out] file Tablespace file +@param[in] extend true if the file is being extended +@return false if the file can't be opened, otherwise true */ +bool +Fil_shard::open_file(fil_node_t* file, bool extend) { - ut_ad(mutex_own(&(fil_system->mutex))); + bool success; + fil_space_t* space = file->space; + + ut_ad(m_id == REDO_SHARD || mutex_owned()); + + ut_a(!file->is_open); + ut_a(file->n_pending == 0); + + while (file->in_use > 0) { + + /* We increment the reference count when extending + the file. */ + if (file->in_use == 1 && extend) { + break; + } - ut_a(node->is_open); - ut_a(node->in_use == 0); - ut_a(node->n_pending == 0); - ut_a(node->n_pending_flushes == 0); + mutex_release(); + os_thread_sleep(100000); + + mutex_acquire(); + } + + if (file->is_open) { + + return(true); + } + + bool read_only_mode; + + read_only_mode = !fsp_is_system_temporary(space->id) + && srv_read_only_mode; + + if (file->size == 0 + || (space->size_in_header == 0 + && space->purpose == FIL_TYPE_TABLESPACE + && file == &space->files.front() #ifndef UNIV_HOTBACKUP - ut_a(node->modification_counter == node->flush_counter - || node->space->purpose == FIL_TYPE_TEMPORARY - || srv_fast_shutdown == 2); + && undo::is_active(space->id) + && srv_startup_is_before_trx_rollback_phase #endif /* !UNIV_HOTBACKUP */ + )) { - bool ret = os_file_close(node->handle); - ut_a(ret); + /* We don't know the file size yet. */ + dberr_t err = get_file_size(file, read_only_mode); - node->is_open = false; + if (err != DB_SUCCESS) { + return(false); + } + } - ut_a(fil_system->n_open > 0); + /* Open the file for reading and writing, in Windows normally in the + unbuffered async I/O mode, though global variables may make + os_file_create() to fall back to the normal file I/O mode. */ - --fil_n_file_opened; - --fil_system->n_open; + if (space->purpose == FIL_TYPE_LOG) { + file->handle = os_file_create( + innodb_log_file_key, file->name, OS_FILE_OPEN, + OS_FILE_AIO, OS_LOG_FILE, read_only_mode, &success); + } else if (file->is_raw_disk) { + file->handle = os_file_create( + innodb_data_file_key, file->name, OS_FILE_OPEN_RAW, + OS_FILE_AIO, OS_DATA_FILE, read_only_mode, &success); + } else { + file->handle = os_file_create( + innodb_data_file_key, file->name, OS_FILE_OPEN, + OS_FILE_AIO, OS_DATA_FILE, read_only_mode, &success); + } + + ut_a(success); - if (fil_space_belongs_in_lru(node->space)) { + /* The file is ready for IO. */ - ut_a(UT_LIST_GET_LEN(fil_system->LRU) > 0); + file_opened(file); - /* The node is in the LRU list, remove it */ - UT_LIST_REMOVE(fil_system->LRU, node); - } + return(true); +} + +/** Close a tablespace file. +@param[in] LRU_close true if called from LRU close +@param[in,out] file Tablespace file to close */ +void +Fil_shard::close_file(fil_node_t* file, bool LRU_close) +{ + ut_ad(mutex_owned()); - /* Temporary tablespace is recreated on startup. It is never - recovered via redo. We can ignore it. */ + ut_a(file->is_open); + ut_a(file->in_use == 0); + ut_a(file->n_pending == 0); + ut_a(file->n_pending_flushes == 0); - if (node->space->purpose == FIL_TYPE_TABLESPACE - && !srv_read_only_mode) { +#ifndef UNIV_HOTBACKUP + ut_a(file->modification_counter == file->flush_counter + || file->space->purpose == FIL_TYPE_TEMPORARY + || srv_fast_shutdown == 2); +#endif /* !UNIV_HOTBACKUP */ - if (lru_close) { + bool ret = os_file_close(file->handle); - /* To obey the latching order. Note that we have - set the node->is_open above, another thread will - not attempt to open it again after we release the - mutex. */ + ut_a(ret); - ++node->in_use; + file->handle.m_file = (os_file_t) -1; - /* At this point it is safe to release - fil_system mutex. No other thread can rename, delete - or close the file because we have set the - node->in_use flag. */ + file->is_open = false; - mutex_exit(&fil_system->mutex); - } + ut_a(s_n_open > 0); - fil_system->m_open.enter(); - fil_system->m_open.close(node->space->id, node->name); - fil_system->m_open.exit(); + --s_n_open; - if (lru_close) { - mutex_enter(&fil_system->mutex); + ut_a(fil_n_file_opened > 0); - ut_a(node->in_use > 0); - --node->in_use; - } - } + --fil_n_file_opened; + + remove_from_LRU(file); } -/** Tries to close a file in the LRU list. The caller must hold the fil_sys -mutex. -@return true if success, false if should retry later; since i/o's -generally complete in < 100 ms, and as InnoDB writes at most 128 pages -from the buffer pool in a batch, and then immediately flushes the -files, there is a good chance that the next time we find a suitable -node from the LRU list. -Will release the fil_system->mutex if the file was closed -@param[in] print_info if true, prints information why it - cannot close a file */ -static +/** Tries to close a file in the LRU list. +@param[in] print_info if true, prints information why it cannot close + a file +@return true if success, false if should retry later */ bool -fil_try_to_close_file_in_LRU(bool print_info) - +Fil_shard::close_files_in_LRU(bool print_info) { - ut_ad(mutex_own(&fil_system->mutex)); + ut_ad(mutex_owned()); - if (print_info) { - ib::info() << "fil_sys open file LRU len " - << UT_LIST_GET_LEN(fil_system->LRU); - } - - for (auto node = UT_LIST_GET_LAST(fil_system->LRU); - node != NULL; - node = UT_LIST_GET_PREV(LRU, node)) { + for (auto file = UT_LIST_GET_LAST(m_LRU); + file != nullptr; + file = UT_LIST_GET_PREV(LRU, file)) { - if (node->modification_counter == node->flush_counter - && node->n_pending_flushes == 0 - && node->in_use == 0) { + if (file->modification_counter == file->flush_counter + && file->n_pending_flushes == 0 + && file->in_use == 0) { - /* Will release the fil_system->mutex. */ - fil_node_close_file(node, true); + close_file(file, true); return(true); } @@ -1797,22 +2796,27 @@ fil_try_to_close_file_in_LRU(bool print_info) continue; } - if (node->n_pending_flushes > 0) { + if (file->n_pending_flushes > 0) { - ib::info() << "Cannot close file " << node->name + ib::info() + << "Cannot close file " << file->name << ", because n_pending_flushes " - << node->n_pending_flushes; + << file->n_pending_flushes; } - if (node->modification_counter != node->flush_counter) { - ib::warn() << "Cannot close file " << node->name + if (file->modification_counter != file->flush_counter) { + + ib::warn() + << "Cannot close file " << file->name << ", because modification count " - << node->modification_counter << - " != flush count " << node->flush_counter; + << file->modification_counter << + " != flush count " << file->flush_counter; } - if (node->in_use > 0) { - ib::info() << "Cannot close file " << node->name + if (file->in_use > 0) { + + ib::info() + << "Cannot close file " << file->name << ", because it is in use"; } } @@ -1820,354 +2824,467 @@ fil_try_to_close_file_in_LRU(bool print_info) return(false); } -/*******************************************************************//** -Reserves the fil_system mutex and tries to make sure we can open at least one -file while holding it. This should be called before calling -fil_node_prepare_for_io(), because that function may need to open a file. */ -static -void -fil_mutex_enter_and_prepare_for_io( -/*===============================*/ - space_id_t space_id) /*!< in: space id */ +/** Tries to close a file in the LRU list. +@param[in] print_info if true, prints information why it cannot close a file +@return true if success, false if should retry later */ +bool +Fil_system::close_file_in_all_LRU(bool print_info) { - fil_space_t* space; - bool success; - bool print_info = false; - ulint count = 0; - ulint count2 = 0; + for (auto shard : m_shards) { - for (;;) { - mutex_enter(&fil_system->mutex); + shard->mutex_acquire(); - if (space_id == 0 || dict_sys_t::is_reserved(space_id)) { - /* We keep log files and system tablespace files always - open; this is important in preventing deadlocks in this - module, as a page read completion often performs - another read from the insert buffer. The insert buffer - is in tablespace 0, and we cannot end up waiting in - this function. */ - return; + if (print_info) { + ib::info() + << "Open file list len in shard " + << shard->id() << " is " + << UT_LIST_GET_LEN(shard->m_LRU); } - space = fil_space_get_by_id(space_id); + bool success = shard->close_files_in_LRU(print_info); - if (space != NULL && space->stop_ios) { - /* We are going to do a rename file and want to stop - new i/o's for a while. */ - - if (count2 > 20000) { - ib::warn() << "Tablespace " << space->name - << " has i/o ops stopped for a long" - " time " << count2; - } + shard->mutex_release(); - mutex_exit(&fil_system->mutex); + if (success) { + return(true);; + } + } -#ifndef UNIV_HOTBACKUP + return(false); +} - /* Wake the i/o-handler threads to make sure pending - i/o's are performed */ - os_aio_simulated_wake_handler_threads(); +/** We are going to do a rename file and want to stop new I/O for a while. +@param[in,out] space Tablespace for which we want to wait for IO + to stop */ +void +Fil_shard::wait_for_io_to_stop(fil_space_t* space) +{ + /* Note: We are reading the value of space->stop_ios without the + cover of the Fil_shard::mutex. We incremented the in_use counter + before waiting for IO to stop. */ - /* The sleep here is just to give IO helper threads a - bit of time to do some work. It is not required that - all IO related to the tablespace being renamed must - be flushed here as we do fil_flush() in - fil_rename_tablespace() as well. */ - os_thread_sleep(20000); + auto begin_time = ut_time(); + auto start_time = begin_time; -#endif /* !UNIV_HOTBACKUP */ + /* Spam the log after every minute. Ignore any race here. */ - /* Flush tablespaces so that we can close modified - files in the LRU list */ - fil_flush_file_spaces(to_int(FIL_TYPE_TABLESPACE)); + while (space->stop_ios) { - os_thread_sleep(20000); + if ((ut_time() - start_time) == PRINT_INTERVAL_SECS) { - count2++; + start_time = ut_time(); - continue; + ib::warn() + << "Tablespace " << space->name << ", " + << " waiting for IO to stop for " + << ut_time() - begin_time << " seconds"; } - if (fil_system->n_open < fil_system->max_n_open) { +#ifndef UNIV_HOTBACKUP - return; - } + /* Wake the I/O handler threads to make sure + pending I/O's are performed */ + os_aio_simulated_wake_handler_threads(); - /* If the file is already open, no need to do anything; if the - space does not exist, we handle the situation in the function - which called this function. */ +#endif /* UNIV_HOTBACKUP */ - if (space == NULL || UT_LIST_GET_FIRST(space->chain)->is_open) { + /* Give the IO threads some time to work. */ + os_thread_yield(); + } +} - return; - } +/** We keep log files and system tablespace files always open; this is +important in preventing deadlocks in this module, as a page read +completion often performs another read from the insert buffer. The +insert buffer is in tablespace TRX_SYS_SPACE, and we cannot end up +waiting in this function. +@param[in] space_id Tablespace ID to look up +@return tablespace instance */ +fil_space_t* +Fil_shard::get_reserved_space(space_id_t space_id) +{ + if (space_id == TRX_SYS_SPACE) { - if (count > 1) { - print_info = true; - } + return(fil_space_t::s_sys_space); - /* Too many files are open, try to close some */ - do { - /* Note: This function will release the - fil_system->mutex when it closes the file. */ + } else if (space_id == dict_sys_t::s_log_space_first_id + && fil_space_t::s_redo_space != nullptr) { - success = fil_try_to_close_file_in_LRU(print_info); + return(fil_space_t::s_redo_space); + } - } while (success - && fil_system->n_open >= fil_system->max_n_open); + return(get_space_by_id(space_id)); +} - if (fil_system->n_open < fil_system->max_n_open) { - /* Ok */ - return; - } +/** Reserves the mutex and tries to make sure we can open at least +one file while holding it. This should be called before calling +prepare_file_for_io(), because that function may need to open a file. +@param[in] space_id Tablespace ID +@param[out] space Tablespace instance +@return true if a slot was reserved. */ +bool +Fil_shard::mutex_acquire_and_get_space( + space_id_t space_id, + fil_space_t*& space) +{ + mutex_acquire(); - if (count >= 2) { - ib::warn() << "Too many (" << fil_system->n_open - << ") files stay open while the maximum" - " allowed value would be " - << fil_system->max_n_open << ". You may need" - " to raise the value of innodb_open_files in" - " my.cnf."; + if (space_id == TRX_SYS_SPACE || dict_sys_t::is_reserved(space_id)) { - return; - } + space = get_reserved_space(space_id); - mutex_exit(&fil_system->mutex); + return(false); + } -#ifndef UNIV_HOTBACKUP - /* Wake the i/o-handler threads to make sure pending i/o's are - performed */ - os_aio_simulated_wake_handler_threads(); + space = get_space_by_id(space_id); - os_thread_sleep(20000); -#endif /* !UNIV_HOTBACKUP */ - /* Flush tablespaces so that we can close modified files in - the LRU list. */ + if (space == nullptr) { + /* Caller handles the case of a missing tablespce. */ + return(false); + } - fil_flush_file_spaces(to_int(FIL_TYPE_TABLESPACE)); + ut_ad(space->files.size() == 1); - count++; + auto is_open = space->files.front().is_open; + + if (is_open) { + /* Ensure that the file is not closed behind our back. */ + ++space->files.front().in_use; } -} -/** Prepare to free a file node object from a tablespace memory cache. -@param[in,out] node file node -@param[in] space tablespace */ -static -void -fil_node_close_to_free( - fil_node_t* node, - fil_space_t* space) -{ - ut_ad(mutex_own(&fil_system->mutex)); - ut_a(node->magic_n == FIL_NODE_MAGIC_N); - ut_a(node->n_pending == 0); - ut_a(node->in_use == 0); + mutex_release(); - if (node->is_open) { - /* We fool the assertion in fil_node_close_file() to think - there are no unflushed modifications in the file */ + if (is_open) { - node->modification_counter = node->flush_counter; - os_event_set(node->sync_event); + wait_for_io_to_stop(space); - if (fil_buffering_disabled(space)) { + mutex_acquire(); - ut_ad(!space->is_in_unflushed_spaces); - ut_ad(fil_space_is_flushed(space)); + /* We are guaranteed that this file cannot be closed + because we now own the mutex. */ - } else if (space->is_in_unflushed_spaces - && fil_space_is_flushed(space)) { + ut_ad(space->files.front().in_use > 0); + --space->files.front().in_use; - space->is_in_unflushed_spaces = false; + return(false); + } - UT_LIST_REMOVE(fil_system->unflushed_spaces, space); - } + /* The number of open file descriptors is a shared resource, in + order to guarantee that we don't over commit, we use a ticket system + to reserve a slot/ticket to open a file. This slot/ticket should + be released after the file is opened. */ - /* TODO: set second parameter to true, so to release - fil_system mutex before logging tablespace name and id. - To go around Bug#26271853 - POTENTIAL DEADLOCK BETWEEN - FIL_SYSTEM MUTEX AND LOG MUTEX */ - fil_node_close_file(node, true); + while (!reserve_open_slot(m_id)) { + os_thread_yield(); } -} -/** Detach a space object from the tablespace memory cache. -Closes the files in the chain but does not delete them. -There must not be any pending i/o's or flushes on the files. -@param[in,out] space tablespace */ -static -void -fil_space_detach( - fil_space_t* space) -{ - ut_ad(mutex_own(&fil_system->mutex)); + auto begin_time = ut_time(); + auto start_time = begin_time; - fil_system->spaces.erase(space->id); + size_t i; - fil_system->names.erase(space->name); + for (i = 0; i < 3; ++i) { - if (space->is_in_unflushed_spaces) { + /* Flush tablespaces so that we can close modified + files in the LRU list */ - ut_ad(!fil_buffering_disabled(space)); - space->is_in_unflushed_spaces = false; + auto type = to_int(FIL_TYPE_TABLESPACE); - UT_LIST_REMOVE(fil_system->unflushed_spaces, space); - } + fil_system->flush_file_spaces(type); - UT_LIST_REMOVE(fil_system->space_list, space); + os_thread_yield(); - ut_a(space->magic_n == FIL_SPACE_MAGIC_N); - ut_a(space->n_pending_flushes == 0); + /* Reserve an open slot for this shard. So that this + shard's open file succeeds. */ - for (fil_node_t* fil_node = UT_LIST_GET_FIRST(space->chain); - fil_node != NULL; - fil_node = UT_LIST_GET_NEXT(chain, fil_node)) { + while (fil_system->m_max_n_open <= s_n_open + && !fil_system->close_file_in_all_LRU(i > 1)) { - fil_node_close_to_free(fil_node, space); - } -} + if (ut_time() - start_time == PRINT_INTERVAL_SECS) { -/** Free a tablespace object on which fil_space_detach() was invoked. -There must not be any pending i/o's or flushes on the files. + start_time = ut_time(); + + ib::warn() + << "Trying to close a file for " + << ut_time() - begin_time << " seconds" + << ". Configuration only allows for " + << fil_system->m_max_n_open + <<" open files."; + } + } + + if (fil_system->m_max_n_open > s_n_open) { + break; + } + +#ifndef UNIV_HOTBACKUP + /* Wake the I/O-handler threads to make sure pending I/Os are + performed */ + os_aio_simulated_wake_handler_threads(); + + os_thread_yield(); +#endif /* !UNIV_HOTBACKUP */ + } + + if (i >= 2) { + + ib::warn() + << "Too many (" << s_n_open + << ") files are open the maximum allowed" + << " value is " << fil_system->m_max_n_open + << ". You should raise the value of" + << " --innodb-open-files in my.cnf."; + } + + mutex_acquire(); + + return(true); +} + +/** Prepare to free a file. Remove from the unflushed list if there +are no pending flushes. +@param[in,out] file File instance to free */ +void +Fil_shard::prepare_to_free_file(fil_node_t* file) +{ + ut_ad(mutex_owned()); + + fil_space_t* space = file->space; + + if (space->is_in_unflushed_spaces + && space_is_flushed(space)) { + + + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE(m_unflushed_spaces, space); + } +} + +/** Prepare to free a file object from a tablespace memory cache. +@param[in,out] file Tablespace file +@param[in] space tablespace */ +void +Fil_shard::file_close_to_free( + fil_node_t* file, + fil_space_t* space) +{ + ut_ad(mutex_owned()); + ut_a(file->magic_n == FIL_NODE_MAGIC_N); + ut_a(file->n_pending == 0); + ut_a(file->in_use == 0); + ut_a(file->space == space); + + if (file->is_open) { + + /* We fool the assertion in Fil_system::close_file() to think + there are no unflushed modifications in the file */ + + file->modification_counter = file->flush_counter; + + os_event_set(file->sync_event); + + if (fil_buffering_disabled(space)) { + + ut_ad(!space->is_in_unflushed_spaces); + ut_ad(space_is_flushed(space)); + + } else { + prepare_to_free_file(file); + } + + /* TODO: set second parameter to true, so to release + fil_system mutex before logging tablespace name and id. + To go around Bug#26271853 - POTENTIAL DEADLOCK BETWEEN + FIL_SYSTEM MUTEX AND LOG MUTEX */ + close_file(file, true); + } +} + +/** Detach a space object from the tablespace memory cache. +Closes the tablespace files but does not delete them. +There must not be any pending I/O's or flushes on the files. +@param[in,out] space tablespace */ +void +Fil_shard::space_detach(fil_space_t* space) +{ + ut_ad(mutex_owned()); + + m_names.erase(space->name); + + if (space->is_in_unflushed_spaces) { + + ut_ad(!fil_buffering_disabled(space)); + + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE(m_unflushed_spaces, space); + } + + ut_a(space->magic_n == FIL_SPACE_MAGIC_N); + ut_a(space->n_pending_flushes == 0); + + for (auto& file : space->files) { + + file_close_to_free(&file, space); + } +} + +/** Free a tablespace object on which fil_space_detach() was invoked. +There must not be any pending I/O's or flushes on the files. @param[in,out] space tablespace */ -static void -fil_space_free_low(fil_space_t* space) +Fil_shard::space_free_low(fil_space_t*& space) { ut_ad(srv_fast_shutdown == 2 || space->max_lsn == 0); - for (fil_node_t* node = UT_LIST_GET_FIRST(space->chain); - node != NULL; ) { - ut_d(space->size -= node->size); - os_event_destroy(node->sync_event); - ut_free(node->name); - fil_node_t* old_node = node; - node = UT_LIST_GET_NEXT(chain, node); - ut_free(old_node); + for (auto& file : space->files) { + + ut_d(space->size -= file.size); + + os_event_destroy(file.sync_event); + + ut_free(file.name); } + call_destructor(&space->files); + ut_ad(space->size == 0); rw_lock_free(&space->latch); ut_free(space->name); ut_free(space); + + space = nullptr; +} + +/** Frees a space object from the tablespace memory cache. +Closes a tablespaces' files but does not delete them. +There must not be any pending I/O's or flushes on the files. +@param[in] space_id Tablespace ID +@return fil_space_t instance on success or nullptr */ +fil_space_t* +Fil_shard::space_free(space_id_t space_id) +{ + mutex_acquire(); + + fil_space_t* space = get_space_by_id(space_id); + + if (space != nullptr) { + + space_detach(space); + + space_delete(space_id); + } + + mutex_release(); + + return(space); } /** Frees a space object from the tablespace memory cache. -Closes the files in the chain but does not delete them. +Closes a tablespaces' files but does not delete them. There must not be any pending i/o's or flushes on the files. -@param[in] id tablespace identifier -@param[in] x_latched whether the caller holds X-mode space->latch +@param[in] space_id Tablespace ID +@param[in] x_latched Whether the caller holds X-mode space->latch @return true if success */ static bool -fil_space_free( - space_id_t id, - bool x_latched) +fil_space_free(space_id_t space_id, bool x_latched) { - ut_ad(id != TRX_SYS_SPACE); + ut_ad(space_id != TRX_SYS_SPACE); - mutex_enter(&fil_system->mutex); - fil_space_t* space = fil_space_get_by_id(id); + auto shard = fil_system->shard_by_id(space_id); + auto space = shard->space_free(space_id); - if (space != NULL) { - fil_space_detach(space); + if (space == nullptr) { + return(false); } - mutex_exit(&fil_system->mutex); - - if (space != NULL) { - if (x_latched) { - rw_lock_x_unlock(&space->latch); - } - - fil_space_free_low(space); + if (x_latched) { + rw_lock_x_unlock(&space->latch); } - return(space != NULL); + Fil_shard::space_free_low(space); + ut_a(space == nullptr); + + return(true); } /** Create a space memory object and put it to the fil_system hash table. The tablespace name is independent from the tablespace file-name. Error messages are issued to the server log. -@param[in] name Tablespace name -@param[in] id Tablespace identifier -@param[in] flags Tablespace flags -@param[in] purpose Tablespace purpose +@param[in] name Tablespace name +@param[in] space_id Tablespace identifier +@param[in] flags Tablespace flags +@param[in] purpose Tablespace purpose @return pointer to created tablespace, to be filled in with fil_node_create() -@retval NULL on failure (such as when the same tablespace exists) */ +@retval nullptr on failure (such as when the same tablespace exists) */ fil_space_t* -fil_space_create( +Fil_shard::space_create( const char* name, - space_id_t id, + space_id_t space_id, ulint flags, fil_type_t purpose) { - ut_ad(fsp_flags_is_valid(flags)); - ut_ad(srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0); + ut_ad(mutex_owned()); - DBUG_EXECUTE_IF("fil_space_create_failure", return(NULL);); + /* Look for a matching tablespace. */ + fil_space_t* space = get_space_by_name(name); - /* Must set back to active before returning from function. */ - clone_mark_abort(true); - mutex_enter(&fil_system->mutex); + if (space == nullptr) { + space = get_space_by_id(space_id); + } - /* Look for a matching tablespace. */ - fil_space_t* space = fil_space_get_by_name(name); + if (space != nullptr) { - if (space != NULL) { - mutex_exit(&fil_system->mutex); + std::ostringstream oss; - ib::warn() << "Tablespace '" << name << "' exists in the" - " cache with id " << space->id << " != " << id; + for (size_t i = 0; i < space->files.size(); ++i) { - clone_mark_active(); - return(NULL); - } + oss << "'" << space->files[i].name << "'"; - space = fil_space_get_by_id(id); + if (i < space->files.size() - 1) { + oss << ", "; + } + } - if (space != NULL) { - ib::error() << "Trying to add tablespace '" << name - << "' with id " << id - << " to the tablespace memory cache, but tablespace '" - << name << "' already exists in the cache!"; - mutex_exit(&fil_system->mutex); - clone_mark_active(); - return(NULL); + ib::info() + << "Trying to add tablespace '" << name << "'" + << " with id " << space_id << " to the tablespace" + << " memory cache, but tablespace" + << " '" << space->name << "'" + << " already exists in the cache with space ID " + << space->id + << ". It maps to the following file(s): " + << oss.str(); + + return(nullptr); } space = static_cast(ut_zalloc_nokey(sizeof(*space))); - space->id = id; + space->id = space_id; space->name = mem_strdup(name); - UT_LIST_INIT(space->chain, &fil_node_t::chain); + new(&space->files) fil_space_t::Files(); #ifndef UNIV_HOTBACKUP - if (fil_type_is_data(purpose) + if (fil_system->is_greater_than_max_id(space_id) + && fil_type_is_data(purpose) && !recv_recovery_on - && id > fil_system->max_assigned_id - && !dict_sys_t::is_reserved(id)) { - - if (!fil_system->space_id_reuse_warned) { - fil_system->space_id_reuse_warned = true; + && !dict_sys_t::is_reserved(space_id)) { - ib::warn() << "Allocated tablespace ID " << id - << " for " << name << ", old maximum was " - << fil_system->max_assigned_id; - } - - fil_system->max_assigned_id = id; + fil_system->set_maximum_space_id(space); } #endif /* !UNIV_HOTBACKUP */ space->purpose = purpose; - space->flags = flags; + + ut_a(flags < std::numeric_limits::max()); + space->flags = (uint32_t) flags; space->magic_n = FIL_SPACE_MAGIC_N; @@ -2181,59 +3298,99 @@ fil_space_create( } #endif /* !UNIV_HOTBACKUP */ - { - auto it = fil_system->spaces.insert( - Spaces::value_type(id, space)); + space_add(space); - ut_a(it.second); - } + return(space); +} - { - auto it = fil_system->names.insert( - Names::value_type(space->name, space)); +/** Create a space memory object and put it to the fil_system hash table. +The tablespace name is independent from the tablespace file-name. +Error messages are issued to the server log. +@param[in] name Tablespace name +@param[in] space_id Tablespace ID +@param[in] flags Tablespace flags +@param[in] purpose Tablespace purpose +@return pointer to created tablespace, to be filled in with fil_node_create() +@retval nullptr on failure (such as when the same tablespace exists) */ +fil_space_t* +fil_space_create( + const char* name, + space_id_t space_id, + ulint flags, + fil_type_t purpose) +{ + ut_ad(fsp_flags_is_valid(flags)); + ut_ad(srv_page_size == UNIV_PAGE_SIZE_ORIG || flags != 0); - ut_a(it.second); + DBUG_EXECUTE_IF("fil_space_create_failure", return(nullptr);); + + fil_system->mutex_acquire_all(); + + /* Must set back to active before returning from function. */ + clone_mark_abort(true); + + auto shard = fil_system->shard_by_id(space_id); + + auto space = shard->space_create(name, space_id, flags, purpose); + + if (space == nullptr) { + + /* Duplicate error. */ + fil_system->mutex_release_all(); + + clone_mark_active(); + + return(nullptr); } - UT_LIST_ADD_LAST(fil_system->space_list, space); + /* Cache the system tablespaces, avoid looking them up during IO. */ + + if (space->id == TRX_SYS_SPACE) { + + ut_a(fil_space_t::s_sys_space == nullptr + || fil_space_t::s_sys_space == space); + + fil_space_t::s_sys_space = space; - if (!dict_sys_t::is_reserved(id) && id > fil_system->max_assigned_id) { + } else if (space->id == dict_sys_t::s_log_space_first_id) { - fil_system->max_assigned_id = id; + ut_a(fil_space_t::s_redo_space == nullptr + || fil_space_t::s_redo_space == space); + + fil_space_t::s_redo_space = space; } - mutex_exit(&fil_system->mutex); + fil_system->mutex_release_all(); clone_mark_active(); + return(space); } -/*******************************************************************//** -Assigns a new space id for a new single-table tablespace. This works simply by -incrementing the global counter. If 4 billion id's is not enough, we may need -to recycle id's. +/** Assigns a new space id for a new single-table tablespace. This works +simply by incrementing the global counter. If 4 billion id's is not enough, +we may need to recycle id's. +@param[out] space_id Set this to the new tablespace ID @return true if assigned, false if not */ bool -fil_assign_new_space_id( -/*====================*/ - space_id_t* space_id) /*!< in/out: space id */ +Fil_system::assign_new_space_id(space_id_t* space_id) { - space_id_t id; - bool success; - - mutex_enter(&fil_system->mutex); + mutex_acquire_all(); - id = *space_id; + space_id_t id = *space_id; - if (id < fil_system->max_assigned_id) { - id = fil_system->max_assigned_id; + if (id < m_max_assigned_id) { + id = m_max_assigned_id; } - id++; + ++id; space_id_t reserved_space_id = dict_sys_t::s_reserved_space_id; + if (id > (reserved_space_id / 2) && (id % 1000000UL == 0)) { - ib::warn() << "You are running out of new single-table" + + ib::warn() + << "You are running out of new single-table" " tablespace id's. Current counter is " << id << " and it must not exceed " << reserved_space_id << "! To reset the counter to zero you have to dump" @@ -2241,160 +3398,157 @@ fil_assign_new_space_id( " installation."; } - success = !dict_sys_t::is_reserved(id); + bool success = !dict_sys_t::is_reserved(id); if (success) { - *space_id = fil_system->max_assigned_id = id; + + *space_id = m_max_assigned_id = id; + } else { - ib::warn() << "You have run out of single-table tablespace" + + ib::warn() + << "You have run out of single-table tablespace" " id's! Current counter is " << id << ". To reset the counter to zero" " you have to dump all your tables and" " recreate the whole InnoDB installation."; + *space_id = SPACE_UNKNOWN; } - mutex_exit(&fil_system->mutex); + mutex_release_all(); return(success); } -/** Prepares the tablespace file for IO. If the file is closed, it is -opened. Also makes sure that we can open at least one file. -@param[in] space the space object -@return fil_space_t object (this can happen if current space object -is evicted) on success, else nullptr */ -static +/** Assigns a new space id for a new single-table tablespace. This works +simply by incrementing the global counter. If 4 billion id's is not enough, +we may need to recycle id's. +@param[out] space_id Set this to the new tablespace ID +@return true if assigned, false if not */ +bool +fil_assign_new_space_id(space_id_t* space_id) +{ + return(fil_system->assign_new_space_id(space_id)); +} + +/** Opens the files associated with a tablespace and returns a pointer to +the fil_space_t that is in the memory cache associated with a space id. +@param[in] space_id Get the tablespace instance or this ID +@return file_space_t pointer, nullptr if space not found */ fil_space_t* -fil_space_prepare_for_io( - fil_space_t* space) +Fil_shard::space_load(space_id_t space_id) { - fil_node_t* node; + ut_ad(mutex_owned()); + + fil_space_t* space = get_space_by_id(space_id); + + if (space == nullptr || space->size != 0) { + return(space); + } switch (space->purpose) { case FIL_TYPE_LOG: break; + + case FIL_TYPE_IMPORT: case FIL_TYPE_TEMPORARY: case FIL_TYPE_TABLESPACE: - case FIL_TYPE_IMPORT: - ut_a(space->id != 0); - mutex_exit(&fil_system->mutex); + ut_a(space_id != TRX_SYS_SPACE); - /* It is possible that the space gets evicted at this point - before the fil_mutex_enter_and_prepare_for_io() acquires - the fil_system->mutex. Check for this after completing the - call to fil_mutex_enter_and_prepare_for_io(). */ - fil_mutex_enter_and_prepare_for_io(space->id); + mutex_release(); + + auto slot = mutex_acquire_and_get_space(space_id, space); - /* We are still holding the fil_system->mutex. Check if - the space is still in memory cache. */ - space = fil_space_get_by_id(space->id); if (space == nullptr) { + + if (slot) { + release_open_slot(m_id); + } + return(nullptr); } - /* The following code must change when InnoDB supports - multiple datafiles per tablespace. */ - ut_a(1 == UT_LIST_GET_LEN(space->chain)); + ut_a(1 == space->files.size()); - node = UT_LIST_GET_FIRST(space->chain); + { + fil_node_t* file; - /* It must be a single-table tablespace and we have not opened - the file yet; the following calls will open it and update the - size fields */ + file = &space->files.front(); - if (!fil_node_prepare_for_io(node, fil_system, space, false)) { - /* The single-table tablespace can't be opened, - because the ibd file is missing. */ - return(nullptr); - } + /* It must be a single-table tablespace and + we have not opened the file yet; the following + calls will open it and update the size fields */ - fil_node_complete_io(node, fil_system, IORequestRead); - } + bool success = prepare_file_for_io(file, false); - return(space); -} + if (slot) { + release_open_slot(m_id); + } -/*******************************************************************//** -Returns a pointer to the fil_space_t that is in the memory cache -associated with a space id. The caller must lock fil_system->mutex. -@return file_space_t pointer, NULL if space not found */ -UNIV_INLINE -fil_space_t* -fil_space_get_space( -/*================*/ - space_id_t id) /*!< in: space id */ -{ - fil_space_t* space; + if (!success) { - ut_ad(fil_system); + /* The single-table tablespace can't be opened, + because the ibd file is missing. */ - space = fil_space_get_by_id(id); - if (space == NULL || space->size != 0) { - return(space); + return(nullptr); + } + + complete_io(file, IORequestRead); + } } - return(fil_space_prepare_for_io(space)); + return(space); } /** Returns the path from the first fil_node_t found with this space ID. The caller is responsible for freeing the memory allocated here for the value returned. -@param[in] id Tablespace ID -@return own: A copy of fil_node_t::path, NULL if space ID is zero +@param[in] space_id Tablespace ID +@return own: A copy of fil_node_t::path, nullptr if space ID is zero or not found. */ char* -fil_space_get_first_path(space_id_t id) +fil_space_get_first_path(space_id_t space_id) { - fil_space_t* space; - fil_node_t* node; - char* path; - - ut_ad(fil_system); - ut_a(id); + ut_a(space_id != TRX_SYS_SPACE); - fil_mutex_enter_and_prepare_for_io(id); + auto shard = fil_system->shard_by_id(space_id); - space = fil_space_get_space(id); + shard->mutex_acquire(); - if (space == NULL) { - mutex_exit(&fil_system->mutex); - - return(NULL); - } + fil_space_t* space = shard->space_load(space_id); - ut_ad(mutex_own(&fil_system->mutex)); + char* path; - node = UT_LIST_GET_FIRST(space->chain); + if (space != nullptr) { - path = mem_strdup(node->name); + path = mem_strdup(space->files.front().name); + } else { + path = nullptr; + } - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(path); } -/*******************************************************************//** -Returns the size of the space in pages. The tablespace must be cached in the -memory cache. +/** Returns the size of the space in pages. The tablespace must be cached +in the memory cache. +@param[in] space_id Tablespace ID @return space size, 0 if space not found */ page_no_t -fil_space_get_size( -/*===============*/ - space_id_t id) /*!< in: space id */ +fil_space_get_size(space_id_t space_id) { - fil_space_t* space; - page_no_t size; + auto shard = fil_system->shard_by_id(space_id); - ut_ad(fil_system); - mutex_enter(&fil_system->mutex); + shard->mutex_acquire(); - space = fil_space_get_space(id); + fil_space_t* space = shard->space_load(space_id); - size = space ? space->size : 0; + page_no_t size = space ? space->size : 0; - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(size); } @@ -2406,20 +3560,17 @@ in the memory cache. ulint fil_space_get_flags(space_id_t space_id) { - mutex_enter(&fil_system->mutex); + auto shard = fil_system->shard_by_id(space_id); - fil_space_t* space = fil_space_get_space(space_id); - - if (space == nullptr) { + shard->mutex_acquire(); - mutex_exit(&fil_system->mutex); + fil_space_t* space = shard->space_load(space_id); - return(ULINT_UNDEFINED); - } + ulint flags; - ulint flags = space->flags; + flags = (space != nullptr) ? space->flags : ULINT_UNDEFINED; - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(flags); } @@ -2429,70 +3580,64 @@ fil_space_get_flags(space_id_t space_id) @retval true if all file nodes were opened @retval false on failure */ bool -fil_space_open(space_id_t space_id) +Fil_shard::space_open(space_id_t space_id) { - mutex_enter(&fil_system->mutex); + ut_ad(mutex_owned()); - fil_space_t* space = fil_space_get_by_id(space_id); + fil_space_t* space = get_space_by_id(space_id); - for (auto node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { + for (auto& file : space->files) { - if (!node->is_open && !fil_node_open_file(node, false)) { - - mutex_exit(&fil_system->mutex); + if (!file.is_open && !open_file(&file, false)) { return(false); } } - mutex_exit(&fil_system->mutex); - return(true); } +/** Open each file of a tablespace if not already open. +@param[in] space_id tablespace identifier +@retval true if all file nodes were opened +@retval false on failure */ +bool +fil_space_open(space_id_t space_id) +{ + auto shard = fil_system->shard_by_id(space_id); + + shard->mutex_acquire(); + + bool success = shard->space_open(space_id); + + shard->mutex_release(); + + return(success); +} + /** Close each file of a tablespace if open. @param[in] space_id tablespace identifier */ void fil_space_close(space_id_t space_id) { - if (fil_system == NULL) { + if (fil_system == nullptr) { return; } - mutex_enter(&fil_system->mutex); - - fil_space_t* space = fil_space_get_by_id(space_id); - - if (space == NULL) { - mutex_exit(&fil_system->mutex); - return; - } + auto shard = fil_system->shard_by_id(space_id); - for (auto node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { - - if (node->is_open) { - fil_node_close_file(node, false); - } - } - - mutex_exit(&fil_system->mutex); -} + shard->close_file(space_id); +} /** Returns the page size of the space and whether it is compressed or not. The tablespace must be cached in the memory cache. -@param[in] id space id -@param[out] found true if tablespace was found +@param[in] space_id Tablespace ID +@param[out] found true if tablespace was found @return page size */ const page_size_t -fil_space_get_page_size( - space_id_t id, - bool* found) +fil_space_get_page_size(space_id_t space_id, bool* found) { - const ulint flags = fil_space_get_flags(id); + const ulint flags = fil_space_get_flags(space_id); if (flags == ULINT_UNDEFINED) { *found = false; @@ -2504,86 +3649,48 @@ fil_space_get_page_size( return(page_size_t(flags)); } -/****************************************************************//** -Initializes the tablespace memory cache. */ +/** Initializes the tablespace memory cache. +@param[in] max_n_open Maximum number of open files */ void -fil_init( -/*=====*/ - ulint hash_size, /*!< in: hash table size */ - ulint max_n_open) /*!< in: max number of open files */ +fil_init(ulint max_n_open) { - ut_a(fil_system == NULL); - - ut_a(hash_size > 0); - ut_a(max_n_open > 0); - - fil_system = static_cast( - ut_zalloc_nokey(sizeof(*fil_system))); + static_assert( + (1 << UNIV_PAGE_SIZE_SHIFT_MAX) == UNIV_PAGE_SIZE_MAX, + "(1 << UNIV_PAGE_SIZE_SHIFT_MAX) != UNIV_PAGE_SIZE_MAX"); - mutex_create(LATCH_ID_FIL_SYSTEM, &fil_system->mutex); + static_assert( + (1 << UNIV_PAGE_SIZE_SHIFT_MIN) == UNIV_PAGE_SIZE_MIN, + "(1 << UNIV_PAGE_SIZE_SHIFT_MIN) != UNIV_PAGE_SIZE_MIN"); - new(&fil_system->names) Names(); - new(&fil_system->spaces) Spaces(); + ut_a(fil_system == nullptr); - if (fil_scanned != nullptr) { - - new(&fil_system->m_open) Fil_Open(*fil_scanned); - - UT_DELETE(fil_scanned); - - fil_scanned = nullptr; - } else { - new(&fil_system->m_open) Fil_Open(); - } - - UT_LIST_INIT(fil_system->LRU, &fil_node_t::LRU); - UT_LIST_INIT(fil_system->space_list, &fil_space_t::space_list); - UT_LIST_INIT(fil_system->unflushed_spaces, - &fil_space_t::unflushed_spaces); + ut_a(max_n_open > 0); - fil_system->max_n_open = max_n_open; + fil_system = UT_NEW_NOKEY(Fil_system(MAX_SHARDS, max_n_open)); } -/*******************************************************************//** -Opens all log files and system tablespace data files. They stay open until the -database server shutdown. This should be called at a server startup after the -space objects for the log and the system tablespace have been created. The -purpose of this operation is to make sure we never run out of file descriptors -if we need to read from the insert buffer or to write to the log. */ +/** Open all the system files. +@param[in] max_n_open Maximum number of open files allowed +@param[in,out] n_open Current number of open files */ void -fil_open_log_and_system_tablespace_files(void) -/*==========================================*/ +Fil_shard::open_system_tablespaces(size_t max_n_open, size_t* n_open) { - fil_space_t* space; + mutex_acquire(); - mutex_enter(&fil_system->mutex); + for (auto elem : m_spaces) { - for (space = UT_LIST_GET_FIRST(fil_system->space_list); - space != NULL; - space = UT_LIST_GET_NEXT(space_list, space)) { + auto space = elem.second; - fil_node_t* node; - - if (fil_space_belongs_in_lru(space)) { + if (Fil_system::space_belongs_in_LRU(space)) { continue; } - if (space->id == TRX_SYS_SPACE) { - - ut_a(fil_space_t::s_sys_space == nullptr - || fil_space_t::s_sys_space == space); - - fil_space_t::s_sys_space = space; - } - - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { + for (auto& file : space->files) { - if (!node->is_open) { + if (!file.is_open) { - if (!fil_node_open_file(node, false)) { + if (!open_file(&file, false)) { /* This func is called during server's startup. If some file of log or system tablespace is missing, the server @@ -2591,11 +3698,14 @@ fil_open_log_and_system_tablespace_files(void) assert for it. */ ut_a(0); } + + ++*n_open; } - if (fil_system->max_n_open < 10 + fil_system->n_open) { + if (max_n_open < 10 + *n_open) { - ib::warn() << "You must raise the value of" + ib::warn() + << "You must raise the value of" " innodb_open_files in my.cnf!" " Remember that InnoDB keeps all" " log files and all system" @@ -2605,170 +3715,247 @@ fil_open_log_and_system_tablespace_files(void) " some .ibd files if the" " file-per-table storage model is used." " Current open files " - << fil_system->n_open + << *n_open << ", max allowed open files " - << fil_system->max_n_open + << max_n_open << "."; } } } - mutex_exit(&fil_system->mutex); + mutex_release(); } -/*******************************************************************//** -Closes all open files. There must not be any pending i/o's or not flushed -modifications in the files. */ +/** Opens all log files and system tablespace data files in all shards. */ void -fil_close_all_files(void) -/*=====================*/ +Fil_system::open_all_system_tablespaces() { - fil_space_t* space; + size_t n_open = 0; + + for (auto shard : m_shards) { + + shard->open_system_tablespaces(m_max_n_open, &n_open); + } +} + +/** Opens all log files and system tablespace data files. They stay open +until the database server shutdown. This should be called at a server +startup after the space objects for the log and the system tablespace +have been created. The purpose of this operation is to make sure we +never run out of file descriptors if we need to read from the insert +buffer or to write to the log. */ +void +fil_open_log_and_system_tablespace_files() +{ + fil_system->open_all_system_tablespaces(); +} + +/** Close all open files. */ +void +Fil_shard::close_all_files() +{ + ut_ad(mutex_owned()); - mutex_enter(&fil_system->mutex); + auto end = m_spaces.end(); - for (space = UT_LIST_GET_FIRST(fil_system->space_list); - space != NULL; ) { - fil_node_t* node; - fil_space_t* prev_space = space; + for (auto it = m_spaces.begin(); it != end; it = m_spaces.erase(it)) { - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { + auto space = it->second; - if (node->is_open) { - fil_node_close_file(node, false); + ut_a(space->id == TRX_SYS_SPACE + || space->purpose == FIL_TYPE_TEMPORARY + || space->id == dict_sys_t::s_log_space_first_id + || space->files.size() == 1); + + for (auto& file: space->files) { + + if (file.is_open) { + close_file(&file, false); } } - space = UT_LIST_GET_NEXT(space_list, space); - fil_space_detach(prev_space); - fil_space_free_low(prev_space); + space_detach(space); + + space_free_low(space); + + ut_a(space == nullptr); + } +} + +/** Close all open files. */ +void +Fil_system::close_all_files() +{ + for (auto shard : m_shards) { + + shard->mutex_acquire(); + + shard->close_all_files(); + + shard->mutex_release(); } +} - mutex_exit(&fil_system->mutex); +/** Closes all open files. There must not be any pending i/o's or not flushed +modifications in the files. */ +void +fil_close_all_files() +{ + fil_system->close_all_files(); } -/*******************************************************************//** -Closes the redo log files. There must not be any pending i/o's or not -flushed modifications in the files. */ +/** Close log files. +@param[in] free_all If set then free all instances */ void -fil_close_log_files( -/*================*/ - bool free) /*!< in: whether to free the memory object */ +Fil_shard::close_log_files(bool free_all) { - fil_space_t* space; + mutex_acquire(); - mutex_enter(&fil_system->mutex); + auto end = m_spaces.end(); - space = UT_LIST_GET_FIRST(fil_system->space_list); + for (auto it = m_spaces.begin(); it != end; /* No op */) { - while (space != NULL) { - fil_node_t* node; - fil_space_t* prev_space = space; + auto space = it->second; if (space->purpose != FIL_TYPE_LOG) { - space = UT_LIST_GET_NEXT(space_list, space); + ++it; continue; } + if (space->id == dict_sys_t::s_log_space_first_id) { + + ut_a(fil_space_t::s_redo_space == space); + + fil_space_t::s_redo_space = nullptr; + } + ut_ad(space->max_lsn == 0); - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { + for (auto& file : space->files) { - if (node->is_open) { - fil_node_close_file(node, false); + if (file.is_open) { + close_file(&file, false); } } - space = UT_LIST_GET_NEXT(space_list, space); + if (free_all) { + + space_detach(space); + space_free_low(space); + ut_a(space == nullptr); + + it = m_spaces.erase(it); - if (free) { - fil_space_detach(prev_space); - fil_space_free_low(prev_space); + } else { + ++it; } } - mutex_exit(&fil_system->mutex); + mutex_release(); +} + +/** Close all log files in all shards. +@param[in] free_all If set then free all instances */ +void +Fil_system::close_all_log_files(bool free_all) +{ + for (auto shard : m_shards) { + + shard->close_log_files(free_all); + } +} + +/** Closes the redo log files. There must not be any pending i/o's or not +flushed modifications in the files. +@param[in] free_all If set then free all instances */ +void +fil_close_log_files(bool free_all) +{ + fil_system->close_all_log_files(free_all); } /** Iterate through all persistent tablespace files (FIL_TYPE_TABLESPACE) returning the nodes via callback function cbk. -@param[in] include_log include log files -@param[in] context callback function context -@param[in] callback callback function +@param[in] include_log Include log files, if true +@param[in] f Callback @return any error returned by the callback function. */ dberr_t -fil_iterate_tablespace_files( - bool include_log, - void* context, - fil_node_cbk_t* callback) +Fil_shard::iterate(bool include_log, Fil_iterator::Function& f) { - fil_space_t* space; - dberr_t err = DB_SUCCESS; - - mutex_enter(&fil_system->mutex); - - space = UT_LIST_GET_FIRST(fil_system->space_list); + mutex_acquire(); - while (space != nullptr) { - fil_node_t* node; + for (auto& elem : m_spaces) { - if (space->purpose != FIL_TYPE_TABLESPACE) { + auto space = elem.second; - if (!include_log || space->purpose != FIL_TYPE_LOG) { + if (space->purpose != FIL_TYPE_TABLESPACE + && (!include_log || space->purpose != FIL_TYPE_LOG)) { - space = UT_LIST_GET_NEXT(space_list, space); - continue; - } + continue; } - for (node = UT_LIST_GET_FIRST(space->chain); - node != nullptr; - node = UT_LIST_GET_NEXT(chain, node)) { + for (auto& file: space->files) { + + /* Note: The callback can release the mutex. */ - err = callback(node, context); + dberr_t err = f(&file); if (err != DB_SUCCESS) { - break; + mutex_release(); + + return(err);; } } + } - if (err != DB_SUCCESS) { + mutex_release(); - break; - } + return(DB_SUCCESS); +} - space = UT_LIST_GET_NEXT(space_list, space); +/** Iterate through all persistent tablespace files (FIL_TYPE_TABLESPACE) +returning the nodes via callback function cbk. +@param[in] include_log include log files, if true +@param[in] f callback function +@return any error returned by the callback function. */ +dberr_t +Fil_system::iterate(bool include_log, Fil_iterator::Function& f) +{ + for (auto shard : m_shards) { + + dberr_t err = shard->iterate(include_log, f); + + if (err != DB_SUCCESS) { + return(err); + } } - mutex_exit(&fil_system->mutex); - return(err); + return(DB_SUCCESS); +} + +/** Iterate through all persistent tablespace files (FIL_TYPE_TABLESPACE) +returning the nodes via callback function cbk. +@param[in] include_log include log files, if true +@param[in] f Callback +@return any error returned by the callback function. */ +dberr_t +Fil_iterator::iterate(bool include_log, Function&& f) +{ + return(fil_system->iterate(include_log, f)); } -/*******************************************************************//** -Sets the max tablespace id counter if the given number is bigger than the -previous value. */ +/** Sets the max tablespace id counter if the given number is bigger than the +previous value. +@param[in] max_id Maximum known tablespace ID */ void -fil_set_max_space_id_if_bigger( -/*===========================*/ - space_id_t max_id) /*!< in: maximum known id */ +fil_set_max_space_id_if_bigger(space_id_t max_id) { if (dict_sys_t::is_reserved(max_id)) { ib::fatal() << "Max tablespace id is too high, " << max_id; } - mutex_enter(&fil_system->mutex); - - if (fil_system->max_assigned_id < max_id) { - - fil_system->max_assigned_id = max_id; - } - - mutex_exit(&fil_system->mutex); + fil_system->update_maximum_space_id(max_id); } /** Write the flushed LSN to the page header of the first page in the @@ -2797,7 +3984,7 @@ fil_write_flushed_lsn( err = fil_write(page_id, univ_page_size, 0, univ_page_size.physical(), buf); - fil_flush_file_spaces(to_int(FIL_TYPE_TABLESPACE)); + fil_system->flush_file_spaces(to_int(FIL_TYPE_TABLESPACE)); } ut_free(buf1); @@ -2808,33 +3995,32 @@ fil_write_flushed_lsn( /** Acquire a tablespace when it could be dropped concurrently. Used by background threads that do not necessarily hold proper locks for concurrency control. -@param[in] id tablespace ID -@param[in] silent whether to silently ignore missing tablespaces -@return the tablespace, or NULL if missing or being deleted */ +@param[in] space_id Tablespace ID +@param[in] silent Whether to silently ignore missing tablespaces +@return the tablespace, or nullptr if missing or being deleted */ inline fil_space_t* -fil_space_acquire_low( - space_id_t id, - bool silent) +fil_space_acquire_low(space_id_t space_id, bool silent) { - fil_space_t* space; + auto shard = fil_system->shard_by_id(space_id); - mutex_enter(&fil_system->mutex); + shard->mutex_acquire(); - space = fil_space_get_by_id(id); + fil_space_t* space = shard->get_space_by_id(space_id); - if (space == NULL) { + if (space == nullptr) { if (!silent) { - ib::warn() << "Trying to access missing" - " tablespace " << id; + ib::warn() + << "Trying to access missing tablespace " + << space_id; } } else if (space->stop_new_ops) { - space = NULL; + space = nullptr; } else { - space->n_pending_ops++; + ++space->n_pending_ops; } - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(space); } @@ -2842,509 +4028,186 @@ fil_space_acquire_low( /** Acquire a tablespace when it could be dropped concurrently. Used by background threads that do not necessarily hold proper locks for concurrency control. -@param[in] id tablespace ID -@return the tablespace, or NULL if missing or being deleted */ +@param[in] space_id Tablespace ID +@return the tablespace, or nullptr if missing or being deleted */ fil_space_t* -fil_space_acquire(space_id_t id) +fil_space_acquire(space_id_t space_id) { - return(fil_space_acquire_low(id, false)); + return(fil_space_acquire_low(space_id, false)); } /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. -@param[in] id tablespace ID -@return the tablespace, or NULL if missing or being deleted */ +@param[in] space_id Tablespace ID +@return the tablespace, or nullptr if missing or being deleted */ fil_space_t* -fil_space_acquire_silent(space_id_t id) +fil_space_acquire_silent(space_id_t space_id) { - return(fil_space_acquire_low(id, true)); + return(fil_space_acquire_low(space_id, true)); } /** Release a tablespace acquired with fil_space_acquire(). @param[in,out] space tablespace to release */ void -fil_space_release( - fil_space_t* space) +fil_space_release(fil_space_t* space) { - mutex_enter(&fil_system->mutex); + auto shard = fil_system->shard_by_id(space->id); + + shard->mutex_acquire(); + ut_ad(space->magic_n == FIL_SPACE_MAGIC_N); ut_ad(space->n_pending_ops > 0); - space->n_pending_ops--; - mutex_exit(&fil_system->mutex); -} -#ifndef UNIV_HOTBACKUP -/** Write a log record about an operation on a tablespace file. -@param[in] type MLOG_FILE_OPEN or MLOG_FILE_DELETE - or MLOG_FILE_CREATE2 or MLOG_FILE_RENAME2 -@param[in] space_id tablespace identifier -@param[in] first_page_no first page number in the file -@param[in] path file path -@param[in] new_path if type is MLOG_FILE_RENAME2, the new name -@param[in] flags if type is MLOG_FILE_CREATE2, the space flags -@param[in,out] mtr mini-transaction */ -static -void -fil_op_write_log( - mlog_id_t type, - space_id_t space_id, - page_no_t first_page_no, - const char* path, - const char* new_path, - ulint flags, - mtr_t* mtr) -{ - byte* log_ptr; - ulint len; + --space->n_pending_ops; - /* TODO: support user-created multi-file tablespaces */ - ut_ad(first_page_no == 0 || space_id == TRX_SYS_SPACE); + shard->mutex_release(); +} - log_ptr = mlog_open(mtr, 11 + 4 + 2 + 1); +/** Check for pending operations. +@param[in] space tablespace +@param[in] count number of attempts so far +@return 0 if no pending operations else count + 1. */ +ulint +Fil_shard::space_check_pending_operations(fil_space_t* space, ulint count) const +{ + ut_ad(mutex_owned()); - if (log_ptr == NULL) { - /* Logging in mtr is switched off during crash recovery: - in that case mlog_open returns NULL */ - return; - } + if (space != nullptr && space->n_pending_ops > 0) { - log_ptr = mlog_write_initial_log_record_low( - type, space_id, first_page_no, log_ptr, mtr); + if (count > 5000) { + ib::warn() + << "Trying to close/delete" + " tablespace '" << space->name + << "' but there are " << space->n_pending_ops + << " pending operations on it."; + } - if (type == MLOG_FILE_CREATE2) { - mach_write_to_4(log_ptr, flags); - log_ptr += 4; + return(count + 1); } - /* Let us store the strings as null-terminated for easier readability - and handling */ - - len = strlen(path) + 1; - - mach_write_to_2(log_ptr, len); - log_ptr += 2; - mlog_close(mtr, log_ptr); - - mlog_catenate_string( - mtr, reinterpret_cast(path), len); - - switch (type) { - case MLOG_FILE_RENAME2: + return(0); +} - ut_ad(strchr(new_path, OS_PATH_SEPARATOR) != NULL); +/** Check for pending IO. +@param[in] operation File operation +@param[in,out] space Tablespace to check +@param[out] file File in space list +@param[in] count number of attempts so far +@return 0 if no pending else count + 1. */ +ulint +Fil_shard::check_pending_io( + fil_operation_t operation, + fil_space_t* space, + fil_node_t** file, + ulint count) const +{ + ut_ad(mutex_owned()); + ut_a(space->n_pending_ops == 0); - len = strlen(new_path) + 1; + switch (operation) { + case FIL_OPERATION_CLOSE: + case FIL_OPERATION_DELETE: + break; + } - log_ptr = mlog_open(mtr, 2 + len); + ut_a(space->id == TRX_SYS_SPACE + || space->purpose == FIL_TYPE_TEMPORARY + || space->id == dict_sys_t::s_log_space_first_id + || space->files.size() == 1); - mach_write_to_2(log_ptr, len); + *file = &space->files.front(); - log_ptr += 2; + if (space->n_pending_flushes > 0 || (*file)->n_pending > 0) { - mlog_close(mtr, log_ptr); + ut_a((*file)->in_use == 0); - mlog_catenate_string( - mtr, reinterpret_cast(new_path), len); - break; + if (count > 1000) { + ib::warn() << "Trying to delete/close" + " tablespace '" << space->name + << "' but there are " + << space->n_pending_flushes + << " flushes and " << (*file)->n_pending + << " pending I/O's on it."; + } - case MLOG_FILE_OPEN: - case MLOG_FILE_DELETE: - case MLOG_FILE_CREATE2: - break; - default: - ut_ad(0); + return(count + 1); } + + return(0); } -/** Write redo log for renaming a file. -@param[in] space_id tablespace id -@param[in] first_page_no first page number in the file -@param[in] old_name tablespace file name -@param[in] new_name tablespace file name after renaming -@param[in,out] mtr mini-transaction */ -static -void -fil_name_write_rename( +/** Check pending operations on a tablespace. +@param[in] space_id Tablespace ID +@param[in] operation File operation +@param[out] space tablespace instance in memory +@param[out] path tablespace path +@return DB_SUCCESS or DB_TABLESPACE_NOT_FOUND. */ +dberr_t +Fil_shard::space_check_pending_operations( space_id_t space_id, - page_no_t first_page_no, - const char* old_name, - const char* new_name, - mtr_t* mtr) + fil_operation_t operation, + fil_space_t** space, + char** path) const { ut_ad(!fsp_is_system_or_temp_tablespace(space_id)); - ut_ad(!fsp_is_undo_tablespace(space_id)); - /* We write out the mapping file before we write the redo log - record because a checkpoint can take place asynchronously at - any time. */ + *space = nullptr; - fil_system->m_open.enter(); - fil_system->m_open.log(space_id, new_name); - fil_system->m_open.to_file(); - fil_system->m_open.exit(); + mutex_acquire(); - /* Note: A checkpoint can take place here. */ - - fil_op_write_log( - MLOG_FILE_RENAME2, - space_id, first_page_no, old_name, new_name, 0, mtr); + fil_space_t* sp = get_space_by_id(space_id); - /* Note: A checkpoint can take place here too before we - have physically renamed the file. */ -} -#endif /* !UNIV_HOTBACKUP */ + if (sp != nullptr) { + sp->stop_new_ops = true; + } -/** Check whether we can rename the file -@param[in] space Tablespace for which to rename -@param[in] name Source file name -@param[in] file Target file that exists on disk -@return DB_SUCCESS if all OK */ -static -dberr_t -fil_rename_validate(fil_space_t* space, const char* name, Datafile& file) -{ - dberr_t err = file.validate_for_recovery(space->id); + mutex_release(); - if (err == DB_TABLESPACE_NOT_FOUND) { + /* Check for pending operations. */ - /* Tablespace header doesn't contain the expected - tablespace ID. */ + ulint count = 0; - return(DB_ERROR); + do { + mutex_acquire(); - } else if (err != DB_SUCCESS) { + sp = get_space_by_id(space_id); - ib::warn() - << "Failed to read the first page of the" - << " file '" << file.filepath() << "'." - << " You will need to verify and move the" - << " file out of the way retry recovery."; + count = space_check_pending_operations(sp, count); - return(DB_ERROR); - } + mutex_release(); - const auto node = UT_LIST_GET_FIRST(space->chain); + if (count > 0) { + os_thread_sleep(20000); + } - if (strcmp(file.filepath(), node->name) == 0) { + } while (count > 0); - /* Check if already points to the correct file. - Must have the same space ID */ + /* Check for pending IO. */ - ib::info() - << "Tablespace ID already maps to: '" - << file.filepath() << "', rename ignored."; + *path = 0; - ut_a(file.space_id() == space->id); + do { + mutex_acquire(); - return(DB_SUCCESS); + sp = get_space_by_id(space_id); - } else if (file.space_id() != space->id) { + if (sp == nullptr) { - /* Target file exists on disk but has a different - tablespce ID. The user should manually delete it. */ + mutex_release(); - ib::error() - << "Cannot rename '" - << name << "' to '" - << file.filepath() << "'. File '" << file.filepath() - << "' tablespace ID " << file.space_id() - << " doesn't match the expected tablespace" - << " ID " << space->id - << ". You will need to verify and move '" - << file.filepath() << "' manually and retry recovery!"; - - return(DB_ERROR); - } - - /* Target file exists on disk and has the same ID. */ - - ib::error() - << "Cannot rename '" << name << "' to '" << file.filepath() - << "'. The File '" << file.filepath() << " already exists on" - << " disk. You will need to verify and move either file" - << " manually and retry recovery!"; - - return(DB_ERROR); -} - -/** Replay a file rename operation if possible. -@param[in] page_id Space ID and first page number in the file -@param[in] name old file name -@param[in] new_name new file name -@return whether the operation was successfully applied -(the name did not exist, or new_name did not exist and -name was successfully renamed to new_name) */ -bool -fil_op_replay_rename( - const page_id_t& page_id, - const char* name, - const char* new_name) -{ -#ifdef UNIV_HOTBACKUP - ut_ad(meb_replay_file_ops); -#endif /* UNIV_HOTBACKUP */ - - ut_ad(page_id.page_no() == 0); - ut_ad(strcmp(name, new_name) != 0); - ut_ad(page_id.space() != TRX_SYS_SPACE); - - /* In order to replay the rename, the following must hold: - * The new name is not already used. - * A tablespace exists with the old name. - * The space ID for that tablepace matches this log entry. - This will prevent unintended renames during recovery. */ - - space_id_t space_id = page_id.space(); - fil_space_t* space = fil_space_get(space_id); - - if (space == nullptr) { - return(true); - } - - Datafile file; - - file.set_filepath(new_name); - - if (file.open_read_only(false) == DB_SUCCESS) { - - dberr_t err = fil_rename_validate(space, name, file); - - file.close(); - - return(err == DB_SUCCESS); - } - - /* Create the database directory for the new name, if - it does not exist yet */ - - const char* namend = strrchr(new_name, OS_PATH_SEPARATOR); - ut_a(namend != nullptr); - - char* dir = static_cast( - ut_malloc_nokey(namend - new_name + 1)); - - memcpy(dir, new_name, namend - new_name); - dir[namend - new_name] = '\0'; - - bool success = os_file_create_directory(dir, false); - ut_a(success); - - ulint dirlen = 0; - - if (const char* dirend = strrchr(dir, OS_PATH_SEPARATOR)) { - - dirlen = dirend - dir + 1; - } - - ut_free(dir); - - char* new_table = mem_strdupl( - new_name + dirlen, strlen(new_name + dirlen) - - 4 /* remove ".ibd" */); - - ut_ad(new_table[namend - new_name - dirlen] == OS_PATH_SEPARATOR); - -#if OS_PATH_SEPARATOR != '/' - new_table[namend - new_name - dirlen] = '/'; -#endif /* OS_PATH_SEPARATOR != '/' */ - - clone_mark_abort(true); - if (!fil_rename_tablespace(space_id, name, new_table, new_name)) { - - ut_error; - } - clone_mark_active(); - - ut_free(new_table); - return(true); -} - -#ifndef UNIV_HOTBACKUP -/** Replay a file rename operation for ddl replay. -@param[in] page_id Space ID and first page number in the file -@param[in] name old file name -@param[in] new_name new file name -@return whether the operation was successfully applied -(the name did not exist, or new_name did not exist and -name was successfully renamed to new_name) */ -bool -fil_op_replay_rename_for_ddl( - const page_id_t& page_id, - const char* name, - const char* new_name) -{ - space_id_t space_id = page_id.space(); - - fil_space_t* space = fil_space_get(space_id); - - if (space == nullptr && Log_DDL::is_in_recovery()) { - /* If can't find the space, try to load it by - the information in tablespace.open.*. */ - fil_system->m_open.open_for_recovery(space_id); - if (!fil_space_get(space_id)) { - ib::info() - << "Can not find space with space ID " - << space_id << " when replaying ddl log " - << "rename from '" << name - << "' to '" << new_name << "'"; - return(true); - } - } - - return(fil_op_replay_rename(page_id, name, new_name)); -} -#endif /* !UNIV_HOTBACKUP */ - -/** File operations for tablespace */ -enum fil_operation_t { - FIL_OPERATION_DELETE, /*!< delete a single-table tablespace */ - FIL_OPERATION_CLOSE /*!< close a single-table tablespace */ -}; - -/** Check for pending operations. -@param[in] space tablespace -@param[in] count number of attempts so far -@return 0 if no operations else count + 1. */ -static -ulint -fil_check_pending_ops( - fil_space_t* space, - ulint count) -{ - ut_ad(mutex_own(&fil_system->mutex)); - - const ulint n_pending_ops = space ? space->n_pending_ops : 0; - - if (n_pending_ops) { - - if (count > 5000) { - ib::warn() - << "Trying to close/delete" - " tablespace '" << space->name - << "' but there are " << n_pending_ops - << " pending operations on it."; - } - - return(count + 1); - } - - return(0); -} - -/*******************************************************************//** -Check for pending IO. -@return 0 if no pending else count + 1. */ -static -ulint -fil_check_pending_io( -/*=================*/ - fil_operation_t operation, /*!< in: File operation */ - fil_space_t* space, /*!< in/out: Tablespace to check */ - fil_node_t** node, /*!< out: Node in space list */ - ulint count) /*!< in: number of attempts so far */ -{ - ut_ad(mutex_own(&fil_system->mutex)); - ut_a(space->n_pending_ops == 0); - - switch (operation) { - case FIL_OPERATION_DELETE: - case FIL_OPERATION_CLOSE: - break; - } - - /* The following code must change when InnoDB supports - multiple datafiles per tablespace. */ - ut_a(UT_LIST_GET_LEN(space->chain) == 1); - - *node = UT_LIST_GET_FIRST(space->chain); - - if (space->n_pending_flushes > 0 || (*node)->n_pending > 0) { - - ut_a((*node)->in_use == 0); - - if (count > 1000) { - ib::warn() << "Trying to delete/close" - " tablespace '" << space->name - << "' but there are " - << space->n_pending_flushes - << " flushes and " << (*node)->n_pending - << " pending i/o's on it."; - } - - return(count + 1); - } - - return(0); -} - -/*******************************************************************//** -Check pending operations on a tablespace. -@return DB_SUCCESS or error failure. */ -static -dberr_t -fil_check_pending_operations( -/*=========================*/ - space_id_t id, /*!< in: space id */ - fil_operation_t operation, /*!< in: File operation */ - fil_space_t** space, /*!< out: tablespace instance - in memory */ - char** path) /*!< out/own: tablespace path */ -{ - ulint count = 0; - - ut_ad(!fsp_is_system_or_temp_tablespace(id)); - ut_ad(space); - - *space = 0; - - mutex_enter(&fil_system->mutex); - fil_space_t* sp = fil_space_get_by_id(id); - if (sp) { - sp->stop_new_ops = true; - } - mutex_exit(&fil_system->mutex); - - /* Check for pending operations. */ - - do { - mutex_enter(&fil_system->mutex); - - sp = fil_space_get_by_id(id); - - count = fil_check_pending_ops(sp, count); - - mutex_exit(&fil_system->mutex); - - if (count > 0) { - os_thread_sleep(20000); - } - - } while (count > 0); - - /* Check for pending IO. */ - - *path = 0; - - do { - mutex_enter(&fil_system->mutex); - - sp = fil_space_get_by_id(id); - - if (sp == NULL) { - mutex_exit(&fil_system->mutex); return(DB_TABLESPACE_NOT_FOUND); } - fil_node_t* node; + fil_node_t* file; - count = fil_check_pending_io(operation, sp, &node, count); + count = check_pending_io(operation, sp, &file, count); if (count == 0) { - *path = mem_strdup(node->name); + *path = mem_strdup(file->name); } - mutex_exit(&fil_system->mutex); + mutex_release(); if (count > 0) { os_thread_sleep(20000); @@ -3352,102 +4215,88 @@ fil_check_pending_operations( } while (count > 0); - ut_ad(sp); + ut_ad(sp != nullptr); *space = sp; + return(DB_SUCCESS); } -/* Convert the paths into absolute paths and compare them. -@param[in] lhs Filename to compare -@param[in] rhs Filename to compare -@return true if they are the same */ -bool -fil_paths_equal(const char* lhs, const char* rhs) +/** Get the real path for a directory or a file name, useful for comparing +symlinked files. If path doesn't exist it will be ignored. +@param[in] path Directory or filename +@return the absolute path of path, or "" on error. */ +std::string +Fil_path::get_real_path(const std::string& path) { - std::string abs_path1(lhs); + char abspath[FN_REFLEN + 2]; - /* Convert to an absolute path */ - if (*lhs == '.') { - const char* ptr = lhs; + /* FIXME: This should be an assertion eventually. */ + if (path.empty()) { + return(path); + } - while (*ptr == '.' || *ptr == OS_PATH_SEPARATOR) { - ++ptr; - } + int ret = my_realpath(abspath, path.c_str(), MYF(0)); - abs_path1 = Fil_Open::get_path(".", ptr); - } + if (ret == -1) { - /* Remove adjacent duplicate characters, comparison should not be - affected if the strings are identical. It will remove '//' */ + ib::info() << "my_realpath(" << path << ") failed!"; - abs_path1.erase( - std::unique( - abs_path1.begin(), abs_path1.end()), abs_path1.end()); + return(path); + } - std::string abs_path2(rhs); + std::string real_path(abspath); - if (*abs_path2.begin() == '.') { - const char* ptr = rhs; + /* On Windows, my_realpath() puts a '\' at the end of any directory + path, on non-Windows it does not. */ - while (*ptr == '.' || *ptr == OS_PATH_SEPARATOR) { - ++ptr; - } + if (!is_separator(real_path.back()) + && get_file_type(real_path) == OS_FILE_TYPE_DIR) { - abs_path2 = Fil_Open::get_path(".", ptr); + real_path.push_back(OS_SEPARATOR); } - /* Remove adjacent duplicate characters, comparison should not be - affected if the strings are identical. It will remove '//' */ - - abs_path2.erase( - std::unique( - abs_path2.begin(), abs_path2.end()), abs_path2.end()); + ut_a(real_path.length() < sizeof(abspath)); - return(abs_path1.compare(abs_path2) == 0); + return(real_path); } -/** Fetch the file name opened for a space_id during recovery -from the file map. -@param[in] space_id undo tablespace id -@return file name that was opened */ -std::string -fil_system_open_fetch(space_id_t space_id) +/** Constructor +@param[in] dir Directory that the files are under */ +Tablespace_files::Tablespace_files(const std::string& dir) + : + m_ibd_paths(), + m_undo_paths(), + m_dir(dir) { - fil_system->m_open.enter(); - std::string name = fil_system->m_open.fetch(space_id); - fil_system->m_open.exit(); - - return(name); + ut_ad(Fil_path::is_separator(dir.back())); } -/*******************************************************************//** -Closes a single-table tablespace. The tablespace must be cached in the +/** Closes a single-table tablespace. The tablespace must be cached in the memory cache. Free all pages used by the tablespace. +@param[in,out] trx Transaction covering the close +@param[in] space_id Tablespace ID @return DB_SUCCESS or error */ dberr_t -fil_close_tablespace( -/*=================*/ - trx_t* trx, /*!< in/out: Transaction covering the close */ - space_id_t id) /*!< in: space id */ +fil_close_tablespace(trx_t* trx, space_id_t space_id) { - char* path = 0; - fil_space_t* space = 0; - dberr_t err; + char* path = nullptr; + fil_space_t* space = nullptr; - ut_ad(!fsp_is_system_or_temp_tablespace(id)); - ut_ad(!fsp_is_undo_tablespace(id)); + ut_ad(!fsp_is_undo_tablespace(space_id)); + ut_ad(!fsp_is_system_or_temp_tablespace(space_id)); - err = fil_check_pending_operations(id, FIL_OPERATION_CLOSE, - &space, &path); + auto shard = fil_system->shard_by_id(space_id); + + dberr_t err = shard->space_check_pending_operations( + space_id, FIL_OPERATION_CLOSE, &space, &path); if (err != DB_SUCCESS) { return(err); } - ut_a(space); - ut_a(path != 0); + ut_a(path != nullptr); rw_lock_x_lock(&space->latch); @@ -3459,13 +4308,13 @@ fil_close_tablespace( completely and permanently. The flag stop_new_ops also prevents fil_flush() from being applied to this tablespace. */ - buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_FLUSH_WRITE, trx); + buf_LRU_flush_or_remove_pages(space_id, BUF_REMOVE_FLUSH_WRITE, trx); #endif /* !UNIV_HOTBACKUP */ /* If the free is successful, the X lock will be released before the space memory data structure is freed. */ - if (!fil_space_free(id, true)) { + if (!fil_space_free(space_id, true)) { rw_lock_x_unlock(&space->latch); err = DB_TABLESPACE_NOT_FOUND; } else { @@ -3475,15 +4324,23 @@ fil_close_tablespace( /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ - char* cfg_name = fil_make_filepath(path, NULL, CFG, false); - if (cfg_name != NULL) { - os_file_delete_if_exists(innodb_data_file_key, cfg_name, NULL); + char* cfg_name = Fil_path::make_cfg(path); + + if (cfg_name != nullptr) { + + os_file_delete_if_exists( + innodb_data_file_key, cfg_name, nullptr); + ut_free(cfg_name); } - char* cfp_name = fil_make_filepath(path, NULL, CFP, false); - if (cfp_name != NULL) { - os_file_delete_if_exists(innodb_data_file_key, cfp_name, NULL); + char* cfp_name = Fil_path::make_cfp(path); + + if (cfp_name != nullptr) { + + os_file_delete_if_exists( + innodb_data_file_key, cfp_name, nullptr); + ut_free(cfp_name); } @@ -3492,43 +4349,138 @@ fil_close_tablespace( return(err); } -/** Deletes an IBD tablespace, either general or single-table. -The tablespace must be cached in the memory cache. This will delete the -datafile, fil_space_t & fil_node_t entries from the file_system_t cache. -@param[in] id Tablespace id -@param[in] buf_remove Specify the action to take on the pages -for this table in the buffer pool. -@return DB_SUCCESS or error */ -dberr_t -fil_delete_tablespace( - space_id_t id, - buf_remove_t buf_remove) +#ifndef UNIV_HOTBACKUP +/** Write a log record about an operation on a tablespace file. +@param[in] type MLOG_FILE_OPEN or MLOG_FILE_DELETE + or MLOG_FILE_CREATE or MLOG_FILE_RENAME +@param[in] space_id tablespace identifier +@param[in] path file path +@param[in] new_path if type is MLOG_FILE_RENAME, the new name +@param[in] flags if type is MLOG_FILE_CREATE, the space flags +@param[in,out] mtr mini-transaction */ +static +void +fil_op_write_log( + mlog_id_t type, + space_id_t space_id, + const char* path, + const char* new_path, + ulint flags, + mtr_t* mtr) { - char* path = 0; - fil_space_t* space = 0; + ut_ad(space_id != TRX_SYS_SPACE); - ut_ad(!fsp_is_system_or_temp_tablespace(id)); - ut_ad(!fsp_is_undo_tablespace(id)); + byte* log_ptr; - dberr_t err = fil_check_pending_operations( - id, FIL_OPERATION_DELETE, &space, &path); + log_ptr = mlog_open(mtr, 11 + 4 + 2 + 1); - if (err != DB_SUCCESS) { + if (log_ptr == nullptr) { + /* Logging in mtr is switched off during crash recovery: + in that case mlog_open returns nullptr */ + return; + } - ib::error() << "Cannot delete tablespace " << id - << " because it is not found in the tablespace" - " memory cache."; + log_ptr = mlog_write_initial_log_record_low( + type, space_id, 0, log_ptr, mtr); - return(err); + if (type == MLOG_FILE_CREATE) { + mach_write_to_4(log_ptr, flags); + log_ptr += 4; } - ut_a(space); - ut_a(path != 0); + /* Let us store the strings as null-terminated for easier readability + and handling */ -#ifndef UNIV_HOTBACKUP - /* IMPORTANT: Because we have set space::stop_new_ops there - can't be any new ibuf merges, reads or flushes. We are here - because node::n_pending was zero above. However, it is still + ulint len = strlen(path) + 1; + + mach_write_to_2(log_ptr, len); + log_ptr += 2; + + mlog_close(mtr, log_ptr); + + mlog_catenate_string(mtr, reinterpret_cast(path), len); + + switch (type) { + case MLOG_FILE_RENAME: + + ut_ad(strchr(new_path, Fil_path::OS_SEPARATOR) != nullptr); + + len = strlen(new_path) + 1; + + log_ptr = mlog_open(mtr, 2 + len); + + mach_write_to_2(log_ptr, len); + + log_ptr += 2; + + mlog_close(mtr, log_ptr); + + mlog_catenate_string( + mtr, reinterpret_cast(new_path), len); + break; + case MLOG_FILE_DELETE: + case MLOG_FILE_CREATE: + break; + default: + ut_ad(0); + } +} + +/** Fetch the file name opened for a space_id during recovery +from the file map. +@param[in] space_id Undo tablespace ID +@return file name that was opened, empty string if space ID not found. */ +std::string +fil_system_open_fetch(space_id_t space_id) +{ + ut_a(dict_sys_t::is_reserved(space_id) + || srv_is_upgrade_mode); + + return(fil_system->find(space_id)); +} + +#endif /* !UNIV_HOTBACKUP */ + +/** Deletes an IBD tablespace, either general or single-table. +The tablespace must be cached in the memory cache. This will delete the +datafile, fil_space_t & fil_node_t entries from the file_system_t cache. +@param[in] space_id Tablespace ID +@param[in] buf_remove Specify the action to take on the pages + for this table in the buffer pool. +@return DB_SUCCESS, DB_TABLESPCE_NOT_FOUND or DB_IO_ERROR */ +dberr_t +Fil_shard::space_delete( + space_id_t space_id, + buf_remove_t buf_remove) +{ + char* path = nullptr; + fil_space_t* space = nullptr; + + ut_ad(!fsp_is_system_or_temp_tablespace(space_id)); + ut_ad(!fsp_is_undo_tablespace(space_id)); + + dberr_t err = space_check_pending_operations( + space_id, FIL_OPERATION_DELETE, &space, &path); + + if (err != DB_SUCCESS) { + + ut_a(err == DB_TABLESPACE_NOT_FOUND); + + ib::error() + << "Cannot delete tablespace " << space_id + << " because it is not found in the tablespace" + " memory cache."; + + return(err); + } + + ut_a(path != nullptr); + ut_a(space != nullptr); + +#ifndef UNIV_HOTBACKUP + /* IMPORTANT: Because we have set space::stop_new_ops there + can't be any new ibuf merges, reads or flushes. We are here + because file::n_pending was zero above. However, it is still possible to have pending read and write requests: A read request can happen because the reader thread has @@ -3547,7 +4499,7 @@ fil_delete_tablespace( To deal with potential read requests, we will check the ::stop_new_ops flag in fil_io(). */ - buf_LRU_flush_or_remove_pages(id, buf_remove, 0); + buf_LRU_flush_or_remove_pages(space_id, buf_remove, 0); #endif /* !UNIV_HOTBACKUP */ @@ -3564,94 +4516,120 @@ fil_delete_tablespace( mtr_t mtr; mtr_start(&mtr); - fil_op_write_log(MLOG_FILE_DELETE, id, 0, path, NULL, 0, &mtr); + + fil_op_write_log( + MLOG_FILE_DELETE, space_id, path, nullptr, 0, &mtr); + mtr_commit(&mtr); + /* Even if we got killed shortly after deleting the tablespace file, the record must have already been written to the redo log. */ log_write_up_to(mtr.commit_lsn(), true); + #endif /* UNIV_HOTBACKUP */ - char* cfg_name = fil_make_filepath(path, NULL, CFG, false); - if (cfg_name != NULL) { + char* cfg_name = Fil_path::make_cfg(path); + + if (cfg_name != nullptr) { + os_file_delete_if_exists( - innodb_data_file_key, cfg_name, NULL); + innodb_data_file_key, cfg_name, nullptr); + ut_free(cfg_name); } - char* cfp_name = fil_make_filepath(path, NULL, CFP, false); - if (cfp_name != NULL) { + char* cfp_name = Fil_path::make_cfp(path); + + if (cfp_name != nullptr) { + os_file_delete_if_exists( - innodb_data_file_key, cfp_name, NULL); + innodb_data_file_key, cfp_name, nullptr); + ut_free(cfp_name); } } /* Must set back to active before returning from function. */ clone_mark_abort(true); - mutex_enter(&fil_system->mutex); + + mutex_acquire(); /* Double check the sanity of pending ops after reacquiring the fil_system::mutex. */ - if (const fil_space_t* s = fil_space_get_by_id(id)) { + if (const fil_space_t* s = get_space_by_id(space_id)) { + ut_a(s == space); ut_a(space->n_pending_ops == 0); - ut_a(UT_LIST_GET_LEN(space->chain) == 1); - - fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + ut_a(space->files.size() == 1); + ut_a(space->files.front().n_pending == 0); - ut_a(node->n_pending == 0); + space_detach(space); - fil_space_detach(space); + space_delete(space_id); - mutex_exit(&fil_system->mutex); + mutex_release(); - fil_space_free_low(space); + space_free_low(space); + ut_a(space == nullptr); if (!os_file_delete(innodb_data_file_key, path) && !os_file_delete_if_exists( - innodb_data_file_key, path, NULL)) { + innodb_data_file_key, path, nullptr)) { /* Note: This is because we have removed the tablespace instance from the cache. */ err = DB_IO_ERROR; } + } else { - mutex_exit(&fil_system->mutex); + + mutex_release(); + err = DB_TABLESPACE_NOT_FOUND; } ut_free(path); - fil_system->m_open.enter(); - fil_system->m_open.deleted(id); - fil_system->m_open.exit(); - clone_mark_active(); return(err); } -/*******************************************************************//** -Prepare for truncating a single-table tablespace. +/** Deletes an IBD tablespace, either general or single-table. +The tablespace must be cached in the memory cache. This will delete the +datafile, fil_space_t & fil_node_t entries from the file_system_t cache. +@param[in] space_id Tablespace ID +@param[in] buf_remove Specify the action to take on the pages + for this table in the buffer pool. +@return DB_SUCCESS, DB_TABLESPCE_NOT_FOUND or DB_IO_ERROR */ +dberr_t +fil_delete_tablespace( + space_id_t space_id, + buf_remove_t buf_remove) +{ + auto shard = fil_system->shard_by_id(space_id); + + return(shard->space_delete(space_id, buf_remove)); +} + +/** Prepare for truncating a single-table tablespace. 1) Check pending operations on a tablespace; 2) Remove all insert buffer entries for the tablespace; +@param[in] space_id Tablespace ID @return DB_SUCCESS or error */ -static dberr_t -fil_prepare_for_truncate( -/*=====================*/ - space_id_t id) /*!< in: space id */ +Fil_shard::space_prepare_for_truncate(space_id_t space_id) { - char* path = 0; - fil_space_t* space = 0; + char* path = nullptr; + fil_space_t* space = nullptr; - ut_ad(!fsp_is_system_or_temp_tablespace(id)); - ut_ad(fsp_is_undo_tablespace(id)); + ut_ad(!fsp_is_system_or_temp_tablespace(space_id)); + ut_ad(fsp_is_undo_tablespace(space_id)); - dberr_t err = fil_check_pending_operations( - id, FIL_OPERATION_CLOSE, &space, &path); + dberr_t err = space_check_pending_operations( + space_id, FIL_OPERATION_CLOSE, &space, &path); ut_free(path); @@ -3659,18 +4637,18 @@ fil_prepare_for_truncate( } /** Truncate the tablespace to needed size. -@param[in] space_id id of tablespace to truncate -@param[in] size_in_pages truncate size. +@param[in] space_id Tablespace ID to truncate +@param[in] size_in_pages Truncate size. @return true if truncate was successful. */ bool -fil_truncate_tablespace( +Fil_shard::space_truncate( space_id_t space_id, page_no_t size_in_pages) { /* Step-1: Prepare tablespace for truncate. This involves stopping all the new operations + IO on that tablespace and ensuring that related pages are flushed to disk. */ - if (fil_prepare_for_truncate(space_id) != DB_SUCCESS) { + if (space_prepare_for_truncate(space_id) != DB_SUCCESS) { return(false); } @@ -3682,26 +4660,26 @@ fil_truncate_tablespace( /* Step-3: Truncate the tablespace and accordingly update the fil_space_t handler that is used to access this tablespace. */ - mutex_enter(&fil_system->mutex); - fil_space_t* space = fil_space_get_by_id(space_id); + mutex_acquire(); + + fil_space_t* space = get_space_by_id(space_id); - /* The following code must change when InnoDB supports - multiple datafiles per tablespace. */ - ut_a(UT_LIST_GET_LEN(space->chain) == 1); + ut_a(space->files.size() == 1); - fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + fil_node_t& file = space->files.front(); - ut_ad(node->is_open); + ut_ad(file.is_open); - space->size = node->size = size_in_pages; + space->size = file.size = size_in_pages; + + bool success = os_file_truncate(file.name, file.handle, 0); - bool success = os_file_truncate(node->name, node->handle, 0); if (success) { os_offset_t size = size_in_pages * UNIV_PAGE_SIZE; success = os_file_set_size( - node->name, node->handle, 0, size, + file.name, file.handle, 0, size, srv_read_only_mode, true); if (success) { @@ -3709,76 +4687,87 @@ fil_truncate_tablespace( } } - mutex_exit(&fil_system->mutex); + mutex_release(); return(success); } +/** Truncate the tablespace to needed size. +@param[in] space_id Tablespace ID to truncate +@param[in] size_in_pages Truncate size. +@return true if truncate was successful. */ +bool +fil_truncate_tablespace( + space_id_t space_id, + page_no_t size_in_pages) +{ + auto shard = fil_system->shard_by_id(space_id); + + return(shard->space_truncate(space_id, size_in_pages)); +} + #ifdef UNIV_DEBUG /** Increase redo skipped count for a tablespace. -@param[in] id space id */ +@param[in] space_id Tablespace ID */ void -fil_space_inc_redo_skipped_count(space_id_t id) +fil_space_inc_redo_skipped_count(space_id_t space_id) { - fil_space_t* space; + auto shard = fil_system->shard_by_id(space_id); - mutex_enter(&fil_system->mutex); + shard->mutex_acquire(); - space = fil_space_get_by_id(id); + fil_space_t* space = shard->get_space_by_id(space_id); - ut_a(space != NULL); + ut_a(space != nullptr); - space->redo_skipped_count++; + ++space->redo_skipped_count; - mutex_exit(&fil_system->mutex); + shard->mutex_release(); } /** Decrease redo skipped count for a tablespace. -@param[in] id space id */ +@param[in] space_id Tablespace id */ void -fil_space_dec_redo_skipped_count(space_id_t id) +fil_space_dec_redo_skipped_count(space_id_t space_id) { - fil_space_t* space; + auto shard = fil_system->shard_by_id(space_id); - mutex_enter(&fil_system->mutex); + shard->mutex_acquire(); - space = fil_space_get_by_id(id); + fil_space_t* space = shard->get_space_by_id(space_id); - ut_a(space != NULL); + ut_a(space != nullptr); ut_a(space->redo_skipped_count > 0); - space->redo_skipped_count--; + --space->redo_skipped_count; - mutex_exit(&fil_system->mutex); + shard->mutex_release(); } -/** -Check whether a single-table tablespace is redo skipped. -@param[in] id space id +/** Check whether a single-table tablespace is redo skipped. +@param[in] space_id Tablespace id @return true if redo skipped */ bool -fil_space_is_redo_skipped(space_id_t id) +fil_space_is_redo_skipped(space_id_t space_id) { - fil_space_t* space; - bool is_redo_skipped; + auto shard = fil_system->shard_by_id(space_id); - mutex_enter(&fil_system->mutex); + shard->mutex_acquire(); - space = fil_space_get_by_id(id); + fil_space_t* space = shard->get_space_by_id(space_id); - ut_a(space != NULL); + ut_a(space != nullptr); - is_redo_skipped = space->redo_skipped_count > 0; + bool is_redo_skipped = space->redo_skipped_count > 0; - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(is_redo_skipped); } -#endif +#endif /* UNIV_DEBUG */ #ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Discards a single-table tablespace. The tablespace must be cached in the +/** Discards a single-table tablespace. The tablespace must be cached in the memory cache. Discarding is like deleting a tablespace, but 1. We do not drop the table from the data dictionary; @@ -3787,26 +4776,31 @@ memory cache. Discarding is like deleting a tablespace, but in DROP TABLE they are only removed gradually in the background; 3. Free all the pages in use by the tablespace. +@param[in] space_id Tablespace ID @return DB_SUCCESS or error */ dberr_t -fil_discard_tablespace( -/*===================*/ - space_id_t id) /*!< in: space id */ +fil_discard_tablespace(space_id_t space_id) { dberr_t err; - switch (err = fil_delete_tablespace(id, BUF_REMOVE_ALL_NO_WRITE)) { + err = fil_delete_tablespace(space_id, BUF_REMOVE_ALL_NO_WRITE); + + switch (err) { case DB_SUCCESS: break; case DB_IO_ERROR: - ib::warn() << "While deleting tablespace " << id + + ib::warn() + << "While deleting tablespace " << space_id << " in DISCARD TABLESPACE. File rename/delete" " failed: " << ut_strerr(err); break; case DB_TABLESPACE_NOT_FOUND: - ib::warn() << "Cannot delete tablespace " << id + + ib::warn() + << "Cannot delete tablespace " << space_id << " in DISCARD TABLESPACE: " << ut_strerr(err); break; @@ -3816,113 +4810,224 @@ fil_discard_tablespace( /* Remove all insert buffer entries for the tablespace */ - ibuf_delete_for_discarded_space(id); + ibuf_delete_for_discarded_space(space_id); return(err); } + +/** Write redo log for renaming a file. +@param[in] space_id tablespace id +@param[in] old_name tablespace file name +@param[in] new_name tablespace file name after renaming +@param[in,out] mtr mini-transaction */ +static +void +fil_name_write_rename( + space_id_t space_id, + const char* old_name, + const char* new_name, + mtr_t* mtr) +{ + ut_ad(!fsp_is_system_or_temp_tablespace(space_id)); + ut_ad(!fsp_is_undo_tablespace(space_id)); + + /* Note: A checkpoint can take place here. */ + + DBUG_EXECUTE_IF( "ib_crash_rename_log_1", DBUG_SUICIDE();); + + static const auto type = MLOG_FILE_RENAME; + + fil_op_write_log(type, space_id, old_name, new_name, 0, mtr); + + DBUG_EXECUTE_IF( "ib_crash_rename_log_2", DBUG_SUICIDE();); + + /* Note: A checkpoint can take place here too before we + have physically renamed the file. */ +} + #endif /* !UNIV_HOTBACKUP */ /** Allocate and build a file name from a path, a table or tablespace name and a suffix. -@param[in] path NULL or the direcory path or the full path and filename -@param[in] name NULL if path is full, or Table/Tablespace name -@param[in] ext the file extension to use -@param[in] trim whether last name on the path should be trimmed +@param[in] path_in nullptr or the direcory path or the full path + and filename +@param[in] name_in nullptr if path is full, or Table/Tablespace + name +@param[in] ext the file extension to use +@param[in] trim whether last name on the path should be trimmed @return own: file name; must be freed by ut_free() */ char* -fil_make_filepath( - const char* path, - const char* name, - ib_extention ext, - bool trim) +Fil_path::make( + const std::string& path_in, + const std::string& name_in, + ib_file_suffix ext, + bool trim) { - /* The path may contain the basename of the file, if so we do not - need the name. If the path is NULL, we can use the default path, - but there needs to be a name. */ - ut_ad(path != NULL || name != NULL); + /* The path should be a directory and should not contain the + basename of the file. If the path is empty, we will use the + default path, */ + + ut_ad(!path_in.empty() || !name_in.empty()); + + std::string path; - /* If we are going to strip a name off the path, there better be a - path and a new name to put back on. */ - ut_ad(!trim || (path != NULL && name != NULL)); + if (path_in.empty()) { - if (path == NULL) { - path = fil_path_to_mysql_datadir; + if (is_absolute_path(name_in)) { + path = ""; + } else { + path.assign(MySQL_datadir_path); + } + } else { + path.assign(path_in); } - ulint len = 0; /* current length */ - ulint path_len = strlen(path); - ulint name_len = (name ? strlen(name) : 0); - const char* suffix = dot_ext[ext]; - ulint suffix_len = strlen(suffix); - ulint full_len = path_len + 1 + name_len + suffix_len + 1; + std::string name; - char* full_name = static_cast(ut_malloc_nokey(full_len)); - if (full_name == NULL) { - return(NULL); + if (!name_in.empty()) { + name.assign(name_in); } - /* If the name is a relative path, do not prepend "./". */ - if (path[0] == '.' - && (path[1] == '\0' || path[1] == OS_PATH_SEPARATOR) - && name != NULL && name[0] == '.') { - path = NULL; - path_len = 0; + + /* Do not prepend the datadir path (which must be DOT_SLASH) + if the name is an absolute path or a relative path like + DOT_SLASH or DOT_DOT_SLASH. */ + if (is_absolute_path(name) + || has_prefix(name, DOT_SLASH) + || has_prefix(name, DOT_DOT_SLASH)) { + + path.clear(); } - if (path != NULL) { - memcpy(full_name, path, path_len); - len = path_len; - full_name[len] = '\0'; - os_normalize_path(full_name); + std::string filepath; + + if (!path.empty()) { + filepath.assign(path); } if (trim) { /* Find the offset of the last DIR separator and set it to null in order to strip off the old basename from this path. */ - char* last_dir_sep = strrchr(full_name, OS_PATH_SEPARATOR); - if (last_dir_sep) { - last_dir_sep[0] = '\0'; - len = strlen(full_name); + auto pos = filepath.find_last_of(SEPARATOR); + + if (pos != std::string::npos) { + filepath.resize(pos); } } - if (name != NULL) { - if (len && full_name[len - 1] != OS_PATH_SEPARATOR) { - /* Add a DIR separator */ - full_name[len] = OS_PATH_SEPARATOR; - full_name[++len] = '\0'; + if (!name.empty()) { + + if (!filepath.empty() && !is_separator(filepath.back())) { + + filepath.push_back(OS_SEPARATOR); } - char* ptr = &full_name[len]; - memcpy(ptr, name, name_len); - len += name_len; - full_name[len] = '\0'; - os_normalize_path(ptr); + filepath.append(name); } - /* Make sure that the specified suffix is at the end of the filepath - string provided. This assumes that the suffix starts with '.'. - If the first char of the suffix is found in the filepath at the same - length as the suffix from the end, then we will assume that there is - a previous suffix that needs to be replaced. */ - if (suffix != NULL) { - /* Need room for the trailing null byte. */ - ut_ad(len < full_len); + /* Make sure that the specified suffix is at the end. */ + if (ext != NO_EXT) { + + const auto suffix = dot_ext[ext]; + size_t len = strlen(suffix); - if ((len > suffix_len) - && (full_name[len - suffix_len] == suffix[0])) { - /* Another suffix exists, make it the one requested. */ - memcpy(&full_name[len - suffix_len], suffix, suffix_len); + /* This assumes that the suffix starts with '.'. If the + first char of the suffix is found in the filepath at the + same length as the suffix from the end, then we will assume + that there is a previous suffix that needs to be replaced. */ + ut_ad(*suffix == '.'); + + if (filepath.length() > len + && *(filepath.end() - len) == *suffix) { + + filepath.replace( + filepath.end() - len, + filepath.end(), suffix); } else { - /* No previous suffix, add it. */ - ut_ad(len + suffix_len < full_len); - memcpy(&full_name[len], suffix, suffix_len); - full_name[len + suffix_len] = '\0'; + + filepath.append(suffix); } } - return(full_name); + normalize(filepath); + + return(mem_strdup(filepath.c_str())); +} + +/** Create an IBD path name after replacing the basename in an old path +with a new basename. The old_path is a full path name including the +extension. The tablename is in the normal form "schema/tablename". + +@param[in] path_in Pathname +@param[in] name_in Contains new base name +@return own: new full pathname */ +std::string +Fil_path::make_new_ibd( + const std::string& path_in, + const std::string& name_in) +{ + ut_a(Fil_path::has_ibd_suffix(path_in)); + ut_a(!Fil_path::has_ibd_suffix(name_in)); + + std::string path(path_in); + + auto pos = path.find_last_of(SEPARATOR); + + ut_a(pos != std::string::npos); + + path.resize(pos); + + pos = path.find_last_of(SEPARATOR); + + ut_a(pos != std::string::npos); + + path.resize(pos + 1); + + path.append(name_in + ".ibd"); + + normalize(path); + + return(path); +} + +/** This function reduces a null-terminated full remote path name +into the path that is sent by MySQL for DATA DIRECTORY clause. +It replaces the 'databasename/tablename.ibd' found at the end of the +path with just 'tablename'. + +Since the result is always smaller than the path sent in, no new +memory is allocated. The caller should allocate memory for the path +sent in. This function manipulates that path in place. If the path +format is not as expected, set data_dir_path to "" and return. + +The result is used to inform a SHOW CREATE TABLE command. +@param[in,out] data_dir_path Full path/data_dir_path */ +void +Fil_path::make_data_dir_path(char* data_dir_path) +{ + /* Replace the period before the extension with a null byte. */ + ut_ad(has_ibd_suffix(data_dir_path)); + char* dot = strrchr((char*) data_dir_path, '.'); + *dot = '\0'; + + /* The tablename starts after the last slash. */ + char* base_slash = strrchr((char*) data_dir_path, OS_PATH_SEPARATOR); + ut_ad(base_slash != nullptr); + + *base_slash = '\0'; + + std::string base_name{base_slash + 1}; + + /* The database name starts after the next to last slash. */ + char* db_slash = strrchr((char*) data_dir_path, OS_SEPARATOR); + ut_ad(db_slash != nullptr); + char* db_name = db_slash + 1; + + + /* Overwrite the db_name with the base_name. */ + memmove(db_name, base_name.c_str(), base_name.length()); + db_name[base_name.length()] = '\0'; } /** Test if a tablespace file can be renamed to a new filepath by checking @@ -3945,7 +5050,9 @@ fil_rename_tablespace_check( if (!is_discarded && os_file_status(old_path, &exists, &ftype) && !exists) { - ib::error() << "Cannot rename '" << old_path + + ib::error() + << "Cannot rename '" << old_path << "' to '" << new_path << "' for space ID " << space_id << " because the source file" @@ -3954,8 +5061,11 @@ fil_rename_tablespace_check( } exists = false; + if (!os_file_status(new_path, &exists, &ftype) || exists) { - ib::error() << "Cannot rename '" << old_path + + ib::error() + << "Cannot rename '" << old_path << "' to '" << new_path << "' for space ID " << space_id << " because the target file exists." @@ -3968,222 +5078,257 @@ fil_rename_tablespace_check( /** Rename a single-table tablespace. The tablespace must exist in the memory cache. -@param[in] id tablespace identifier -@param[in] old_path old file name -@param[in] new_name new table name in the -databasename/tablename format -@param[in] new_path_in new file name, -or NULL if it is located in the normal data directory +@param[in] space_id Tablespace ID +@param[in] old_path Old file name +@param[in] new_name New tablespace name in the schema/space +@param[in] new_path_in New file name, or nullptr if it is located + in the normal data directory @return true if success */ bool -fil_rename_tablespace( - space_id_t id, +Fil_shard::space_rename( + space_id_t space_id, const char* old_path, const char* new_name, const char* new_path_in) { - bool sleep = false; - bool flush = false; fil_space_t* space; - fil_node_t* node; - ulint count = 0; - bool write_ddl_log = true; - ut_d(static uint32_t crash_injection_rename_tablespace_counter = 1;); + ulint count = 0; + fil_node_t* file = nullptr; + bool write_ddl_log = true; + auto start_time = ut_time(); - ut_a(id != 0); - ut_ad(strchr(new_name, '/') != NULL); +#ifdef UNIV_DEBUG + static uint32_t crash_injection_rename_tablespace_counter = 1; +#endif /* UNIV_DEBUG */ -retry: - count++; + ut_a(space_id != TRX_SYS_SPACE); + ut_ad(strchr(new_name, '/') != nullptr); - if (!(count % 1000)) { - ib::warn() << "Cannot rename file " << old_path - << " (space id " << id << "), retried " << count - << " times." - " There are either pending IOs or flushes or" - " the file is being extended."; - } + for (;;) { + bool retry = false; + bool flush = false; - mutex_enter(&fil_system->mutex); + ++count; - space = fil_space_get_by_id(id); + if (!(count % 1000)) { + ib::warn() + << "Cannot rename file " << old_path + << " (space id " << space_id << ")," + << " retried " << count << " times." + << " There are either pending IOs or" + << " flushes or the file is being extended."; + } - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_1", space = NULL; ); + /* The name map and space ID map are in the same shard. */ + mutex_acquire(); - if (space == nullptr) { + space = get_space_by_id(space_id); - ib::error() - << "Cannot find space id " << id - << " in the tablespace memory cache, though the file '" - << old_path - << "' in a rename operation should have that id."; + DBUG_EXECUTE_IF("fil_rename_tablespace_failure_1", + space = nullptr; ); - mutex_exit(&fil_system->mutex); - return(false); + if (space == nullptr) { - } else if (count > 25000) { + ib::error() + << "Cannot find space id " << space_id + << " in the tablespace memory cache," + << " though the file '" << old_path << "'" + << " in a rename operation should have" + << " that ID."; - space->stop_ios = false; + mutex_release(); - mutex_exit(&fil_system->mutex); + return(false); - return(false); + } else if (space->stop_ios) { - } else if (space != fil_space_get_by_name(space->name)) { + /* Some other thread has stopped the IO. We need to + wait for the other thread to complete its operation. */ + mutex_release(); - ib::error() - << "Cannot find " << space->name - << " in tablespace memory cache"; + if (ut_time() - start_time >= PRINT_INTERVAL_SECS) { - space->stop_ios = false; + ib::warn() << "Rename waiting for IO to resume"; - mutex_exit(&fil_system->mutex); + start_time = ut_time(); + } - return(false); + os_thread_sleep(1000000); - } else { - - auto new_space = fil_space_get_by_name(new_name); + continue; - if (new_space != nullptr) { + } else if (count > 25000) { + + mutex_release(); - if (new_space == space) { + return(false); - mutex_exit(&fil_system->mutex); + } else if (space != get_space_by_name(space->name)) { - return(true); + ib::error() + << "Cannot find " << space->name + << " in tablespace memory cache"; - } else if (new_space->id != space->id) { + mutex_release(); - ib::error() - << new_name - << " is already in the tablespace" - << " memory cache"; + return(false); + + } else { - space->stop_ios = false; + auto new_space = get_space_by_name(new_name); - mutex_exit(&fil_system->mutex); + if (new_space != nullptr) { - return(false); + if (new_space == space) { + + mutex_release(); + + return(true); + } + + ut_a(new_space->id == space->id); } } - } + + ut_a(space->files.size() == 1); #ifndef UNIV_HOTBACKUP - /* Don't write DDL log during recovery when log_ddl is not - initialized */ - if (write_ddl_log && log_ddl != nullptr) { - /* Write ddl log when space->stop_ios is true can - cause deadlock: - a. buffer flush thread waits for rename thread to set - stop_ios to false; - b. rename thread waits for buffer flush thread to flush - a page and release page lock. The page is ready for - flush in double write buffer. */ - ut_ad(!space->stop_ios); + /* Don't write DDL log during recovery when log_ddl is + not initialized. */ - ut_a(UT_LIST_GET_LEN(space->chain) == 1); - node = UT_LIST_GET_FIRST(space->chain); + if (write_ddl_log && log_ddl != nullptr) { - char* new_file_name = new_path_in == NULL - ? fil_make_filepath(NULL, new_name, IBD, false) - : mem_strdup(new_path_in); - char* old_file_name = node->name; + /* Write ddl log when space->stop_ios is true + can cause deadlock: + a. buffer flush thread waits for rename thread to set + stop_ios to false; + b. rename thread waits for buffer flush thread to flush + a page and release page lock. The page is ready for + flush in double write buffer. */ - ut_ad(strchr(old_file_name, OS_PATH_SEPARATOR) != NULL); - ut_ad(strchr(new_file_name, OS_PATH_SEPARATOR) != NULL); + ut_ad(!space->stop_ios); - mutex_exit(&fil_system->mutex); + file = &space->files.front(); - /* Rename ddl log is for rollback, so we exchange old file - name with new file name. */ - log_ddl->write_rename_space_log( - id, new_file_name, old_file_name); + char* new_file_name = new_path_in == nullptr + ? Fil_path::make_ibd_from_table_name(new_name) + : mem_strdup(new_path_in); - ut_free(new_file_name); + char* old_file_name = file->name; - write_ddl_log = false; + ut_ad(strchr(old_file_name, OS_PATH_SEPARATOR) + != nullptr); - goto retry; - } -#endif /* !UNIV_HOTBACKUP */ + ut_ad(strchr(new_file_name, OS_PATH_SEPARATOR) + != nullptr); - space->stop_ios = true; + mutex_release(); - /* The following code must change when InnoDB supports - multiple datafiles per tablespace. */ - ut_a(UT_LIST_GET_LEN(space->chain) == 1); - node = UT_LIST_GET_FIRST(space->chain); + /* Rename ddl log is for rollback, so we exchange + old file name with new file name. */ + log_ddl->write_rename_space_log( + space_id, new_file_name, old_file_name); - if (node->n_pending > 0 - || node->n_pending_flushes > 0 - || node->in_use > 0) { - /* There are pending i/o's or flushes or the file is - currently being extended, sleep for a while and - retry */ - sleep = true; + ut_free(new_file_name); - } else if (node->modification_counter > node->flush_counter) { - /* Flush the space */ - sleep = flush = true; + write_ddl_log = false; + continue; + } +#endif /* !UNIV_HOTBACKUP */ - } else if (node->is_open) { - /* Close the file */ + /* We temporarily close the .ibd file because we do + not trust that operating systems can rename an open + file. For the closing we have to wait until there + are no pending I/O's or flushes on the file. */ - fil_node_close_file(node, false); - } + space->stop_ios = true; - mutex_exit(&fil_system->mutex); + file = &space->files.front(); - if (sleep) { - os_thread_sleep(20000); + if (file->n_pending > 0 + || file->n_pending_flushes > 0 + || file->in_use > 0) { - if (flush) { - fil_flush(id); - } + /* There are pending I/O's or flushes or the + file is currently being extended, sleep for + a while and retry */ - sleep = flush = false; - goto retry; - } + retry = true; - ut_ad(space->stop_ios); + space->stop_ios = false; - char* new_file_name = new_path_in == NULL - ? fil_make_filepath(NULL, new_name, IBD, false) - : mem_strdup(new_path_in); - char* old_file_name = node->name; - char* new_space_name = mem_strdup(new_name); - char* old_space_name = space->name; + } else if (file->modification_counter > file->flush_counter) { - ut_ad(strchr(old_file_name, OS_PATH_SEPARATOR) != NULL); - ut_ad(strchr(new_file_name, OS_PATH_SEPARATOR) != NULL); + /* Flush the space */ + + retry = flush = true; + + space->stop_ios = false; + + } else if (file->is_open) { + + close_file(file, false); + } + + mutex_release(); + + if (!retry) { + ut_ad(space->stop_ios); + break; + } + + os_thread_sleep(100000); + + if (flush) { + + mutex_acquire(); + + space_flush(space->id); + + mutex_release(); + } + } + + ut_ad(space->stop_ios); + + char* new_file_name; + + if (new_path_in == nullptr) { + + new_file_name = Fil_path::make_ibd_from_table_name(new_name); + } else { + new_file_name = mem_strdup(new_path_in); + } + + char* old_file_name = file->name; + char* old_space_name = space->name; + char* new_space_name = mem_strdup(new_name); #ifndef UNIV_HOTBACKUP if (!recv_recovery_on) { - mtr_t mtr; + mtr_t mtr; mtr.start(); + fil_name_write_rename( - id, 0, old_file_name, new_file_name, &mtr); + space_id, old_file_name, new_file_name, &mtr); + mtr.commit(); - log_mutex_enter(); } #endif /* !UNIV_HOTBACKUP */ - /* log_sys->mutex is above fil_system->mutex in the latching order */ - ut_ad(log_mutex_own()); - mutex_enter(&fil_system->mutex); + ut_ad(strchr(old_file_name, OS_PATH_SEPARATOR) != nullptr); + ut_ad(strchr(new_file_name, OS_PATH_SEPARATOR) != nullptr); + + mutex_acquire(); - ut_ad(space->name == old_space_name); /* We already checked these. */ - ut_ad(space == fil_space_get_by_name(old_space_name)); - ut_ad(!fil_space_get_by_name(new_space_name)); - ut_ad(node->name == old_file_name); + ut_ad(space == get_space_by_name(old_space_name)); + ut_ad(get_space_by_name(new_space_name) == nullptr); bool success; - DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", - goto skip_rename; ); + DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", goto skip_rename; ); DBUG_INJECT_CRASH("ddl_crash_before_rename_tablespace", crash_injection_rename_tablespace_counter++); @@ -4197,29 +5342,13 @@ fil_rename_tablespace( DBUG_INJECT_CRASH("ddl_crash_after_rename_tablespace", crash_injection_rename_tablespace_counter++); - ut_ad(node->name == old_file_name); - if (success) { - node->name = new_file_name; - } - -#ifndef UNIV_HOTBACKUP - if (!recv_recovery_on) { - log_mutex_exit(); - } -#endif /* !UNIV_HOTBACKUP */ - - ut_ad(space->name == old_space_name); + file->name = new_file_name; - if (success) { - fil_system->names.erase(space->name); + update_space_name_map(space, new_space_name); space->name = new_space_name; - auto it = fil_system->names.insert( - Names::value_type(space->name, space)); - - ut_a(it.second); } else { /* Because nothing was renamed, we must free the new names, not the old ones. */ @@ -4229,7 +5358,8 @@ fil_rename_tablespace( ut_ad(space->stop_ios); space->stop_ios = false; - mutex_exit(&fil_system->mutex); + + mutex_release(); ut_free(old_file_name); ut_free(old_space_name); @@ -4237,59 +5367,128 @@ fil_rename_tablespace( return(success); } -/* Rename a tablespace by its name only +/** Rename a single-table tablespace. +The tablespace must exist in the memory cache. +@param[in] space_id Tablespace ID +@param[in] old_path Old file name +@param[in] new_name New tablespace name in the schema/name format +@param[in] new_path_in New file name, or nullptr if it is located + in the normal data directory +@return true if success */ +bool +fil_rename_tablespace( + space_id_t space_id, + const char* old_path, + const char* new_name, + const char* new_path_in) +{ + auto shard = fil_system->shard_by_id(space_id); + + bool success = shard->space_rename( + space_id, old_path, new_name, new_path_in); + + return(success); +} + +/** Rename a tablespace by its name only @param[in] old_name old tablespace name @param[in] new_name new tablespace name @return DB_SUCCESS on success */ -/* purecov: begin inspected */ dberr_t -fil_rename_tablespace_by_name( - const char* old_name, - const char* new_name) +Fil_system::rename_tablespace_name(const char* old_name, const char* new_name) { - mutex_enter(&fil_system->mutex); - fil_space_t* space = fil_space_get_by_name(old_name); + mutex_acquire_all(); + + Fil_shard* old_shard = nullptr; + fil_space_t* old_space = nullptr; + + for (auto shard : m_shards) { + + old_space = shard->get_space_by_name(old_name); + + if (old_space != nullptr) { + old_shard = shard; + break; + } + } + + if (old_space == nullptr) { + + mutex_release_all(); - if (!space) { - mutex_exit(&fil_system->mutex); ib::error() - << "Cannot find space for " << old_name + << "Cannot find tablespace for '" << old_name << "'" << " in tablespace memory cache"; return(DB_TABLESPACE_NOT_FOUND); } - auto new_space = fil_space_get_by_name(new_name); + Fil_shard* new_shard = nullptr; + fil_space_t* new_space = nullptr; + + for (auto shard : m_shards) { + + new_space = shard->get_space_by_name(new_name); + + if (new_space != nullptr) { + new_shard = shard; + break; + } + } if (new_space != nullptr) { - mutex_exit(&fil_system->mutex); - if (new_space->id != space->id) { + + mutex_release_all(); + + if (new_space->id != old_space->id) { + ib::error() - << new_name + << "'" << new_name << "'" << " is already in the tablespace" << " memory cache"; return(DB_TABLESPACE_EXISTS); + } else { + ut_a(new_shard == old_shard); } + return(DB_SUCCESS); } - fil_space_update_name(space, new_name, true); + auto new_space_name = mem_strdup(new_name); + auto old_space_name = old_space->name; + + old_shard->update_space_name_map(old_space, new_space_name); + + old_space->name = new_space_name; - mutex_exit(&fil_system->mutex); + mutex_release_all(); + + ut_free(old_space_name); return(DB_SUCCESS); } -/* purecov: end */ +/** Rename a tablespace by its name only +@param[in] old_name old tablespace name +@param[in] new_name new tablespace name +@return DB_SUCCESS on success */ +dberr_t +fil_rename_tablespace_by_name( + const char* old_name, + const char* new_name) +{ + return(fil_system->rename_tablespace_name(old_name, new_name)); +} /** Create a tablespace file. @param[in] space_id Tablespace ID @param[in] name Tablespace name in dbname/tablename format. -For general tablespaces, the 'dbname/' part may be missing. + For general tablespaces, the 'dbname/' part + may be missing. @param[in] path Path and filename of the datafile to create. @param[in] flags Tablespace flags @param[in] size Initial size of the tablespace file in pages, -must be >= FIL_IBD_FILE_INITIAL_SIZE + must be >= FIL_IBD_FILE_INITIAL_SIZE @return DB_SUCCESS or error code */ dberr_t fil_ibd_create( @@ -4305,7 +5504,7 @@ fil_ibd_create( byte* page; bool success; bool has_shared_space = FSP_FLAGS_GET_SHARED(flags); - fil_space_t* space = NULL; + fil_space_t* space = nullptr; ut_ad(!fsp_is_system_or_temp_tablespace(space_id)); ut_ad(!srv_read_only_mode); @@ -4317,7 +5516,9 @@ fil_ibd_create( /* Create the subdirectories in the path, if they are not there already. */ if (!has_shared_space) { + err = os_file_create_subdirs_if_needed(path); + if (err != DB_SUCCESS) { return(err); } @@ -4338,7 +5539,8 @@ fil_ibd_create( ib::error() << "Cannot create file '" << path << "'"; if (error == OS_FILE_ALREADY_EXISTS) { - ib::error() << "The file '" << path << "'" + ib::error() + << "The file '" << path << "'" " already exists though the" " corresponding table did not exist." " Have you moved InnoDB .ibd files" @@ -4439,6 +5641,7 @@ fil_ibd_create( flush would write to it. */ buf2 = static_cast(ut_malloc_nokey(3 * page_size.logical())); + /* Align the memory for file i/o if we might have O_DIRECT set */ page = static_cast(ut_align(buf2, page_size.logical())); @@ -4460,7 +5663,7 @@ fil_ibd_create( if (!page_size.is_compressed()) { buf_flush_init_for_writing( - NULL, page, NULL, 0, + nullptr, page, nullptr, 0, fsp_is_checksum_disabled(space_id)); err = os_file_write( @@ -4480,7 +5683,7 @@ fil_ibd_create( page_zip.n_blobs = 0; buf_flush_init_for_writing( - NULL, page, &page_zip, 0, + nullptr, page, &page_zip, 0, fsp_is_checksum_disabled(space_id)); err = os_file_write( @@ -4509,8 +5712,11 @@ fil_ibd_create( success = os_file_flush(file); if (!success) { - ib::error() << "File flush of tablespace '" + + ib::error() + << "File flush of tablespace '" << path << "' failed"; + os_file_close(file); os_file_delete(innodb_data_file_key, path); return(DB_ERROR); @@ -4518,37 +5724,34 @@ fil_ibd_create( space = fil_space_create(name, space_id, flags, FIL_TYPE_TABLESPACE); + if (space == nullptr){ + os_file_close(file); + os_file_delete(innodb_data_file_key, path); + return(DB_ERROR); + } + DEBUG_SYNC_C("fil_ibd_created_space"); - err = fil_node_create_low( - path, size, space, false, punch_hole, atomic_write) - ? DB_SUCCESS - : DB_ERROR; + auto shard = fil_system->shard_by_id(space_id); + + fil_node_t* file_node = shard->create_node( + path, size, space, false, punch_hole, atomic_write); + + err = (file_node == nullptr) ? DB_ERROR : DB_SUCCESS; + #ifndef UNIV_HOTBACKUP if (err == DB_SUCCESS) { - const fil_node_t* file = UT_LIST_GET_FIRST(space->chain); - - fil_system->m_open.enter(); - fil_system->m_open.open(space_id, file->name, log_get_lsn()); - fil_system->m_open.exit(); + const auto& file = space->files.front(); mtr_t mtr; mtr_start(&mtr); fil_op_write_log( - MLOG_FILE_CREATE2, space_id, 0, file->name, - NULL, space->flags, &mtr); - mtr_commit(&mtr); + MLOG_FILE_CREATE, space_id, file.name, + nullptr, space->flags, &mtr); - /* If a checkpoint happens here then we won't have - access to the space ID -> filename mapping in the - redo log and tablespace.open.* files either. - We can't rely on file open to write the redo log - record because we can write changes to the redo - log before we open the file. Therefore we have to - ensure that the mapping file is first written to - the disk. */ + mtr_commit(&mtr); DBUG_EXECUTE_IF( "fil_ibd_create_log", @@ -4556,11 +5759,12 @@ fil_ibd_create( } #endif /* !UNIV_HOTBACKUP */ + /* For encryption tablespace, initial encryption information. */ if (space != nullptr && FSP_FLAGS_GET_ENCRYPTION(space->flags)) { err = fil_set_encryption( - space->id, Encryption::AES, NULL, NULL); + space->id, Encryption::AES, nullptr, nullptr); ut_ad(err == DB_SUCCESS); } @@ -4568,25 +5772,6 @@ fil_ibd_create( os_file_close(file); if (err != DB_SUCCESS) { os_file_delete(innodb_data_file_key, path); - } else { - /* For CREATE TABLE, the file was actually opened on IO. i.e - when we extend datafile. We now estimate the initial size - better, so we longer need to extend datafile. This inturn means, - we may not open the file at all and we miss MLOG_FILE_OPEN entry - to redo log. So open the file now. - - If there are too many open files, ensure we are able to open - atleast one file by closing some unused files in fil_sys LRU.*/ - - /* TODO: WL#8619. Remove this entire else part and the function - fil_space_prepare_for_io() */ - mutex_enter(&fil_system->mutex); -#ifdef UNIV_DEBUG - fil_space_t* prepared_space = -#endif - fil_space_prepare_for_io(space); - ut_ad(prepared_space != nullptr); - mutex_exit(&fil_system->mutex); } return(err); @@ -4607,11 +5792,13 @@ The fil_node_t::handle will not be left open. (read the first page of the file and check that the space id in it matches id) @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_TEMPORARY -@param[in] id tablespace ID +@param[in] space_id Tablespace ID @param[in] flags tablespace flags @param[in] space_name tablespace name of the datafile -If file-per-table, it is the table name in the databasename/tablename format -@param[in] table_name table name in case if need to construct file path + If file-per-table, it is the table name in + the databasename/tablename format +@param[in] table_name table name in case if need to construct + file path @param[in] path_in expected filepath, usually read from dictionary @param[in] strict whether to report error when open ibd failed @param[in] old_space whether it is a 5.7 tablespace opening @@ -4621,7 +5808,7 @@ dberr_t fil_ibd_open( bool validate, fil_type_t purpose, - space_id_t id, + space_id_t space_id, ulint flags, const char* space_name, const char* table_name, @@ -4639,20 +5826,42 @@ fil_ibd_open( return(DB_CORRUPTION); } + /* Check if the file is already open. The space can be loaded + via fil_space_get_first_path() on startup. This is a problem + for partitioning code. It's a convoluted call graph via the DD. + On Windows this can lead to a sharing violation when we attempt + to open it again. */ + + auto shard = fil_system->shard_by_id(space_id); + + shard->mutex_acquire(); + + auto space = shard->get_space_by_id(space_id); + + if (space != nullptr) { + + shard->space_detach(space); + shard->space_delete(space->id); + shard->space_free_low(space); + ut_a(space == nullptr); + } + + shard->mutex_release(); + df.init(space_name, flags); - if (path_in) { + if (path_in != nullptr) { df.set_filepath(path_in); } else { - df.make_filepath(NULL, table_name, IBD); + df.make_filepath(nullptr, table_name, IBD); } - /* Attempt to open the tablespace at the dictionary filepath. */ + /* Attempt to open the tablespace. */ if (df.open_read_only(strict) == DB_SUCCESS) { ut_ad(df.is_open()); } else { ut_ad(!df.is_open()); - return(DB_CORRUPTION); + return(DB_CANNOT_OPEN_FILE); } #if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) @@ -4662,9 +5871,17 @@ fil_ibd_open( const bool atomic_write = false; #endif /* !NO_FALLOCATE && UNIV_LINUX */ + dberr_t err; + if ((validate || is_encrypted) - && df.validate_to_dd(id, flags, for_import) != DB_SUCCESS) { - if (!is_encrypted) { + && (err = df.validate_to_dd(space_id, flags, for_import)) + != DB_SUCCESS) { + + /* We don't reply the rename via the redo log anymore. + Therefore we can get a space ID mismatch when validating + the files during bootstrap. */ + + if (!is_encrypted && err != DB_WRONG_FILE_NAME) { /* The following call prints an error message. For encrypted tablespace we skip print, since it should be keyring plugin issues. */ @@ -4672,28 +5889,32 @@ fil_ibd_open( ib::error() << "Could not find a valid tablespace file" - << " for `" << space_name << "`. " - << TROUBLESHOOT_DATADICT_MSG; + << " for `" << space_name << "`. " + << TROUBLESHOOT_DATADICT_MSG; } - return(DB_CORRUPTION); + return(err); } /* If the encrypted tablespace is already opened, return success. */ - if (validate && is_encrypted && fil_space_get(id)) { + if (validate && is_encrypted && fil_space_get(space_id)) { return(DB_SUCCESS); } - fil_space_t* space = fil_space_create( - space_name, id, flags, purpose); + space = fil_space_create(space_name, space_id, flags, purpose); + + if (space == nullptr){ + return(DB_ERROR); + } /* We do not measure the size of the file, that is why we pass the 0 below */ - if (NULL == fil_node_create_low( - df.filepath(), 0, space, false, true, atomic_write)) { + const fil_node_t* file = shard->create_node( + df.filepath(), 0, space, false, true, atomic_write); + if (file == nullptr) { return(DB_ERROR); } @@ -4706,13 +5927,15 @@ fil_ibd_open( /* For encryption tablespace, initialize encryption information.*/ if (is_encrypted && !for_import) { + dberr_t err; byte* key = df.m_encryption_key; byte* iv = df.m_encryption_iv; + ut_ad(key && iv); - err = fil_set_encryption(space->id, Encryption::AES, - key, iv); + err = fil_set_encryption(space->id, Encryption::AES, key, iv); + if (err != DB_SUCCESS) { return(DB_ERROR); } @@ -4720,29 +5943,29 @@ fil_ibd_open( return(DB_SUCCESS); } -#endif /* !UNIV_HOTBACKUP */ -#ifdef UNIV_HOTBACKUP -/*******************************************************************//** -Allocates a file name for an old version of a single-table tablespace. +#else /* !UNIV_HOTBACKUP */ + +/** Allocates a file name for an old version of a single-table tablespace. The string must be freed by caller with ut_free()! +@param[in] name Original file name @return own: file name */ static char* -meb_make_ibbackup_old_name( -/*=======================*/ - const char* name) /*!< in: original file name */ +meb_make_ibbackup_old_name(const char* name) { - static const char suffix[] = "_ibbackup_old_vers_"; char* path; ulint len = strlen(name); + static const char suffix[] = "_ibbackup_old_vers_"; path = static_cast(ut_malloc_nokey(len + 15 + sizeof(suffix))); memcpy(path, name, len); memcpy(path + len, suffix, sizeof(suffix) - 1); + meb_sprintf_timestamp_without_extra_chars( path + len + sizeof(suffix) - 1); + return(path); } #endif /* UNIV_HOTBACKUP */ @@ -4762,75 +5985,71 @@ fil_space_read_name_and_filepath( char** filepath) { bool success = false; - *name = NULL; - *filepath = NULL; - mutex_enter(&fil_system->mutex); + *name = nullptr; + *filepath = nullptr; + + auto shard = fil_system->shard_by_id(space_id); - fil_space_t* space = fil_space_get_by_id(space_id); + shard->mutex_acquire(); + + fil_space_t* space = shard->get_space_by_id(space_id); + + if (space != nullptr) { - if (space != NULL) { *name = mem_strdup(space->name); - fil_node_t* node = UT_LIST_GET_FIRST(space->chain); - *filepath = mem_strdup(node->name); + *filepath = mem_strdup(space->files.front().name); success = true; } - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(success); } -/** Convert a file name to a tablespace name. +/** Convert a file name to a tablespace name. Strip the file name +prefix and suffix, leaving only databasename/tablename. @param[in] filename directory/databasename/tablename.ibd @return database/tablename string, to be freed with ut_free() */ char* -fil_path_to_space_name( - const char* filename) -{ - /* Strip the file name prefix and suffix, leaving - only databasename/tablename. */ - ulint filename_len = strlen(filename); - const char* end = filename + filename_len; -#ifdef HAVE_MEMRCHR - const char* tablename = 1 + static_cast( - memrchr(filename, OS_PATH_SEPARATOR, - filename_len)); - const char* dbname = 1 + static_cast( - memrchr(filename, OS_PATH_SEPARATOR, - tablename - filename - 1)); -#else /* HAVE_MEMRCHR */ - const char* tablename = filename; - const char* dbname = NULL; - - while (const char* t = static_cast( - memchr(tablename, OS_PATH_SEPARATOR, - end - tablename))) { - dbname = tablename; - tablename = t + 1; - } -#endif /* HAVE_MEMRCHR */ - - ut_ad(dbname != NULL); - ut_ad(tablename > dbname); - ut_ad(tablename < end); - ut_ad(end - tablename > 4); +fil_path_to_space_name(const char* filename) +{ + std::string path{filename}; + auto pos = path.find_last_of(Fil_path::SEPARATOR); + + ut_a(pos != std::string::npos && !Fil_path::is_separator(path.back())); + + std::string db_name = path.substr(0, pos); + std::string space_name = path.substr(pos + 1, path.length()); + + /* If it is a path such as a/b/c.ibd, ignore everything before 'b'. */ + pos = db_name.find_last_of(Fil_path::SEPARATOR); + + if (pos != std::string::npos){ + db_name = db_name.substr(pos + 1); + } char* name; - if (!memcmp(end - 4, DOT_IBD, 4)) { - name = mem_strdupl(dbname, (end - 4) - dbname); + if (Fil_path::has_ibd_suffix(space_name)) { + + /* fil_space_t::name always uses '/' . */ + + path = db_name; + path.push_back('/'); + + /* Strip the ".ibd" suffix. */ + path.append(space_name.substr(0, space_name.length() - 4)); + + name = mem_strdupl(path.c_str(), path.length()); - ut_ad(name[tablename - dbname - 1] == OS_PATH_SEPARATOR); -#if OS_PATH_SEPARATOR != '/' - /* space->name uses '/', not OS_PATH_SEPARATOR. */ - name[tablename - dbname - 1] = '/'; -#endif } else { - ut_ad(!memcmp(tablename, "undo", 4)); - name = mem_strdupl(filename, filename_len); + /* Must have an "undo" prefix. */ + ut_ad(space_name.find("undo") == 0); + + name = mem_strdupl(space_name.c_str(), space_name.length()); } return(name); @@ -4838,7 +6057,7 @@ fil_path_to_space_name( /** Open an ibd tablespace and add it to the InnoDB data structures. This is similar to fil_ibd_open() except that it is used while processing -the REDO log, so the data dictionary is not available and very little +the redo and DDL log, so the data dictionary is not available and very little validation is done. The tablespace name is extracted from the dbname/tablename.ibd portion of the filename, which assumes that the file is a file-per-table tablespace. Any name will do for now. General @@ -4846,166 +6065,127 @@ tablespace names will be read from the dictionary after it has been recovered. The tablespace flags are read at this time from the first page of the file in validate_for_recovery(). @param[in] space_id tablespace ID -@param[in] filename path/to/databasename/tablename.ibd -@param[out] space the tablespace, or NULL on error +@param[in] path path/to/databasename/tablename.ibd +@param[out] space the tablespace, or nullptr on error @return status of the operation */ -static fil_load_status -fil_ibd_open_for_recovery( - space_id_t space_id, - const char* filename, - fil_space_t*& space) +Fil_shard::ibd_open_for_recovery( + space_id_t space_id, + const std::string& path, + fil_space_t*& space) { /* If the a space is already in the file system cache with this space ID, then there is nothing to do. */ - mutex_enter(&fil_system->mutex); - space = fil_space_get_by_id(space_id); - mutex_exit(&fil_system->mutex); - if (space != NULL) { + mutex_acquire(); - /* Compare the filename we are trying to open with the - filename of all the nodes of the tablespace we opened - previously. Fail if it is different. */ + space = get_space_by_id(space_id); - fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + mutex_release(); - if (0 == strcmp(filename, node->name)) { - return (FIL_LOAD_OK); - } + const char* filename = path.c_str(); - /* The same space id can be mapped to many ibdata* files. - It depends on the innodb_data_file_path configuration. - In this case, traverse all the chain from the fil_node_t */ - if (space_id == TRX_SYS_SPACE) { + if (space != nullptr) { - for (fil_node_t* node1 = node; node1 != NULL; - node1 = UT_LIST_GET_NEXT(chain, node1)) { + ut_a(space->files.size() == 1); - if (0 == strcmp(filename, node1->name)) { - return (FIL_LOAD_OK); - } - } + const auto& file = space->files.front(); + + /* Compare the real paths. */ + if (Fil_path::equal(filename, file.name)) { + return(FIL_LOAD_OK); } #ifdef UNIV_HOTBACKUP - ib::trace_2() + ib::trace_2() #else /* UNIV_HOTBACKUP */ ib::info() #endif /* UNIV_HOTBACKUP */ << "Ignoring data file '" << filename << "' with space ID " << space->id - << ". Another data file called " << node->name - << " exists with the same space ID."; + << ". Another data file called '" << file.name + << "' exists with the same space ID"; + + space = nullptr; - space = NULL; return(FIL_LOAD_ID_CHANGED); } - Datafile file; + Datafile df; - file.set_filepath(filename); + df.set_filepath(filename); - if (file.open_read_only(false) != DB_SUCCESS) { + if (df.open_read_only(false) != DB_SUCCESS) { return(FIL_LOAD_NOT_FOUND); } - ut_ad(file.is_open()); - - os_offset_t size; + ut_ad(df.is_open()); /* Read and validate the first page of the tablespace. Assign a tablespace name based on the tablespace type. */ - switch (file.validate_for_recovery(space_id)) { - os_offset_t minimum_size; - case DB_SUCCESS: - - if (file.space_id() != space_id) { -#ifdef UNIV_HOTBACKUP - ib::trace_2() -#else /* UNIV_HOTBACKUP */ - ib::info() -#endif /* UNIV_HOTBACKUP */ - << "Ignoring data file '" - << file.filepath() - << "' with space ID " << file.space_id() - << ", since the redo log references " - << file.filepath() << " with space ID " - << space_id << "."; - return(FIL_LOAD_ID_CHANGED); - } - - /* Get and test the file size. */ - size = os_file_get_size(file.handle()); + dberr_t err = df.validate_for_recovery(space_id); - /* Every .ibd file is created >= FIL_IBD_FILE_INITIAL_SIZE - pages in size. Smaller files cannot be OK. */ - { - const page_size_t page_size(file.flags()); - minimum_size = FIL_IBD_FILE_INITIAL_SIZE - * page_size.physical(); - } - - if (size == static_cast(-1)) { - /* The following call prints an error message */ - os_file_get_last_error(true); - - ib::error() << "Could not measure the size of" - " single-table tablespace file '" - << file.filepath() << "'"; + ut_a(err == DB_SUCCESS); + ut_a(df.space_id() == space_id); - } else if (size < minimum_size) { -#ifndef UNIV_HOTBACKUP - ib::error() << "The size of tablespace file '" - << file.filepath() << "' is only " << size - << ", should be at least " << minimum_size - << "!"; -#else /* !UNIV_HOTBACKUP */ - /* In MEB, we work around this error. */ - file.set_space_id(SPACE_UNKNOWN); - file.set_flags(0); - break; -#endif /* !UNIV_HOTBACKUP */ - } else { - /* Everything is fine so far. */ - break; - } + /* Get and test the file size. */ + os_offset_t size = os_file_get_size(df.handle()); - /* Fall through to error handling */ + /* Every .ibd file is created >= 4 pages in size. + Smaller files cannot be OK. */ + os_offset_t minimum_size; - case DB_TABLESPACE_EXISTS: - return(FIL_LOAD_INVALID); + /* Every .ibd file is created >= FIL_IBD_FILE_INITIAL_SIZE + pages in size. Smaller files cannot be OK. */ + { + const page_size_t page_size(df.flags()); - case DB_TABLESPACE_NOT_FOUND: + minimum_size = FIL_IBD_FILE_INITIAL_SIZE * page_size.physical(); + } - /* Tablespace header doesn't contain the expected - tablespace ID. */ + if (size == static_cast(-1)) { + /* The following call prints an error message */ + os_file_get_last_error(true); - return(FIL_LOAD_MISMATCH); + ib::error() + << "Could not measure the size of" + " single-table tablespace file '" + << df.filepath() << "'"; - default: - return(FIL_LOAD_NOT_FOUND); + } else if (size < minimum_size) { +#ifndef UNIV_HOTBACKUP + ib::error() + << "The size of tablespace file '" + << df.filepath() << "' is only " << size + << ", should be at least " << minimum_size + << "!"; +#else + /* In MEB, we work around this error. */ + df.set_space_id(SPACE_UNKNOWN); + df.set_flags(0); +#endif /* !UNIV_HOTBACKUP */ } - ut_ad(space == NULL); + ut_ad(space == nullptr); #ifdef UNIV_HOTBACKUP - if (file.space_id() == SPACE_UNKNOWN || file.space_id() == 0) { + if (df.space_id() == SPACE_UNKNOWN || df.space_id() == 0) { char* new_path; - ib::info() << "Renaming tablespace file '" << file.filepath() - << "' with space ID " << file.space_id() << " to " - << file.name() << "_ibbackup_old_vers_" - << " because its size " << size << " is too small" - << " (< 4 pages 16 kB each), or the space id in the" - << " file header is not sensible. This can happen in" - << " an mysqlbackup run, and is not dangerous."; - file.close(); + ib::info() + << "Renaming tablespace file '" << df.filepath() + << "' with space ID " << df.space_id() << " to " + << df.name() << "_ibbackup_old_vers_" + " because its size " << df.size() << " is too small" + " (< 4 pages 16 kB each), or the space id in the" + " file header is not sensible. This can happen in" + " an mysqlbackup run, and is not dangerous."; + df.close(); - new_path = meb_make_ibbackup_old_name(file.filepath()); + new_path = meb_make_ibbackup_old_name(df.filepath()); bool success = os_file_rename( - innodb_data_file_key, file.filepath(), new_path); + innodb_data_file_key, df.filepath(), new_path); ut_a(success); @@ -5021,24 +6201,29 @@ fil_ibd_open_for_recovery( file than delete it, because if there is a bug, we do not want to destroy valuable data. */ - mutex_enter(&fil_system->mutex); - space = fil_space_get_by_id(space_id); - mutex_exit(&fil_system->mutex); + mutex_acquire(); + + space = get_space_by_id(space_id); - if (space != NULL) { - ib::info() << "Renaming data file '" << file.filepath() + mutex_release(); + + if (space != nullptr) { + + ib::info() + << "Renaming data file '" << df.filepath() << "' with space ID " << space_id << " to " - << file.name() + << df.name() << "_ibbackup_old_vers_ because space " << space->name << " with the same id was scanned" " earlier. This can happen if you have renamed tables" " during an mysqlbackup run."; - file.close(); - char* new_path = meb_make_ibbackup_old_name(file.filepath()); + df.close(); + + char* new_path = meb_make_ibbackup_old_name(df.filepath()); bool success = os_file_rename( - innodb_data_file_key, file.filepath(), new_path); + innodb_data_file_key, df.filepath(), new_path); ut_a(success); @@ -5049,44 +6234,51 @@ fil_ibd_open_for_recovery( std::string tablespace_name; #ifndef UNIV_HOTBACKUP - dd_filename_to_spacename(file.name(), &tablespace_name); + dd_filename_to_spacename(df.name(), &tablespace_name); #else /* During the apply-log operation, MEB already has translated the - file name, so file name to space name conversion is not required */ - tablespace_name = file.name(); -#endif + file name, so file name to space name conversin is not required. */ + + tablespace_name = df.name(); +#endif /* !UNIV_HOTBACKUP */ - space = fil_space_create( - tablespace_name.c_str(), space_id, - file.flags(), FIL_TYPE_TABLESPACE); + fil_system->mutex_acquire_all(); - if (space == NULL) { + space = space_create( + tablespace_name.c_str(), + space_id, df.flags(), FIL_TYPE_TABLESPACE); + + fil_system->mutex_release_all(); + + if (space == nullptr) { return(FIL_LOAD_INVALID); } - ut_ad(space->id == file.space_id()); + ut_ad(space->id == df.space_id()); ut_ad(space->id == space_id); /* We do not use the size information we have about the file, because the rounding formula for extents and pages is somewhat complex; we - let fil_node_open() do that task. */ + let fil_node_create() do that task. */ - if (!fil_node_create_low( - file.filepath(), 0, space, false, true, false)) { + const fil_node_t* file; - ut_error; - } + file = create_node(df.filepath(), 0, space, false, true, false); + + ut_a(file != nullptr); /* For encryption tablespace, initial encryption information. */ if (FSP_FLAGS_GET_ENCRYPTION(space->flags) - && file.m_encryption_key != NULL) { + && df.m_encryption_key != nullptr) { dberr_t err = fil_set_encryption( space->id, Encryption::AES, - file.m_encryption_key, file.m_encryption_iv); + df.m_encryption_key, df.m_encryption_iv); if (err != DB_SUCCESS) { - ib::error() << "Can't set encryption information for" + + ib::error() + << "Can't set encryption information for" " tablespace " << space->name << "!"; } } @@ -5095,17 +6287,46 @@ fil_ibd_open_for_recovery( return(FIL_LOAD_OK); } +/** Open an ibd tablespace and add it to the InnoDB data structures. +This is similar to fil_ibd_open() except that it is used while processing +the redo log, so the data dictionary is not available and very little +validation is done. The tablespace name is extracted from the +dbname/tablename.ibd portion of the filename, which assumes that the file +is a file-per-table tablespace. Any name will do for now. General +tablespace names will be read from the dictionary after it has been +recovered. The tablespace flags are read at this time from the first page +of the file in validate_for_recovery(). +@param[in] space_id tablespace ID +@param[in] path path/to/databasename/tablename.ibd +@param[out] space the tablespace, or nullptr on error +@return status of the operation */ +fil_load_status +Fil_system::ibd_open_for_recovery( + space_id_t space_id, + const std::string& path, + fil_space_t*& space) +{ + /* System tablespace open should never come here. It should be + opened explicitly using the config path. */ + + ut_a(space_id != TRX_SYS_SPACE); + + auto shard = shard_by_id(space_id); + + return(shard->ibd_open_for_recovery(space_id, path, space)); +} + #ifndef UNIV_HOTBACKUP -/*******************************************************************//** -Report that a tablespace for a table was not found. */ + +/** Report that a tablespace for a table was not found. +@param[in] name Table name +@param[in] space_id Table's space ID */ static void -fil_report_missing_tablespace( -/*===========================*/ - const char* name, /*!< in: table name */ - space_id_t space_id) /*!< in: table's space id */ +fil_report_missing_tablespace(const char* name, space_id_t space_id) { - ib::error() << "Table " << name + ib::error() + << "Table " << name << " in the InnoDB data dictionary has tablespace id " << space_id << "," " but tablespace with that id or name does not exist. Have" @@ -5113,12 +6334,11 @@ fil_report_missing_tablespace( } /** Returns true if a matching tablespace exists in the InnoDB tablespace -memory cache. Note that if we have not done a crash recovery at the database -startup, there may be many tablespaces which are not yet in the memory cache. -@param[in] id Tablespace ID +memory cache. +@param[in] space_id Tablespace ID @param[in] name Tablespace name used in fil_space_create(). -@param[in] print_err_if_not_exist Print detailed error information to the +@param[in] print_err Print detailed error information to the error log if a matching tablespace is not found from memory. @param[in] adjust_space Whether to adjust space id on mismatch @@ -5126,79 +6346,67 @@ startup, there may be many tablespaces which are not yet in the memory cache. @param[in] table_id table id @return true if a matching tablespace exists in the memory cache */ bool -fil_space_for_table_exists_in_mem( - space_id_t id, +Fil_shard::space_check_exists( + space_id_t space_id, const char* name, - bool print_err_if_not_exist, + bool print_err, bool adjust_space, mem_heap_t* heap, table_id_t table_id) { - fil_space_t* fnamespace = NULL; - fil_space_t* space; + fil_space_t* fnamespace = nullptr; - ut_ad(fil_system); - - mutex_enter(&fil_system->mutex); + mutex_acquire(); /* Look if there is a space with the same id */ + fil_space_t* space = get_space_by_id(space_id); - space = fil_space_get_by_id(id); + /* name is nullptr when replaying a DELETE ddl log. */ + if (name == nullptr) { - /* name is NULL when replay DELETE ddl log. */ - if (name == NULL) { - mutex_exit(&fil_system->mutex); + mutex_release(); - if (space != NULL) { - return(true); - } else { - return(false); - } + return(space != nullptr); } - if (space != NULL + if (space != nullptr && FSP_FLAGS_GET_SHARED(space->flags) && adjust_space && srv_sys_tablespaces_open && 0 == strncmp(space->name, general_space_name, strlen(general_space_name))) { - /* This name was assigned during recovery in - fil_ibd_open_for_recovery(). This general tablespace - was opened from an MLOG_FILE_OPEN log entry where the - tablespace name does not exist. Replace the temporary - name with this name and return this space. */ - - fil_system->names.erase(space->name); - ut_free(space->name); + char* old_name = space->name; + char* new_name = mem_strdup(name); - space->name = mem_strdup(name); + update_space_name_map(space, new_name); - auto it = fil_system->names.insert( - Names::value_type(space->name, space)); + space->name = new_name; - ut_a(it.second); + ut_free(old_name); - mutex_exit(&fil_system->mutex); + mutex_release(); return(true); - } - if (space != NULL) { + } else if (space != nullptr) { + if (FSP_FLAGS_GET_SHARED(space->flags) && !srv_sys_tablespaces_open) { /* No need to check the name */ - mutex_exit(&fil_system->mutex); + mutex_release(); + return(true); } /* If this space has the expected name, use it. */ - fnamespace = fil_space_get_by_name(name); + fnamespace = get_space_by_name(name); + if (space == fnamespace) { - /* Found */ - mutex_exit(&fil_system->mutex); + /* Found */ + mutex_release(); return(true); } @@ -5210,90 +6418,106 @@ fil_space_for_table_exists_in_mem( mismatching space are between a user table and its temp table, we shall adjust the ibd file name according to system table info */ if (adjust_space - && space != NULL + && space != nullptr && row_is_mysql_tmp_table_name(space->name) && !row_is_mysql_tmp_table_name(name)) { - /* Atomic DDL's "ddl_log" will adjust the tablespace name */ - mutex_exit(&fil_system->mutex); + /* Atomic DDL's "ddl_log" will adjust the tablespace name. */ + mutex_release(); return(true); - } - if (!print_err_if_not_exist) { + } else if (!print_err) { - mutex_exit(&fil_system->mutex); + ; - return(false); - } + } else if (space == nullptr) { - if (space == NULL) { - if (fnamespace == NULL) { - if (print_err_if_not_exist) { - fil_report_missing_tablespace(name, id); + if (fnamespace == nullptr) { + + if (print_err) { + fil_report_missing_tablespace(name, space_id); } + } else { - ib::error() << "Table " << name << " in InnoDB data" - " dictionary has tablespace id " << id + ib::error() + << "Table " << name << " in InnoDB data" + " dictionary has tablespace id " << space_id << ", but a tablespace with that id does not" " exist. There is a tablespace of name " << fnamespace->name << " and id " << fnamespace->id << ", though. Have you" " deleted or moved .ibd files?"; } -error_exit: - ib::warn() << TROUBLESHOOT_DATADICT_MSG; - mutex_exit(&fil_system->mutex); + ib::warn() << TROUBLESHOOT_DATADICT_MSG; - return(false); - } + } else if (0 != strcmp(space->name, name)) { - if (0 != strcmp(space->name, name)) { + ib::error() + << "Table " << name << " in InnoDB data dictionary" + " has tablespace id " << space_id << ", but the" + " tablespace with that id has name " + << space->name << ". Have you deleted or moved .ibd" + " files?"; - ib::error() << "Table " << name << " in InnoDB data dictionary" - " has tablespace id " << id << ", but the tablespace" - " with that id has name " << space->name << "." - " Have you deleted or moved .ibd files?"; + if (fnamespace != nullptr) { - if (fnamespace != NULL) { - ib::error() << "There is a tablespace with the right" + ib::error() + << "There is a tablespace with the right" " name: " << fnamespace->name << ", but its id" " is " << fnamespace->id << "."; } - goto error_exit; + ib::warn() << TROUBLESHOOT_DATADICT_MSG; } - mutex_exit(&fil_system->mutex); + mutex_release(); return(false); } + +/** Returns true if a matching tablespace exists in the InnoDB tablespace +memory cache. +@param[in] space_id Tablespace ID +@param[in] name Tablespace name used in space_create(). +@param[in] print_err Print detailed error information to the + error log if a matching tablespace is + not found from memory. +@param[in] adjust_space Whether to adjust space id on mismatch +@param[in] heap Heap memory +@param[in] table_id table ID +@return true if a matching tablespace exists in the memory cache */ +bool +fil_space_exists_in_mem( + space_id_t space_id, + const char* name, + bool print_err, + bool adjust_space, + mem_heap_t* heap, + table_id_t table_id) +{ + auto shard = fil_system->shard_by_id(space_id); + + return(shard->space_check_exists( + space_id, name, print_err, adjust_space, heap, table_id)); +} #endif /* !UNIV_HOTBACKUP */ /** Return the space ID based on the tablespace name. The tablespace must be found in the tablespace memory cache. -This call is made from external to this module, so the mutex is not owned. -@param[in] tablespace Tablespace name +@param[in] name Tablespace name @return space ID if tablespace found, SPACE_UNKNOWN if space not. */ space_id_t -fil_space_get_id_by_name( - const char* tablespace) +fil_space_get_id_by_name(const char* name) { - mutex_enter(&fil_system->mutex); - - /* Search for a space with the same name. */ - fil_space_t* space = fil_space_get_by_name(tablespace); - space_id_t id = (space == NULL) ? SPACE_UNKNOWN : space->id; + auto space = fil_system->get_space_by_name(name); - mutex_exit(&fil_system->mutex); - - return(id); + return((space == nullptr) ? SPACE_UNKNOWN : space->id); } -/** -Fill the pages with NULs -@param[in] node File node +/** Fill the pages with NULs +@param[in] file Tablespace file @param[in] page_size physical page size @param[in] start Offset from the start of the file in bytes @param[in] len Length in bytes @@ -5303,7 +6527,7 @@ Fill the pages with NULs static dberr_t fil_write_zeros( - const fil_node_t* node, + const fil_node_t* file, ulint page_size, os_offset_t start, ulint len, @@ -5313,8 +6537,10 @@ fil_write_zeros( /* Extend at most 1M at a time */ ulint n_bytes = ut_min(static_cast(1024 * 1024), len); - byte* ptr = reinterpret_cast(ut_zalloc_nokey(n_bytes - + page_size)); + + byte* ptr = reinterpret_cast( + ut_zalloc_nokey(n_bytes + page_size)); + byte* buf = reinterpret_cast(ut_align(ptr, page_size)); os_offset_t offset = start; @@ -5326,13 +6552,13 @@ fil_write_zeros( #ifdef UNIV_HOTBACKUP err = os_file_write( - request, node->name, node->handle, buf, offset, + request, file->name, file->handle, buf, offset, n_bytes); #else /* UNIV_HOTBACKUP */ err = os_aio_func( - request, OS_AIO_SYNC, node->name, - node->handle, buf, offset, n_bytes, read_only_mode, - NULL, NULL); + request, AIO_mode::SYNC, file->name, + file->handle, buf, offset, n_bytes, read_only_mode, + nullptr, nullptr); #endif /* UNIV_HOTBACKUP */ if (err != DB_SUCCESS) { @@ -5353,13 +6579,11 @@ fil_write_zeros( } /** Try to extend a tablespace if it is smaller than the specified size. -@param[in,out] space tablespace -@param[in] size desired size in pages +@param[in,out] space tablespace +@param[in] size desired size in pages @return whether the tablespace is at least as big as requested */ bool -fil_space_extend( - fil_space_t* space, - page_no_t size) +Fil_shard::space_extend(fil_space_t* space, page_no_t size) { /* In read-only mode we allow write to shared temporary tablespace as intrinsic table created by Optimizer reside in this tablespace. */ @@ -5370,87 +6594,133 @@ fil_space_extend( space->print_xdes_pages("xdes_pages.log");); #endif /* !UNIV_HOTBACKUP */ -retry: - + fil_node_t* file; + bool slot; + size_t phy_page_size; bool success = true; - fil_mutex_enter_and_prepare_for_io(space->id); +#ifdef UNIV_HOTBACKUP + page_no_t prev_size = 0; +#endif /* UNIV_HOTBACKUP */ + + for (;;) { - if (space->size >= size) { - /* Space already big enough */ - mutex_exit(&fil_system->mutex); - return(true); - } + slot = mutex_acquire_and_get_space(space->id, space); + + /* Note:If the file is being opened for the first time then + we don't have the file physical size. There is no guarantee + that the file has been opened at this stage. */ + + if (size < space->size) { + + /* Space already big enough */ + mutex_release(); + + if (slot) { + release_open_slot(m_id); + } + + return(true); + } + + file = &space->files.back(); + + page_size_t page_size(space->flags); + + phy_page_size = page_size.physical(); #ifdef UNIV_HOTBACKUP - page_size_t page_length(space->flags); - ulint actual_size = space->size; - ib::trace() - << "Extending space id : " << space->id << ", space name : " - << space->name << ", space size : " << actual_size << " pages," - << " desired space size : " << size << " pages," - << " page size : " << page_length.physical(); + prev_size = space->size; + + ib::trace() + << "Extending space id : " << space->id + << ", space name : " << space->name + << ", space size : " << space->size + << " page, page size : " << phy_page_size; #endif /* UNIV_HOTBACKUP */ - page_size_t pageSize(space->flags); - const ulint page_size = pageSize.physical(); - fil_node_t* node = UT_LIST_GET_LAST(space->chain); + if (file->in_use == 0) { + + /* Mark this file as undergoing extension. This flag + is used by other threads to wait for the extension + opereation to finish or wait for open to complete. */ + + ++file->in_use; + + break; + } + + if (slot) { + release_open_slot(m_id); + } - if (node->in_use == 0) { - /* Mark this node as undergoing extension. This flag - is used by other threads to wait for the extension - opereation to finish or wait for open to complete. */ - ++node->in_use; - } else { /* Another thread is currently using the file. Wait for it to finish. It'd have been better to use an event driven mechanism but the entire module is peppered with polling code. */ - mutex_exit(&fil_system->mutex); + mutex_release(); + os_thread_sleep(100000); - goto retry; } - if (!fil_node_prepare_for_io(node, fil_system, space, true)) { + bool opened = prepare_file_for_io(file, true); + + if (slot) { + release_open_slot(m_id); + } + + if (!opened) { + /* The tablespace data file, such as .ibd file, is missing */ - ut_a(node->in_use > 0); - --node->in_use; - mutex_exit(&fil_system->mutex); + ut_a(file->in_use > 0); + --file->in_use; + + mutex_release(); return(false); } - /* At this point it is safe to release fil_system mutex. No - other thread can rename, delete or close the file because - we have set the node->in_use flag. */ - mutex_exit(&fil_system->mutex); + ut_a(file->is_open); - page_no_t pages_added; + if (size <= space->size) { + + ut_a(file->in_use > 0); + --file->in_use; + + complete_io(file, IORequestRead); - /* Note: This code is going to be executed independent of FusionIO HW - if the OS supports posix_fallocate() */ + mutex_release(); - ut_ad(size > space->size); + return(true); + } + + /* At this point it is safe to release the shard mutex. No + other thread can rename, delete or close the file because + we have set the file->in_use flag. */ + + mutex_release(); + + page_no_t pages_added; + os_offset_t node_start = os_file_get_size(file->handle); - os_offset_t node_start = os_file_get_size(node->handle); ut_a(node_start != (os_offset_t) -1); - /* Node first page number */ - page_no_t node_first_page = space->size - node->size; + /* File first page number */ + page_no_t node_first_page = space->size - file->size; - /* Number of physical pages in the node/file */ + /* Number of physical pages in the file */ page_no_t n_node_physical_pages - = static_cast(node_start / page_size); + = static_cast(node_start / phy_page_size); - /* Number of pages to extend in the node/file */ + /* Number of pages to extend in the file */ page_no_t n_node_extend; - n_node_extend = size - (node_first_page + node->size); + n_node_extend = size - (node_first_page + file->size); /* If we already have enough physical pages to satisfy the - extend request on the node then ignore it */ - if (node->size + n_node_extend > n_node_physical_pages) { + extend request on the file then ignore it */ + if (file->size + n_node_extend > n_node_physical_pages) { DBUG_EXECUTE_IF("ib_crash_during_tablespace_extension", DBUG_SUICIDE();); @@ -5458,12 +6728,16 @@ fil_space_extend( os_offset_t len; dberr_t err = DB_SUCCESS; - len = ((node->size + n_node_extend) * page_size) - node_start; + len = ((file->size + n_node_extend) + * phy_page_size) - node_start; + ut_ad(len > 0); #if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) /* This is required by FusionIO HW/Firmware */ - int ret = posix_fallocate(node->handle.m_file, node_start, len); + + int ret = posix_fallocate( + file->handle.m_file, node_start, len); /* We already pass the valid offset and len in, if EINVAL is returned, it could only mean that the file system doesn't @@ -5474,7 +6748,7 @@ fil_space_extend( ib::error() << "posix_fallocate(): Failed to preallocate" " data for file " - << node->name << ", desired size " + << file->name << ", desired size " << len << " bytes." " Operating system error number " << ret << ". Check" @@ -5488,7 +6762,7 @@ fil_space_extend( } #endif /* NO_FALLOCATE || !UNIV_LINUX */ - if (!node->atomic_write || err == DB_IO_ERROR) { + if (!file->atomic_write || err == DB_IO_ERROR) { bool read_only_mode; @@ -5496,28 +6770,28 @@ fil_space_extend( ? false : srv_read_only_mode); err = fil_write_zeros( - node, page_size, node_start, + file, phy_page_size, node_start, static_cast(len), read_only_mode); if (err != DB_SUCCESS) { ib::warn() << "Error while writing " << len - << " zeroes to " << node->name + << " zeroes to " << file->name << " starting at offset " << node_start; } } /* Check how many pages actually added */ - os_offset_t end = os_file_get_size(node->handle); + os_offset_t end = os_file_get_size(file->handle); ut_a(end != static_cast(-1) && end >= node_start); os_has_said_disk_full = !(success = (end == node_start + len)); - pages_added = static_cast(end / page_size); + pages_added = static_cast(end / phy_page_size); - ut_a(pages_added >= node->size); - pages_added -= node->size; + ut_a(pages_added >= file->size); + pages_added -= file->size; } else { success = true; @@ -5525,23 +6799,24 @@ fil_space_extend( os_has_said_disk_full = FALSE; } - mutex_enter(&fil_system->mutex); + mutex_acquire(); - node->size += pages_added; + file->size += pages_added; space->size += pages_added; - ut_a(node->in_use > 0); - --node->in_use; + ut_a(file->in_use > 0); + --file->in_use; - fil_node_complete_io(node, fil_system, IORequestWrite); + complete_io(file, IORequestWrite); #ifndef UNIV_HOTBACKUP /* Keep the last data file size info up to date, rounded to full megabytes */ page_no_t pages_per_mb = static_cast( - (1024 * 1024) / page_size); - page_no_t size_in_pages = ((node->size / pages_per_mb) - * pages_per_mb); + (1024 * 1024) / phy_page_size); + + page_no_t size_in_pages = + ((file->size / pages_per_mb) * pages_per_mb); if (space->id == TRX_SYS_SPACE) { srv_sys_space.set_last_file_size(size_in_pages); @@ -5551,49 +6826,58 @@ fil_space_extend( #else /* !UNIV_HOTBACKUP */ ib::trace_2() << "Extended space : " << space->name << " from " - << actual_size << " pages to " << space->size << " pages " + << prev_size << " pages to " << space->size << " pages " << ", desired space size : " << size << " pages."; #endif /* !UNIV_HOTBACKUP */ - mutex_exit(&fil_system->mutex); + space_flush(space->id); - fil_flush(space->id); + mutex_release(); return(success); } +/** Try to extend a tablespace if it is smaller than the specified size. +@param[in,out] space tablespace +@param[in] size desired size in pages +@return whether the tablespace is at least as big as requested */ +bool +fil_space_extend(fil_space_t* space, page_no_t size) +{ + auto shard = fil_system->shard_by_id(space->id); + + return(shard->space_extend(space, size)); +} + #ifdef UNIV_HOTBACKUP -/** Extends all tablespaces to the size stored in the space header. -During the mysqlbackup --apply-log phase we extended the spaces -on-demand so that log records could be applied, but that may have left -spaces still too small compared to the size stored in the space -header. */ +/** Extends all tablespaces to the size stored in the space header. During the +mysqlbackup --apply-log phase we extended the spaces on-demand so that log +records could be applied, but that may have left spaces still too small +compared to the size stored in the space header. */ void -meb_extend_tablespaces_to_stored_len(void) +Fil_shard::meb_extend_tablespaces_to_stored_len() { - byte* buf1; - byte* buf; - ulint actual_size; - ulint size_in_header; - dberr_t error; - bool success; + ut_ad(mutex_owned()); - buf1 = static_cast(ut_malloc_nokey(2 * UNIV_PAGE_SIZE)); - buf = static_cast(ut_align(buf1, UNIV_PAGE_SIZE)); + byte* buf = static_cast(ut_malloc_nokey(UNIV_PAGE_SIZE)); + + ut_a(buf != nullptr); - mutex_enter(&fil_system->mutex); + for (auto& elem : m_spaces) { - for (fil_space_t* space = UT_LIST_GET_FIRST(fil_system->space_list); - space != NULL; - space = UT_LIST_GET_NEXT(space_list, space)) { + auto space = elem.second; ut_a(space->purpose == FIL_TYPE_TABLESPACE); - mutex_exit(&fil_system->mutex); /* no need to protect with a - mutex, because this is a - single-threaded operation */ + /* No need to protect with a mutex, because this is + a single-threaded operation */ + + mutex_release(); + + dberr_t error; const page_size_t page_size(space->flags); + error = fil_read( page_id_t(space->id, 0), page_size, @@ -5601,25 +6885,41 @@ meb_extend_tablespaces_to_stored_len(void) ut_a(error == DB_SUCCESS); + ulint size_in_header; + size_in_header = fsp_header_get_field(buf, FSP_SIZE); - success = fil_space_extend(space, size_in_header); + bool success; + + success = space_extend(space, size_in_header); + if (!success) { - ib::error() << "Could not extend the tablespace of " + + ib::error() + << "Could not extend the tablespace of " << space->name << " to the size stored in" " header, " << size_in_header << " pages;" - " size after extension " << actual_size + " size after extension " << 0 << " pages. Check that you have free disk" " space and retry!"; + ut_a(success); } - mutex_enter(&fil_system->mutex); + mutex_acquire(); } - mutex_exit(&fil_system->mutex); + ut_free(buf); +} - ut_free(buf1); +/** Extends all tablespaces to the size stored in the space header. During the +mysqlbackup --apply-log phase we extended the spaces on-demand so that log +records could be applied, but that may have left spaces still too small +compared to the size stored in the space header. */ +void +meb_extend_tablespaces_to_stored_len() +{ + fil_system->meb_extend_tablespaces_to_stored_len(); } /** Determine if file is intermediate / temporary. These files are @@ -5628,15 +6928,18 @@ created during reorganize partition, rename tables, add / drop columns etc. @retvalue true if it is intermediate file @retvalue false if it is normal file */ bool -meb_is_intermediate_file( - const std::string& filepath) +meb_is_intermediate_file(const std::string& filepath) { - std::string file_name = filepath; + std::string file_name = filepath; - /* extract file name from relative or absolute file name */ - std::size_t pos = file_name.rfind(OS_PATH_SEPARATOR); - if (pos != std::string::npos) { - file_name = file_name.substr(++pos); + { + /* extract file name from relative or absolute file name */ + auto pos = file_name.rfind(OS_PATH_SEPARATOR); + + if (pos != std::string::npos) { + ++pos; + file_name = file_name.substr(pos); + } } transform( @@ -5644,7 +6947,7 @@ meb_is_intermediate_file( ::tolower); if (file_name[0] != '#') { - pos = file_name.rfind("#tmp#.ibd"); + auto pos = file_name.rfind("#tmp#.ibd"); if (pos != std::string::npos) { return(true); } else { @@ -5652,16 +6955,13 @@ meb_is_intermediate_file( } } - std::vector file_name_patterns = { "#sql-", "#sql2-", - "#tmp#", "#ren#" }; + static std::vector prefixes = { + "#sql-", "#sql2-", "#tmp#", "#ren#" }; /* search for the unsupported patterns */ - for (auto itr = file_name_patterns.begin(); - itr != file_name_patterns.end(); - ++itr) { + for (const auto& prefix : prefixes) { - if (0 == std::strncmp(file_name.c_str(), - itr->c_str(), itr->length())) { + if (Fil_path::has_prefix(file_name, prefix)) { return(true); } } @@ -5678,152 +6978,175 @@ tablespaces directories identified at the time memory cache created. @param[in, out] tablespace Tablespace name @return space ID if tablespace found, SPACE_UNKNOWN if not found. */ space_id_t -meb_fil_space_get_rem_gen_ts_id_by_name( - std::string& tablespace) -{ - space_id_t space_id = SPACE_UNKNOWN; - size_t pos = std::string::npos; - std::string newpath; - for (auto itr = rem_gen_ts_dirs.begin(); - itr != rem_gen_ts_dirs.end(); - ++itr) { - newpath = *itr; - pos = tablespace.rfind(OS_PATH_SEPARATOR); +meb_fil_space_get_rem_gen_ts_id_by_name(std::string& tablespace) +{ + space_id_t space_id = SPACE_UNKNOWN; + + for (auto newpath : rem_gen_ts_dirs) { + + auto pos = tablespace.rfind(OS_PATH_SEPARATOR); + if (pos == std::string::npos) { break; } + newpath += tablespace.substr(pos); + space_id = fil_space_get_id_by_name(newpath.c_str()); + if (space_id != SPACE_UNKNOWN) { tablespace = newpath; break; } } - return space_id; + + return(space_id); } /** Tablespace item during recovery */ -struct file_name_t { +struct MEB_file_name { + /** Constructor */ + MEB_file_name(std::string name, bool deleted) + : m_name(name), m_space(), m_deleted(deleted) {} + /** Tablespace file name (MLOG_FILE_NAME) */ - std::string name; + std::string m_name; + /** Tablespace object (NULL if not valid or not found) */ - fil_space_t* space; - /** Whether the tablespace has been deleted */ - bool deleted; + fil_space_t* m_space; - /** Constructor */ - file_name_t(std::string name_, bool deleted_) : - name(name_), space(NULL), deleted (deleted_) {} + /** Whether the tablespace has been deleted */ + bool m_deleted; }; /** Map of dirty tablespaces during recovery */ -typedef std::map< - space_id_t, - file_name_t, +using MEB_recv_spaces = std::map< + space_id_t, MEB_file_name, std::less, - ut_allocator > > recv_spaces_t; + ut_allocator>>; -static recv_spaces_t recv_spaces; +static MEB_recv_spaces recv_spaces; -/** Process a file name from a MLOG_FILE_* record. -@param[in,out] name file name -@param[in] len length of the file name -@param[in] space_id the tablespace ID -@param[in] deleted whether this is a MLOG_FILE_DELETE record */ +/** Set the keys for an encrypted tablespace. +@param[in] space Tablespace for which to set the key */ static void -meb_name_process( +meb_set_encryption_key(const fil_space_t* space) +{ + ut_ad(FSP_FLAGS_GET_ENCRYPTION(space->flags)); + + for (auto& key : *recv_sys->keys) { + + if (key.space_id != space->id) { + continue; + } + + dberr_t err; + + err = fil_set_encryption( + space->id, Encryption::AES, key.ptr, key.iv); + + if (err != DB_SUCCESS) { + + ib::error() + << "Can't set encryption information" + << " for tablespace" << space->name + << "!"; + } + + ut_free(key.iv); + ut_free(key.ptr); + + key.iv = nullptr; + key.ptr = nullptr; + key.space_id = 0; + } +} + +/** Process a file name passed as an input +Wrapper around meb_name_process() +@param[in,out] name absolute path of tablespace file +@param[in] space_id The tablespace ID +@param[in] deleted true if MLOG_FILE_DELETE */ +void +Fil_system::meb_name_process( char* name, - ulint len, space_id_t space_id, bool deleted) { ut_ad(space_id != TRX_SYS_SPACE); - /* We will also insert space=NULL into the map, so that + /* We will also insert space=nullptr into the map, so that further checks can ensure that a MLOG_FILE_NAME record was scanned before applying any page records for the space_id. */ - os_normalize_path(name); - file_name_t fname(std::string(name, len - 1), deleted); - std::pair p = recv_spaces.insert( - std::make_pair(space_id, fname)); + Fil_path::normalize(name); + + size_t len = std::strlen(name); + + MEB_file_name fname(std::string(name, len - 1), deleted); + + auto p = recv_spaces.insert(std::make_pair(space_id, fname)); + ut_ad(p.first->first == space_id); - file_name_t& f = p.first->second; + MEB_file_name& f = p.first->second; if (deleted) { /* Got MLOG_FILE_DELETE */ - if (!p.second && !f.deleted) { - f.deleted = true; - if (f.space != NULL) { - f.space = NULL; + if (!p.second && !f.m_deleted) { + + f.m_deleted = true; + + if (f.m_space != nullptr) { + f.m_space = nullptr; } } - ut_ad(f.space == NULL); - } else if (p.second /* the first MLOG_FILE_NAME or MLOG_FILE_RENAME2 */ - || f.name != fname.name) { + ut_ad(f.m_space == nullptr); + + } else if (p.second || f.m_name != fname.m_name) { + fil_space_t* space; /* Check if the tablespace file exists and contains the space_id. If not, ignore the file after displaying a note. Abort if there are multiple files with the same space_id. */ - switch (fil_ibd_open_for_recovery(space_id, name, space)) { + + switch (ibd_open_for_recovery(space_id, name, space)) { case FIL_LOAD_OK: - ut_ad(space != NULL); + ut_ad(space != nullptr); /* For encrypted tablespace, set key and iv. */ if (FSP_FLAGS_GET_ENCRYPTION(space->flags) - && recv_sys->keys != NULL) { - dberr_t err; - recv_sys_t::Encryption_Keys::iterator it; - - for (it = recv_sys->keys->begin(); - it != recv_sys->keys->end(); - it++) { - if (it->space_id == space->id) { - err = fil_set_encryption( - space->id, - Encryption::AES, - it->ptr, - it->iv); - if (err != DB_SUCCESS) { - ib::error() - << "Can't set" - " encryption" - " information" - " for" - " tablespace" - << space->name - << "!"; - } - ut_free(it->ptr); - ut_free(it->iv); - it->ptr = NULL; - it->iv = NULL; - it->space_id = 0; - } - } + && recv_sys->keys != nullptr) { + + meb_set_encryption_key(space); } - if (f.space == NULL || f.space == space) { - f.name = fname.name; - f.space = space; - f.deleted = false; + if (f.m_space == nullptr || f.m_space == space) { + + f.m_name = fname.m_name; + f.m_space = space; + f.m_deleted = false; + } else { - ib::error() << "Tablespace " << space_id + + ib::error() + << "Tablespace " << space_id << " has been found in two places: '" - << f.name << "' and '" << name << "'." + << f.m_name << "' and '" << name << "'." " You must delete one of them."; + recv_sys->found_corrupt_fs = true; } break; case FIL_LOAD_ID_CHANGED: - ut_ad(space == NULL); + ut_ad(space == nullptr); + ib::trace() << "Ignoring file " << name << " for space-id mismatch " << space_id; @@ -5833,17 +7156,17 @@ meb_name_process( /* No matching tablespace was found; maybe it was renamed, and we will find a subsequent MLOG_FILE_* record. */ - ut_ad(space == NULL); + ut_ad(space == nullptr); break; case FIL_LOAD_INVALID: - ut_ad(space == NULL); - ib::warn() - << "Invalid tablespace " << name; + ut_ad(space == nullptr); + + ib::warn() << "Invalid tablespace " << name; break; case FIL_LOAD_MISMATCH: - ut_ad(space == NULL); + ut_ad(space == nullptr); break; } } @@ -5852,21 +7175,15 @@ meb_name_process( /** Process a file name passed as an input Wrapper around meb_name_process() @param[in] name absolute path of tablespace file -@param[in] space_id the tablespace ID -@retval true if able to process file successfully. -@retval false if unable to process the file */ +@param[in] space_id the tablespace ID */ void meb_fil_name_process( const char* name, space_id_t space_id) { - size_t length = strlen(name); - ++length; - - char* file_name = static_cast(ut_malloc_nokey(length)); - strncpy(file_name, name,length); + char* file_name = static_cast(mem_strdup(name)); - meb_name_process(file_name, length, space_id, false); + fil_system->meb_name_process(file_name, space_id, false); ut_free(file_name); } @@ -5885,168 +7202,331 @@ void meb_make_abs_file_path( const std::string& name, ulint flags, - ulint space_id, + space_id_t space_id, std::string& absolute_path, std::string& tablespace_name) { Datafile df; - std::string file_name = name; - size_t pos = std::string::npos; + std::string file_name = name; + + if (Fil_path::is_absolute_path(file_name.c_str())) { - if (is_absolute_path(file_name.c_str())) { if (replay_in_datadir) { df.set_filepath(file_name.c_str()); } else { - pos = file_name.rfind(OS_PATH_SEPARATOR); + auto pos = file_name.rfind(OS_PATH_SEPARATOR); /* if it is file per tablespace, then include the schema directory as well */ - if (fsp_is_file_per_table(space_id, flags) && - pos != std::string::npos) { - pos = file_name.rfind(OS_PATH_SEPARATOR, pos-1); + if (fsp_is_file_per_table(space_id, flags) + && pos != std::string::npos) { + + pos = file_name.rfind( + OS_PATH_SEPARATOR, pos - 1); } if (pos == std::string::npos) { - ib::fatal() << "Could not extract the tabelspace file " - << "name from the in the path : " << name; + ib::fatal() + << "Could not extract the tabelspace" + << " file name from the in the path : " + << name; } ++pos; + file_name = file_name.substr(pos); - df.make_filepath(fil_path_to_mysql_datadir, - file_name.c_str(), IBD); + + df.make_filepath( + MySQL_datadir_path, file_name.c_str(), IBD); } + } else { - pos = file_name.find(OS_PATH_SEPARATOR); + auto pos = file_name.find(OS_PATH_SEPARATOR); + /* Remove the cur dir from the path as this will cause the path name mismatch when we try to find out the space_id based on tablespace name */ + if (file_name.substr(0, pos) == ".") { ++pos; file_name = file_name.substr(pos); } - df.make_filepath(fil_path_to_mysql_datadir, - file_name.c_str(), IBD); + + df.make_filepath(MySQL_datadir_path, file_name.c_str(), IBD); } df.set_flags(flags); df.set_space_id(space_id); - df.set_name(NULL); + df.set_name(nullptr); + absolute_path = df.filepath(); + tablespace_name = df.name(); } + +/** Process a MLOG_FILE_CREATE redo record. +@param[in] page_id Page id of the redo log record +@param[in] flags Tablespace flags +@param[in] name Tablespace filename */ +static +void +meb_tablespace_redo_create( + const page_id_t& page_id, + ulint flags, + const char* name) +{ + std::string abs_file_path; + std::string tablespace_name; + + meb_make_abs_file_path( + name, flags, page_id.space(), abs_file_path, tablespace_name); + + if (!meb_replay_file_ops + || meb_is_intermediate_file(abs_file_path.c_str()) + || fil_space_get(page_id.space()) + || fil_space_get_id_by_name(tablespace_name.c_str()) + != SPACE_UNKNOWN + || meb_fil_space_get_rem_gen_ts_id_by_name(tablespace_name) + != SPACE_UNKNOWN) { + + /* Don't create table while :- + 1. scanning the redo logs during backup + 2. apply-log on a partial backup + 3. if it is intermediate file + 4. tablespace is already loaded in memory + 5. tablespace is a remote general tablespace which is + already loaded for recovery/apply-log from different + directory path */ + + ib::trace() + << "Ignoring the log record. No need to " + << "create the tablespace : " << abs_file_path; + } else { + + auto it = recv_spaces.find(page_id.space()); + + if (it == recv_spaces.end() + || it->second.m_name != abs_file_path) { + + ib::trace() + << "Creating the tablespace : " + << abs_file_path + << ", space_id : " << page_id.space(); + + dberr_t ret = fil_ibd_create( + page_id.space(), tablespace_name.c_str(), + abs_file_path.c_str(), + flags, FIL_IBD_FILE_INITIAL_SIZE); + + if (ret != DB_SUCCESS) { + + ib::fatal() + << "Could not create the tablespace : " + << abs_file_path << " with space Id : " + << page_id.space(); + } + } + } +} + +/** Process a MLOG_FILE_RENAME redo record. +@param[in] page_id Page id of the redo log record +@param[in] from_name Tablespace from filename +@param[in] to_name Tablespace to filename */ +static +void +meb_tablespace_redo_rename( + const page_id_t& page_id, + const char* from_name, + const char* to_name) +{ + std::string abs_to_path; + std::string abs_from_path; + std::string tablespace_name; + + meb_make_abs_file_path( + from_name, 0, page_id.space(), abs_from_path, tablespace_name); + + meb_make_abs_file_path( + to_name, 0, page_id.space(), abs_to_path, tablespace_name); + + char* new_name = nullptr; + + if (!meb_replay_file_ops + || meb_is_intermediate_file(from_name) + || meb_is_intermediate_file(to_name) + || fil_space_get_id_by_name(tablespace_name.c_str()) + != SPACE_UNKNOWN + || meb_fil_space_get_rem_gen_ts_id_by_name(tablespace_name) + != SPACE_UNKNOWN + || fil_space_get(page_id.space()) == nullptr){ + + /* Don't rename table while : + 1. Scanning the redo logs during backup + 2. Apply-log on a partial backup + 3. Either of old or new tables are intermediate table + 4. The new name is already loaded for recovery/apply-log + 5. The new name is a remote general tablespace which is + already loaded for recovery/apply-log from different + directory path + 6. Tablespace is not yet loaded in memory. + This will prevent unintended renames during recovery. */ + + ib::trace() + << "Ignoring the log record. " + << "No need to rename tablespace"; + + return; + + } else { + + ib::trace() + << "Renaming space id : " << page_id.space() + << ", old tablespace name : " << from_name + << " to new tablespace name : " << to_name; + + new_name = static_cast(mem_strdup(abs_to_path.c_str())); + } + + meb_fil_name_process(from_name, page_id.space()); + meb_fil_name_process(new_name, page_id.space()); + + if (meb_replay_file_ops) { + + if (!fil_op_replay_rename( + page_id, abs_from_path.c_str(), abs_to_path.c_str())) { + + recv_sys->found_corrupt_fs = true; + } + + meb_fil_name_process(to_name, page_id.space()); + } + + ut_free(new_name); +} + +/** Process a MLOG_FILE_DELETE redo record. +@param[in] page_id Page id of the redo log record +@param[in] name Tablespace filename */ +static +void +meb_tablespace_redo_delete( + const page_id_t& page_id, + const char* name) +{ + std::string abs_file_path; + std::string tablespace_name; + + meb_make_abs_file_path( + name, 0, page_id.space(), abs_file_path, tablespace_name); + + char* file_name = static_cast(mem_strdup(name)); + + fil_system->meb_name_process(file_name, page_id.space(), true); + + if (meb_replay_file_ops && fil_space_get(page_id.space())) { + + dberr_t err = fil_delete_tablespace( + page_id.space(), BUF_REMOVE_FLUSH_NO_WRITE); + + ut_a(err == DB_SUCCESS); + } + + ut_free(file_name); +} + #endif /* UNIV_HOTBACKUP */ /*========== RESERVE FREE EXTENTS (for a B-tree split, for example) ===*/ -/*******************************************************************//** -Tries to reserve free extents in a file space. +/** Tries to reserve free extents in a file space. +@param[in] space_id Tablespace ID +@param[in] n_free_now Number of free extents now +@param[in] n_to_reserve How many one wants to reserve @return true if succeed */ bool fil_space_reserve_free_extents( -/*===========================*/ - space_id_t id, /*!< in: space id */ - ulint n_free_now, /*!< in: number of free extents now */ - ulint n_to_reserve) /*!< in: how many one wants to reserve */ + space_id_t space_id, + ulint n_free_now, + ulint n_to_reserve) { - fil_space_t* space; - bool success; + auto shard = fil_system->shard_by_id(space_id); - ut_ad(fil_system); + shard->mutex_acquire(); - mutex_enter(&fil_system->mutex); + fil_space_t* space = shard->get_space_by_id(space_id); - space = fil_space_get_by_id(id); - - ut_a(space); + bool success; if (space->n_reserved_extents + n_to_reserve > n_free_now) { success = false; } else { - space->n_reserved_extents += n_to_reserve; + ut_a(n_to_reserve < std::numeric_limits::max()); + space->n_reserved_extents += (uint32_t) n_to_reserve; success = true; } - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(success); } -/*******************************************************************//** -Releases free extents in a file space. */ +/** Releases free extents in a file space. +@param[in] space_id Tablespace ID +@param[in] n_reserved How many were reserved */ void -fil_space_release_free_extents( -/*===========================*/ - space_id_t id, /*!< in: space id */ - ulint n_reserved) /*!< in: how many one reserved */ +fil_space_release_free_extents(space_id_t space_id, ulint n_reserved) { - fil_space_t* space; + auto shard = fil_system->shard_by_id(space_id); - ut_ad(fil_system); + shard->mutex_acquire(); - mutex_enter(&fil_system->mutex); + fil_space_t* space = shard->get_space_by_id(space_id); - space = fil_space_get_by_id(id); - - ut_a(space); + ut_a(n_reserved < std::numeric_limits::max()); ut_a(space->n_reserved_extents >= n_reserved); - space->n_reserved_extents -= n_reserved; + space->n_reserved_extents -= (uint32_t) n_reserved; - mutex_exit(&fil_system->mutex); + shard->mutex_release(); } -/*******************************************************************//** -Gets the number of reserved extents. If the database is silent, this number -should be zero. */ +/** Gets the number of reserved extents. If the database is silent, this number +should be zero. +@param[in] space_id Tablespace ID +@return the number of reserved extents */ ulint -fil_space_get_n_reserved_extents( -/*=============================*/ - space_id_t id) /*!< in: space id */ +fil_space_get_n_reserved_extents(space_id_t space_id) { - fil_space_t* space; - ulint n; - - ut_ad(fil_system); + auto shard = fil_system->shard_by_id(space_id); - mutex_enter(&fil_system->mutex); + shard->mutex_acquire(); - space = fil_space_get_by_id(id); + fil_space_t* space = shard->get_space_by_id(space_id); - ut_a(space); + ulint n = space->n_reserved_extents; - n = space->n_reserved_extents; - - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(n); } /*============================ FILE I/O ================================*/ -/** NOTE: you must call fil_mutex_enter_and_prepare_for_io() first! - -Prepares a file node for i/o. Opens the file if it is closed. Updates the -pending i/o's field in the node and the system appropriately. Takes the node -off the LRU list if it is in the LRU list. The caller must hold the fil_sys -mutex. -@param[in] node File node -@param[in] system Tablespace memory cache -@param[in] space Tablespace instance +/** Prepares a file for I/O. Opens the file if it is closed. Updates the +pending I/O's field in the file and the system appropriately. Takes the file +off the LRU list if it is in the LRU list. +@param[in] file Tablespace file @param[in] extend true if file is being extended @return false if the file can't be opened, otherwise true */ -static bool -fil_node_prepare_for_io( - fil_node_t* node, - fil_system_t* system, - fil_space_t* space, - bool extend) +Fil_shard::prepare_file_for_io(fil_node_t* file, bool extend) { - ut_ad(node && system && space); - ut_ad(mutex_own(&(system->mutex))); + ut_ad(mutex_owned()); + + fil_space_t* space = file->space; - if (system->n_open > system->max_n_open + 5) { + if (s_n_open > fil_system->m_max_n_open + 5) { static ulint prev_time; auto curr_time = ut_time(); @@ -6056,181 +7536,417 @@ fil_node_prepare_for_io( if ((curr_time - prev_time) > 60) { ib::warn() - << "Open files " << system->n_open + << "Open files " << s_n_open << " exceeds the limit " - << system->max_n_open; + << fil_system->m_max_n_open; + + prev_time = curr_time; + } + } + + if (!file->is_open) { + + ut_a(file->n_pending == 0); + + if (!open_file(file, extend)) { + return(false); + } + } + + if (file->n_pending == 0 && Fil_system::space_belongs_in_LRU(space)) { + + /* The file is in the LRU list, remove it */ + + ut_a(UT_LIST_GET_LEN(m_LRU) > 0); + + UT_LIST_REMOVE(m_LRU, file); + } + + ++file->n_pending; + + return(true); +} + +/** If the tablespace is not on the unflushed list, add it. +@param[in,out] space Tablespace to add */ +void +Fil_shard::add_to_unflushed_list(fil_space_t* space) +{ + ut_ad(m_id == REDO_SHARD || mutex_owned()); + + if (!space->is_in_unflushed_spaces) { + + space->is_in_unflushed_spaces = true; + + UT_LIST_ADD_FIRST(m_unflushed_spaces, space); + } +} + +/** Note that a write IO has completed. +@param[in,out] file File on which a write was completed */ +void +Fil_shard::write_completed(fil_node_t* file) +{ + ut_ad(m_id == REDO_SHARD || mutex_owned()); + + ++m_modification_counter; + + file->modification_counter = m_modification_counter; + + if (fil_buffering_disabled(file->space)) { + + /* We don't need to keep track of unflushed + changes as user has explicitly disabled + buffering. */ + ut_ad(!file->space->is_in_unflushed_spaces); + + file->flush_counter = file->modification_counter; + + } else { + add_to_unflushed_list(file->space); + } +} + +/** Updates the data structures when an I/O operation finishes. Updates the +pending i/o's field in the file appropriately. +@param[in] file Tablespace file +@param[in] type Marks the file as modified if type == WRITE */ +void +Fil_shard::complete_io(fil_node_t* file, const IORequest& type) +{ + ut_ad(m_id == REDO_SHARD || mutex_owned()); + + ut_a(file->n_pending > 0); + + --file->n_pending; + + ut_ad(type.validate()); + + if (type.is_write()) { + + ut_ad(!srv_read_only_mode + || fsp_is_system_temporary(file->space->id)); + + write_completed(file); + } + + if (file->n_pending == 0 + && Fil_system::space_belongs_in_LRU(file->space)) { + + /* The file must be put back to the LRU list */ + UT_LIST_ADD_FIRST(m_LRU, file); + } +} + +/** Report information about an invalid page access. +@param[in] block_offset Block offset +@param[in] space_id Tablespace ID +@param[in] space_name Tablespace name +@param[in] byte_offset Byte offset +@param[in] len I/O length +@param[in] is_read I/O type +@param[in] line Line called from */ +static +void +fil_report_invalid_page_access_low( + page_no_t block_offset, + space_id_t space_id, + const char* space_name, + ulint byte_offset, + ulint len, + bool is_read, + int line) +{ + ib::error() + << "Trying to access page number " << block_offset << " in" + " space " << space_id << ", space name " << space_name << "," + " which is outside the tablespace bounds. Byte offset " + << byte_offset << ", len " << len << ", i/o type " << + (is_read ? "read" : "write") + << ". If you get this error at mysqld startup, please check" + " that your my.cnf matches the ibdata files that you have in" + " the MySQL server."; + + ib::error() << "Server exits" +#ifdef UNIV_DEBUG + << " at " << "fil0fil.cc" << "[" << line << "]" +#endif /* UNIV_DEBUG */ + << "."; + + ut_error; + + _exit(1); +} + +#define fil_report_invalid_page_access(b, s, n, o, l, t) \ + fil_report_invalid_page_access_low( \ + (b), (s), (n), (o), (l), (t), __LINE__) + +/** Set encryption information for IORequest. +@param[in,out] req_type IO request +@param[in] page_id page id +@param[in] space table space */ +void +fil_io_set_encryption( + IORequest& req_type, + const page_id_t& page_id, + fil_space_t* space) +{ + /* Don't encrypt page 0 of all tablespaces except redo log + tablespace, all pages from the system tablespace. */ + if (space->encryption_type == Encryption::NONE + || (page_id.page_no() == 0 && !req_type.is_log())) { + req_type.clear_encrypted(); + return; + } + + /* For writting redo log, if encryption for redo log is disabled, + skip set encryption. */ + if (req_type.is_log() + && req_type.is_write() + && !srv_redo_log_encrypt) { + req_type.clear_encrypted(); + return; + } + + /* For writting undo log, if encryption for undo log is disabled, + skip set encryption. */ + if (fsp_is_undo_tablespace(space->id) + && !srv_undo_log_encrypt && req_type.is_write()) { + req_type.clear_encrypted(); + return; + } + + /* Make any active clone operation to abort, in case + log encryption is set after clone operation is started. */ + clone_mark_abort(true); + clone_mark_active(); + + req_type.encryption_key( + space->encryption_key, space->encryption_klen, + space->encryption_iv); + + req_type.encryption_algorithm(Encryption::AES); +} + +/** Get the AIO mode. +@param[in] req_type IO request type +@param[in] sync true if Synchronous IO +return the AIO mode */ +AIO_mode +Fil_shard::get_AIO_mode(const IORequest& req_type, bool sync) +{ +#ifndef UNIV_HOTBACKUP + if (sync) { + + return(AIO_mode::SYNC); + + } else if (req_type.is_log()) { + + return(AIO_mode::LOG); + + } else { + return(AIO_mode::NORMAL); + } +#else /* !UNIV_HOTBACKUP */ + ut_a(sync); + return(AIO_mode::SYNC); +#endif /* !UNIV_HOTBACKUP */ +} + +/** Get the file name for IO and the local offset within that file. +@param[in] req_type IO context +@param[in,out] space Tablespace for IO +@param[in,out] page_no The relative page number in the file +@param[out] file File node +@return DB_SUCCESS or error code */ +dberr_t +Fil_shard::get_file_for_io( + const IORequest& req_type, + fil_space_t* space, + page_no_t* page_no, + fil_node_t*& file) +{ + if (space->files.size() > 1) { + + ut_a(space->id == TRX_SYS_SPACE + || space->purpose == FIL_TYPE_TEMPORARY + || space->id == dict_sys_t::s_log_space_first_id); + + for (auto& f : space->files) { + + if (f.size > *page_no) { + file = &f; + return(DB_SUCCESS); + + } + + *page_no -= f.size; + } + + } else if (!space->files.empty()) { + + fil_node_t& f = space->files.front(); + + if ((fsp_is_ibd_tablespace(space->id) && f.size == 0) + || f.size > *page_no) { + + /* We do not know the size of a single-table tablespace + before we open the file */ + + file = &f; + + return(DB_SUCCESS); + + } else { +#ifndef UNIV_HOTBACKUP + if (space->id != TRX_SYS_SPACE + && req_type.is_read() + && undo::is_inactive(space->id)) { + + file = nullptr; + + /* Page access request for a page that is + outside the truncated UNDO tablespace bounds. */ + + return(DB_TABLE_NOT_FOUND); + } +#else /* !UNIV_HOTBACKUP */ + /* In backup, is_under_construction() is always false */ +#endif /* !UNIV_HOTBACKUP */ + } + } + + file = nullptr; + return(DB_ERROR); +} + +#ifndef UNIV_HOTBACKUP +/** Read or write log file data synchronously. +@param[in] type IO context +@param[in] page_id page id +@param[in] page_size page size +@param[in] byte_offset remainder of offset in bytes; in AIO + this must be divisible by the OS block + size +@param[in] len how many bytes to read or write; this + must not cross a file boundary; in AIO + this must be a block size multiple +@param[in,out] buf buffer where to store read data or + from where to write +@return error code +@retval DB_SUCCESS on success */ +dberr_t +Fil_shard::do_redo_io( + const IORequest& type, + const page_id_t& page_id, + const page_size_t& page_size, + ulint byte_offset, + ulint len, + void* buf) +{ + IORequest req_type(type); + + ut_ad(len > 0); + ut_ad(req_type.is_log()); + ut_ad(req_type.validate()); + ut_ad(fil_validate_skip()); + ut_ad(byte_offset < UNIV_PAGE_SIZE); + ut_ad(UNIV_PAGE_SIZE == (ulong)(1 << UNIV_PAGE_SIZE_SHIFT)); - prev_time = curr_time; - } - } + if (req_type.is_read()) { - if (!node->is_open) { + srv_stats.data_read.add(len); - /* File is closed: open it */ - ut_a(node->n_pending == 0); + } else if (req_type.is_write()) { - if (!fil_node_open_file(node, extend)) { - return(false); - } + ut_ad(!srv_read_only_mode); + srv_stats.data_written.add(len); } - if (node->n_pending == 0 && fil_space_belongs_in_lru(space)) { - /* The node is in the LRU list, remove it */ - - ut_a(UT_LIST_GET_LEN(system->LRU) > 0); + fil_space_t* space = get_space_by_id(page_id.space()); - UT_LIST_REMOVE(system->LRU, node); - } + fil_node_t* file; + page_no_t page_no = page_id.page_no(); + dberr_t err = get_file_for_io(req_type, space, &page_no, file); - node->n_pending++; + ut_a(file != nullptr); + ut_a(err == DB_SUCCESS); + ut_a(page_size.physical() == page_size.logical()); - return(true); -} + os_offset_t offset = (os_offset_t) page_no * page_size.physical(); -/********************************************************************//** -Updates the data structures when an i/o operation finishes. Updates the -pending i/o's field in the node appropriately. */ -static -void -fil_node_complete_io( -/*=================*/ - fil_node_t* node, /*!< in: file node */ - fil_system_t* system, /*!< in: tablespace memory cache */ - const IORequest&type) /*!< in: IO_TYPE_*, marks the node as - modified if TYPE_IS_WRITE() */ -{ - ut_ad(mutex_own(&system->mutex)); - ut_a(node->n_pending > 0); + offset += byte_offset; - --node->n_pending; + ut_a(file->size - page_no + >= (byte_offset + len + (page_size.physical() - 1)) + / page_size.physical()); - ut_ad(type.validate()); + ut_a((len % OS_FILE_LOG_BLOCK_SIZE) == 0); + ut_a(byte_offset % OS_FILE_LOG_BLOCK_SIZE == 0); - if (type.is_write()) { + /* Set encryption information. */ + fil_io_set_encryption(req_type, page_id, space); - ut_ad(!srv_read_only_mode - || fsp_is_system_temporary(node->space->id)); + req_type.block_size(file->block_size); - ++system->modification_counter; + if (!file->is_open) { - node->modification_counter = system->modification_counter; + ut_a(file->n_pending == 0); - if (fil_buffering_disabled(node->space)) { + bool success = open_file(file, false); - /* We don't need to keep track of unflushed - changes as user has explicitly disabled - buffering. */ - ut_ad(!node->space->is_in_unflushed_spaces); - node->flush_counter = node->modification_counter; + ut_a(success); + } - } else if (!node->space->is_in_unflushed_spaces) { + if (req_type.is_read()) { - node->space->is_in_unflushed_spaces = true; + err = os_file_read(req_type, file->handle, buf, offset, len); - UT_LIST_ADD_FIRST( - system->unflushed_spaces, node->space); - } - } + } else { - if (node->n_pending == 0 && fil_space_belongs_in_lru(node->space)) { + ut_ad(!srv_read_only_mode); - /* The node must be put back to the LRU list */ - UT_LIST_ADD_FIRST(system->LRU, node); + err = os_file_write( + req_type, file->name, file->handle, buf, offset, len); } -} -/** Report information about an invalid page access. */ -static -void -fil_report_invalid_page_access( - page_no_t block_offset, /*!< in: block offset */ - space_id_t space_id, /*!< in: space id */ - const char* space_name, /*!< in: space name */ - ulint byte_offset, /*!< in: byte offset */ - ulint len, /*!< in: I/O length */ - bool is_read) /*!< in: I/O type */ -{ - ib::error() - << "Trying to access page number " << block_offset << " in" - " space " << space_id << ", space name " << space_name << "," - " which is outside the tablespace bounds. Byte offset " - << byte_offset << ", len " << len << ", i/o type " << - (is_read ? "read" : "write") - << ". If you get this error at mysqld startup, please check" - " that your my.cnf matches the ibdata files that you have in" - " the MySQL server."; + if (type.is_write()) { - ib::error() << "Server exits" -#ifdef UNIV_DEBUG - << " at " << __FILE__ << "[" << __LINE__ << "]" -#endif - << "."; + mutex_acquire(); - _exit(1); -} + ++m_modification_counter; -/** Set encryption information for IORequest. -@param[in,out] req_type IO request -@param[in] page_id page id -@param[in] space table space */ -void -fil_io_set_encryption( - IORequest& req_type, - const page_id_t& page_id, - fil_space_t* space) -{ - /* Don't encrypt page 0 of all tablespaces except redo log - tablespace, all pages from the system tablespace. */ - if (space->encryption_type == Encryption::NONE - || (page_id.page_no() == 0 && !req_type.is_log())) { - req_type.clear_encrypted(); - return; - } + file->modification_counter = m_modification_counter; - /* For writting redo log, if encryption for redo log is disabled, - skip set encryption. */ - if (req_type.is_log() && req_type.is_write() - && !srv_redo_log_encrypt) { - req_type.clear_encrypted(); - return; - } + add_to_unflushed_list(file->space); - /* For writting undo log, if encryption for undo log is disabled, - skip set encryption. */ - if (fsp_is_undo_tablespace(space->id) - && !srv_undo_log_encrypt && req_type.is_write()) { - req_type.clear_encrypted(); - return; + mutex_release(); } - /* Make any active clone operation to abort, in case - log encryption is set after clone operation is started. */ - clone_mark_abort(true); - clone_mark_active(); - - req_type.encryption_key(space->encryption_key, - space->encryption_klen, - space->encryption_iv); - req_type.encryption_algorithm(Encryption::AES); + return(err); } +#endif /* !UNIV_HOTBACKUP */ /** Read or write data. This operation could be asynchronous (aio). -@param[in,out] type IO context +@param[in] type IO context @param[in] sync whether synchronous aio is desired @param[in] page_id page id @param[in] page_size page size @param[in] byte_offset remainder of offset in bytes; in aio this -must be divisible by the OS block size + must be divisible by the OS block size @param[in] len how many bytes to read or write; this must -not cross a file boundary; in aio this must be a block size multiple + not cross a file boundary; in AIO this must + be a block size multiple @param[in,out] buf buffer where to store read data or from where -to write; in aio this must be appropriately aligned + to write; in aio this must be appropriately + aligned @param[in] message message for aio handler if !sync, else ignored @return error code @retval DB_SUCCESS on success @retval DB_TABLESPACE_DELETED if the tablespace does not exist */ dberr_t -fil_io( +Fil_shard::do_io( const IORequest& type, bool sync, const page_id_t& page_id, @@ -6240,7 +7956,6 @@ fil_io( void* buf, void* message) { - os_offset_t offset; IORequest req_type(type); ut_ad(req_type.validate()); @@ -6249,18 +7964,10 @@ fil_io( ut_ad(byte_offset < UNIV_PAGE_SIZE); ut_ad(!page_size.is_compressed() || byte_offset == 0); ut_ad(UNIV_PAGE_SIZE == (ulong)(1 << UNIV_PAGE_SIZE_SHIFT)); -#if (1 << UNIV_PAGE_SIZE_SHIFT_MAX) != UNIV_PAGE_SIZE_MAX -# error "(1 << UNIV_PAGE_SIZE_SHIFT_MAX) != UNIV_PAGE_SIZE_MAX" -#endif -#if (1 << UNIV_PAGE_SIZE_SHIFT_MIN) != UNIV_PAGE_SIZE_MIN -# error "(1 << UNIV_PAGE_SIZE_SHIFT_MIN) != UNIV_PAGE_SIZE_MIN" -#endif - ut_ad(fil_validate_skip()); - ulint mode; + ut_ad(fil_validate_skip()); #ifndef UNIV_HOTBACKUP - /* ibuf bitmap pages must be read in the sync AIO mode: */ ut_ad(recv_no_ibuf_operations || req_type.is_write() @@ -6268,31 +7975,24 @@ fil_io( || sync || req_type.is_log()); - if (sync) { - - mode = OS_AIO_SYNC; - - } else if (req_type.is_log()) { - - mode = OS_AIO_LOG; + AIO_mode aio_mode = get_AIO_mode(req_type, sync); - } else if (req_type.is_read() - && !recv_no_ibuf_operations - && ibuf_page(page_id, page_size, NULL)) { + if (req_type.is_read()) { - mode = OS_AIO_IBUF; + srv_stats.data_read.add(len); - /* Reduce probability of deadlock bugs in connection with ibuf: - do not let the ibuf i/o handler sleep */ + if (aio_mode == AIO_mode::NORMAL + && !recv_no_ibuf_operations + && ibuf_page(page_id, page_size, nullptr)) { - req_type.clear_do_not_wake(); - } else { - mode = OS_AIO_NORMAL; - } + /* Reduce probability of deadlock bugs + in connection with ibuf: do not let the + ibuf I/O handler sleep */ - if (req_type.is_read()) { + req_type.clear_do_not_wake(); - srv_stats.data_read.add(len); + aio_mode = AIO_mode::IBUF; + } } else if (req_type.is_write()) { @@ -6303,27 +8003,32 @@ fil_io( } #else /* !UNIV_HOTBACKUP */ ut_a(sync); - mode = OS_AIO_SYNC; + AIO_mode aio_mode = AIO_mode::SYNC; #endif /* !UNIV_HOTBACKUP */ - /* Reserve the fil_system mutex and make sure that we can open at + /* Reserve the mutex and make sure that we can open at least one file while holding it, if the file is not already open */ - fil_mutex_enter_and_prepare_for_io(page_id.space()); + fil_space_t* space; + + bool slot = mutex_acquire_and_get_space(page_id.space(), space); - fil_space_t* space = fil_space_get_by_id(page_id.space()); + /* If we are deleting a tablespace we don't allow async read + operations on that. However, we do allow write operations and + sync read operations. */ + if (space == nullptr + || (req_type.is_read() && !sync && space->stop_new_ops)) { - /* If we are deleting a tablespace we don't allow async read operations - on that. However, we do allow write operations and sync read operations. */ - if (space == NULL - || (req_type.is_read() - && !sync - && space->stop_new_ops)) { + if (slot) { + release_open_slot(m_id); + } - mutex_exit(&fil_system->mutex); + mutex_release(); if (!req_type.ignore_missing()) { - if (space == NULL) { + + if (space == nullptr) { + ib::error() << "Trying to do I/O on a tablespace" << " which does not exist. I/O type: " @@ -6332,6 +8037,7 @@ fil_io( << ", page: " << page_id << ", I/O length: " << len << " bytes"; } else { + ib::error() << "Trying to do async read on a" << " tablespace which is being deleted." @@ -6344,73 +8050,63 @@ fil_io( return(DB_TABLESPACE_DELETED); } - ut_ad(mode != OS_AIO_IBUF || fil_type_is_data(space->purpose)); - - page_no_t cur_page_no = page_id.page_no(); - fil_node_t* node = UT_LIST_GET_FIRST(space->chain); - - for (;;) { + ut_ad(aio_mode != AIO_mode::IBUF || fil_type_is_data(space->purpose)); - if (node == NULL) { + fil_node_t* file; + page_no_t page_no = page_id.page_no(); + dberr_t err = get_file_for_io(req_type, space, &page_no, file); - if (req_type.ignore_missing()) { - mutex_exit(&fil_system->mutex); - return(DB_ERROR); - } + if (err == DB_TABLE_NOT_FOUND) { - fil_report_invalid_page_access( - page_id.page_no(), page_id.space(), - space->name, byte_offset, len, - req_type.is_read()); + mutex_release(); - } else if (fsp_is_ibd_tablespace(space->id) - && node->size == 0) { + return(err); - /* We do not know the size of a single-table tablespace - before we open the file */ - break; + } else if (file == nullptr) { - } else if (node->size > cur_page_no) { - /* Found! */ - break; + ut_ad(err == DB_ERROR); - } else { -#ifndef UNIV_HOTBACKUP - /* In backup, is_under_construction() is always false */ - if (space->id != TRX_SYS_SPACE - && UT_LIST_GET_LEN(space->chain) == 1 - && req_type.is_read() - && undo::is_inactive(space->id)) { + if (req_type.ignore_missing()) { - /* Handle page which is outside the truncated - tablespace bounds when recovering from a crash - that happened during a truncation */ - mutex_exit(&fil_system->mutex); - return(DB_TABLESPACE_DELETED); + if (slot) { + release_open_slot(m_id); } -#endif /* !UNIV_HOTBACKUP */ - cur_page_no -= node->size; - node = UT_LIST_GET_NEXT(chain, node); + mutex_release(); + + return(DB_ERROR); } + + /* This is a hard error. */ + fil_report_invalid_page_access( + page_id.page_no(), page_id.space(), + space->name, byte_offset, len, + req_type.is_read()); + } + + bool opened = prepare_file_for_io(file, false); + + if (slot) { + release_open_slot(m_id); } - /* Open file if closed */ - if (!fil_node_prepare_for_io(node, fil_system, space, false)) { + if (!opened) { + if (fil_type_is_data(space->purpose) && fsp_is_ibd_tablespace(space->id)) { - mutex_exit(&fil_system->mutex); + + mutex_release(); if (!req_type.ignore_missing()) { + ib::error() << "Trying to do I/O to a tablespace" - " which exists without .ibd data file." - " I/O type: " + " which exists without an .ibd data" + << " file. I/O type: " << (req_type.is_read() ? "read" : "write") << ", page: " - << page_id_t(page_id.space(), - cur_page_no) + << page_id_t(page_id.space(), page_no) << ", I/O length: " << len << " bytes"; } @@ -6426,74 +8122,55 @@ fil_io( /* Check that at least the start offset is within the bounds of a single-table tablespace, including rollback tablespaces. */ - if (node->size <= cur_page_no + if (file->size <= page_no && space->id != TRX_SYS_SPACE && fil_type_is_data(space->purpose)) { if (req_type.ignore_missing()) { + /* If we can tolerate the non-existent pages, we should return with DB_ERROR and let caller decide what to do. */ - fil_node_complete_io(node, fil_system, req_type); - mutex_exit(&fil_system->mutex); + + complete_io(file, req_type); + + mutex_release(); + return(DB_ERROR); } + /* This is a hard error. */ fil_report_invalid_page_access( page_id.page_no(), page_id.space(), space->name, byte_offset, len, req_type.is_read()); } - /* Now we have made the changes in the data structures of fil_system */ - mutex_exit(&fil_system->mutex); - - /* Calculate the low 32 bits and the high 32 bits of the file offset */ - - if (!page_size.is_compressed()) { - - offset = ((os_offset_t) cur_page_no - << UNIV_PAGE_SIZE_SHIFT) + byte_offset; - - ut_a(node->size - cur_page_no - >= ((byte_offset + len + (UNIV_PAGE_SIZE - 1)) - / UNIV_PAGE_SIZE)); - } else { - ulint size_shift; + mutex_release(); - switch (page_size.physical()) { - case 1024: size_shift = 10; break; - case 2048: size_shift = 11; break; - case 4096: size_shift = 12; break; - case 8192: size_shift = 13; break; - case 16384: size_shift = 14; break; - case 32768: size_shift = 15; break; - case 65536: size_shift = 16; break; - default: ut_error; - } + ut_a(page_size.is_compressed() + || page_size.physical() == page_size.logical()); - offset = ((os_offset_t) cur_page_no << size_shift) - + byte_offset; + os_offset_t offset = (os_offset_t) page_no * page_size.physical(); - ut_a(node->size - cur_page_no - >= (len + (page_size.physical() - 1)) - / page_size.physical()); - } + offset += byte_offset; - /* Do AIO */ + ut_a(file->size - page_no + >= (byte_offset + len + (page_size.physical() - 1)) + / page_size.physical()); - ut_a(byte_offset % OS_FILE_LOG_BLOCK_SIZE == 0); ut_a((len % OS_FILE_LOG_BLOCK_SIZE) == 0); + ut_a(byte_offset % OS_FILE_LOG_BLOCK_SIZE == 0); /* Don't compress the log, page 0 of all tablespaces, tables - compresssed with the old scheme and all pages from the system - tablespace. */ + compresssed with the old compression scheme and all pages from + the system tablespace. */ if (req_type.is_write() && !req_type.is_log() && !page_size.is_compressed() && page_id.page_no() > 0 && IORequest::is_punch_hole_supported() - && node->punch_hole) { + && file->punch_hole) { ut_ad(!req_type.is_log()); @@ -6508,15 +8185,13 @@ fil_io( /* Set encryption information. */ fil_io_set_encryption(req_type, page_id, space); - req_type.block_size(node->block_size); - - dberr_t err; + req_type.block_size(file->block_size); #ifdef UNIV_HOTBACKUP - /* In mysqlbackup do normal i/o, not aio */ + /* In mysqlbackup do normal I/O, not AIO */ if (req_type.is_read()) { - err = os_file_read(req_type, node->handle, buf, offset, len); + err = os_file_read(req_type, file->handle, buf, offset, len); } else { @@ -6524,16 +8199,16 @@ fil_io( || fsp_is_system_temporary(page_id.space())); err = os_file_write( - req_type, node->name, node->handle, buf, offset, len); + req_type, file->name, file->handle, buf, offset, len); } #else /* UNIV_HOTBACKUP */ /* Queue the aio request */ err = os_aio( req_type, - mode, node->name, node->handle, buf, offset, len, + aio_mode, file->name, file->handle, buf, offset, len, fsp_is_system_temporary(page_id.space()) ? false : srv_read_only_mode, - node, message); + file, message); #endif /* UNIV_HOTBACKUP */ @@ -6541,14 +8216,14 @@ fil_io( err = DB_SUCCESS; - if (node->punch_hole) { + if (file->punch_hole) { ib::warn() << "Punch hole failed for '" - << node->name << "'"; + << file->name << "'"; } - fil_no_punch_hole(node); + fil_no_punch_hole(file); } /* We an try to recover the page from the double write buffer if @@ -6560,11 +8235,11 @@ fil_io( /* The i/o operation is already completed when we return from os_aio: */ - mutex_enter(&fil_system->mutex); + mutex_acquire(); - fil_node_complete_io(node, fil_system, req_type); + complete_io(file, req_type); - mutex_exit(&fil_system->mutex); + mutex_release(); ut_ad(fil_validate_skip()); } @@ -6573,39 +8248,71 @@ fil_io( } #ifndef UNIV_HOTBACKUP -/**********************************************************************//** -Waits for an aio operation to complete. This function is used to write the +/** Read or write redo log data (synchronous buffered IO). +@param[in] type IO context +@param[in] page_id where to read or write +@param[in] page_size page size +@param[in] byte_offset remainder of offset in bytes +@param[in] len this must not cross a file boundary; +@param[in,out] buf buffer where to store read data or from where + to write +@retval DB_SUCCESS if all OK */ +dberr_t +fil_redo_io( + const IORequest& type, + const page_id_t& page_id, + const page_size_t& page_size, + ulint byte_offset, + ulint len, + void* buf) +{ + ut_ad(type.is_log()); + + auto shard = fil_system->shard_by_id(page_id.space()); +#if defined(_WIN32) && defined(WIN_ASYNC_IO) + /* On Windows we always open the redo log file in AIO mode. ie. we + use the AIO API for the read/write even for sync IO. */ + return(shard->do_io( + type, true, page_id, page_size, byte_offset, len, buf, nullptr)); +#else + return(shard->do_redo_io( + type, page_id, page_size, byte_offset, len, buf)); +#endif /* _WIN32 || WIN_ASYNC_IO*/ +} + +/** Waits for an AIO operation to complete. This function is used to write the handler for completed requests. The aio array of pending requests is divided into segments (see os0file.cc for more info). The thread specifies which -segment it wants to wait for. */ +segment it wants to wait for. +@param[in] segment The number of the segment in the AIO array + to wait for */ void -fil_aio_wait( -/*=========*/ - ulint segment) /*!< in: the number of the segment in the aio - array to wait for */ +fil_aio_wait(ulint segment) { - fil_node_t* node; + fil_node_t* file; IORequest type; void* message; ut_ad(fil_validate_skip()); - dberr_t err = os_aio_handler(segment, &node, &message, &type); + dberr_t err = os_aio_handler(segment, &file, &message, &type); ut_a(err == DB_SUCCESS); - if (node == NULL) { + if (file == nullptr) { ut_ad(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS); return; } - srv_set_io_thread_op_info(segment, "complete io for fil node"); + srv_set_io_thread_op_info(segment, "complete io for file"); + + auto shard = fil_system->shard_by_id(file->space->id); - mutex_enter(&fil_system->mutex); + shard->mutex_acquire(); - fil_node_complete_io(node, fil_system, type); + shard->complete_io(file, type); - mutex_exit(&fil_system->mutex); + shard->mutex_release(); ut_ad(fil_validate_skip()); @@ -6615,49 +8322,189 @@ fil_aio_wait( deadlocks in the i/o system. We keep tablespace 0 data files always open, and use a special i/o thread to serve insert buffer requests. */ - switch (node->space->purpose) { - case FIL_TYPE_TABLESPACE: - case FIL_TYPE_TEMPORARY: + switch (file->space->purpose) { case FIL_TYPE_IMPORT: + case FIL_TYPE_TEMPORARY: + case FIL_TYPE_TABLESPACE: srv_set_io_thread_op_info(segment, "complete io for buf page"); /* async single page writes from the dblwr buffer don't have access to the page */ - if (message != NULL) { + if (message != nullptr) { buf_page_io_complete(static_cast(message)); } return; case FIL_TYPE_LOG: - srv_set_io_thread_op_info(segment, "complete io for log"); + srv_set_io_thread_op_info(segment, "complete checkpoint io"); log_io_complete(static_cast(message)); return; } - ut_ad(0); + ut_ad(0); +} +#endif /* !UNIV_HOTBACKUP */ + +/** Read or write data from a file. +@param[in] type IO context +@param[in] sync If true then do synchronous IO +@param[in] page_id page id +@param[in] page_size page size +@param[in] byte_offset remainder of offset in bytes; in aio this + must be divisible by the OS block size +@param[in] len how many bytes to read or write; this must + not cross a file boundary; in AIO this must + be a block size multiple +@param[in,out] buf buffer where to store read data or from where + to write; in AIO this must be appropriately + aligned +@param[in] message message for AIO handler if !sync, else ignored +@return error code +@retval DB_SUCCESS on success +@retval DB_TABLESPACE_DELETED if the tablespace does not exist */ +dberr_t +fil_io( + const IORequest& type, + bool sync, + const page_id_t& page_id, + const page_size_t& page_size, + ulint byte_offset, + ulint len, + void* buf, + void* message) +{ + auto shard = fil_system->shard_by_id(page_id.space()); + + return(shard->do_io( + type, sync, page_id, page_size, + byte_offset, len, buf, message)); + +} + +/** If the tablespace is on the unflushed list and there are no pending +flushes then remove from the unflushed list. +@param[in,out] space Tablespace to remove */ +void +Fil_shard::remove_from_unflushed_list(fil_space_t* space) +{ + ut_ad(mutex_owned()); + + if (space->is_in_unflushed_spaces && space_is_flushed(space)) { + + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE(m_unflushed_spaces, space); + } +} + +/** Flushes to disk possible writes cached by the OS. */ +void +Fil_shard::redo_space_flush() +{ + ut_ad(mutex_owned()); + ut_ad(m_id == REDO_SHARD); + + fil_space_t* space = fil_space_t::s_redo_space; + + if (space == nullptr) { + space = get_space_by_id(dict_sys_t::s_log_space_first_id); + } else { + ut_ad(space + == get_space_by_id(dict_sys_t::s_log_space_first_id)); + } + + ut_a(!space->stop_new_ops); + ut_a(space->purpose == FIL_TYPE_LOG); + + /* Prevent dropping of the space while we are flushing */ + ++space->n_pending_flushes; + + for (auto& file : space->files) { + + ut_a(!file.is_raw_disk); + + int64_t old_mod_counter = file.modification_counter; + + if (old_mod_counter <= file.flush_counter) { + continue; + } + + ut_a(file.is_open); + ut_a(file.space == space); + + ++fil_n_log_flushes; + ++fil_n_pending_log_flushes; + + bool skip_flush = false; + + /* Wait for some other thread that is flushing. */ + while (file.n_pending_flushes > 0 && !skip_flush) { + + /* Release the mutex to avoid deadlock with + the flushing thread. */ + + int64_t sig_count = os_event_reset(file.sync_event); + + mutex_release(); + + os_event_wait_low(file.sync_event, sig_count); + + mutex_acquire(); + + if (file.flush_counter >= old_mod_counter) { + + skip_flush = true; + } + } + + if (!skip_flush) { + + ut_a(file.is_open); + + ++file.n_pending_flushes; + + mutex_release(); + + os_file_flush(file.handle); + + mutex_acquire(); + + os_event_set(file.sync_event); + + --file.n_pending_flushes; + } + + if (file.flush_counter < old_mod_counter) { + + file.flush_counter = old_mod_counter; + + remove_from_unflushed_list(space); + } + + --fil_n_pending_log_flushes; + } + + --space->n_pending_flushes; } -#endif /* !UNIV_HOTBACKUP */ -/**********************************************************************//** -Flushes to disk possible writes cached by the OS. If the space does not exist -or is being dropped, does not do anything. */ +/** Flushes to disk possible writes cached by the OS. If the space does +not exist or is being dropped, does not do anything. +@param[in] space_id File space ID (this can be a group of log files + or a tablespace of the database) */ void -fil_flush( -/*======*/ - space_id_t space_id) /*!< in: file space id (this can be a - group of log files or a tablespace of - the database) */ +Fil_shard::space_flush(space_id_t space_id) { - fil_node_t* node; - pfs_os_file_t file; + ut_ad(mutex_owned()); - mutex_enter(&fil_system->mutex); + if (space_id == dict_sys_t::s_log_space_first_id) { + redo_space_flush(); + return; + } - fil_space_t* space = fil_space_get_by_id(space_id); + fil_space_t* space = get_space_by_id(space_id); - if (space == NULL + if (space == nullptr || space->purpose == FIL_TYPE_TEMPORARY || space->stop_new_ops) { - mutex_exit(&fil_system->mutex); return; } @@ -6667,158 +8514,162 @@ fil_flush( /* No need to flush. User has explicitly disabled buffering. */ ut_ad(!space->is_in_unflushed_spaces); - ut_ad(fil_space_is_flushed(space)); + ut_ad(space_is_flushed(space)); ut_ad(space->n_pending_flushes == 0); #ifdef UNIV_DEBUG - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { - ut_ad(node->modification_counter - == node->flush_counter); - ut_ad(node->n_pending_flushes == 0); + for (const auto& file : space->files) { + ut_ad(file.modification_counter == file.flush_counter); + ut_ad(file.n_pending_flushes == 0); } #endif /* UNIV_DEBUG */ - mutex_exit(&fil_system->mutex); return; } - space->n_pending_flushes++; /*!< prevent dropping of the space while - we are flushing */ - for (node = UT_LIST_GET_FIRST(space->chain); - node != NULL; - node = UT_LIST_GET_NEXT(chain, node)) { + /* Prevent dropping of the space while we are flushing */ + ++space->n_pending_flushes; + + for (auto& file : space->files) { - int64_t old_mod_counter = node->modification_counter; + int64_t old_mod_counter = file.modification_counter; - if (old_mod_counter <= node->flush_counter) { + if (old_mod_counter <= file.flush_counter) { continue; } - ut_a(node->is_open); + ut_a(file.is_open); switch (space->purpose) { case FIL_TYPE_TEMPORARY: ut_ad(0); // we already checked for this + case FIL_TYPE_TABLESPACE: case FIL_TYPE_IMPORT: - fil_n_pending_tablespace_flushes++; + ++fil_n_pending_tablespace_flushes; break; + case FIL_TYPE_LOG: - fil_n_pending_log_flushes++; - fil_n_log_flushes++; + ut_error; break; } + + bool skip_flush = false; #ifdef _WIN32 - if (node->is_raw_disk) { + if (file.is_raw_disk) { - goto skip_flush; + skip_flush = true;; } #endif /* _WIN32 */ -retry: - if (node->n_pending_flushes > 0) { + + while (file.n_pending_flushes > 0 && !skip_flush) { + /* We want to avoid calling os_file_flush() on the file twice at the same time, because we do not know what bugs OS's may contain in file - i/o */ + I/O */ - int64_t sig_count = os_event_reset(node->sync_event); + int64_t sig_count = os_event_reset(file.sync_event); - mutex_exit(&fil_system->mutex); + mutex_release(); - os_event_wait_low(node->sync_event, sig_count); + os_event_wait_low(file.sync_event, sig_count); - mutex_enter(&fil_system->mutex); + mutex_acquire(); - if (node->flush_counter >= old_mod_counter) { + if (file.flush_counter >= old_mod_counter) { - goto skip_flush; + skip_flush = true; } - - goto retry; } - ut_a(node->is_open); - file = node->handle; - node->n_pending_flushes++; + if (!skip_flush) { + + ut_a(file.is_open); - mutex_exit(&fil_system->mutex); + ++file.n_pending_flushes; - os_file_flush(file); + mutex_release(); - mutex_enter(&fil_system->mutex); + os_file_flush(file.handle); - os_event_set(node->sync_event); + mutex_acquire(); - node->n_pending_flushes--; -skip_flush: - if (node->flush_counter < old_mod_counter) { - node->flush_counter = old_mod_counter; + os_event_set(file.sync_event); - if (space->is_in_unflushed_spaces - && fil_space_is_flushed(space)) { + --file.n_pending_flushes; + } - space->is_in_unflushed_spaces = false; + if (file.flush_counter < old_mod_counter) { - UT_LIST_REMOVE( - fil_system->unflushed_spaces, - space); - } + file.flush_counter = old_mod_counter; + + remove_from_unflushed_list(space); } switch (space->purpose) { case FIL_TYPE_TEMPORARY: ut_ad(0); // we already checked for this + case FIL_TYPE_TABLESPACE: case FIL_TYPE_IMPORT: - fil_n_pending_tablespace_flushes--; + --fil_n_pending_tablespace_flushes; continue; + case FIL_TYPE_LOG: - fil_n_pending_log_flushes--; - continue; + ut_error; } ut_ad(0); } - space->n_pending_flushes--; - - mutex_exit(&fil_system->mutex); + --space->n_pending_flushes; } -/** Flush to disk the writes in file spaces of the given type -possibly cached by the OS. -@param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG, can be ORred */ +/** Flushes to disk possible writes cached by the OS. If the space does +not exist or is being dropped, does not do anything. +@param[in] space_id File space ID (this can be a group of log files + or a tablespace of the database) */ void -fil_flush_file_spaces(uint8_t purpose) +fil_flush(space_id_t space_id) { - ulint n_space_ids; + auto shard = fil_system->shard_by_id(space_id); - ut_ad((purpose & FIL_TYPE_TABLESPACE) || (purpose & FIL_TYPE_LOG)); + shard->mutex_acquire(); - mutex_enter(&fil_system->mutex); + /* Note: Will release and reacquire the Fil_shard::mutex. */ + shard->space_flush(space_id); - n_space_ids = UT_LIST_GET_LEN(fil_system->unflushed_spaces); - if (n_space_ids == 0) { + shard->mutex_release(); +} - mutex_exit(&fil_system->mutex); - return; - } +/** Flush any pending writes to disk for the redo log. */ +void +Fil_shard::flush_file_redo() +{ + /* We never evict the redo log tablespace. It's for all + practical purposes a read-only data structure. */ + + mutex_acquire(); - /* Assemble a list of space ids to flush. Previously, we - traversed fil_system->unflushed_spaces and called UT_LIST_GET_NEXT() - on a space that was just removed from the list by fil_flush(). - Thus, the space could be dropped and the memory overwritten. */ - using Space_Ids = std::vector>; + redo_space_flush(); - Space_Ids space_ids; + mutex_release(); +} + +/** Collect the tablespace IDs of unflushed tablespaces in space_ids. +@param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG, + can be ORred */ +void +Fil_shard::flush_file_spaces(uint8_t purpose) +{ + Space_ids space_ids; - space_ids.reserve(n_space_ids); + ut_ad((purpose & FIL_TYPE_TABLESPACE) || (purpose & FIL_TYPE_LOG)); - n_space_ids = 0; + mutex_acquire(); - for (auto space = UT_LIST_GET_FIRST(fil_system->unflushed_spaces); + for (auto space = UT_LIST_GET_FIRST(m_unflushed_spaces); space != nullptr; space = UT_LIST_GET_NEXT(unflushed_spaces, space)) { @@ -6829,131 +8680,90 @@ fil_flush_file_spaces(uint8_t purpose) } } - mutex_exit(&fil_system->mutex); + mutex_release(); /* Flush the spaces. It will not hurt to call fil_flush() on a non-existing space id. */ for (auto space_id : space_ids) { - fil_flush(space_id); - } -} - -/** Functor to validate the file node list of a tablespace. */ -struct Check { - /** Constructor */ - Check() : m_size(), m_n_open() {} - - /** Visit a file node - @param[in] elem file node to visit */ - void operator()(const fil_node_t* elem) - { - ut_a(elem->is_open || !elem->n_pending); - - if (elem->is_open) { - ++m_n_open; - } + mutex_acquire(); - m_size += elem->size; - } + space_flush(space_id); - /** Validate a tablespace. - @param[in] space tablespace to validate - @return number of open file nodes */ - static ulint validate(const fil_space_t* space) - { - ut_ad(mutex_own(&fil_system->mutex)); - Check check; - ut_list_validate(space->chain, check); - ut_a(space->size == check.m_size); - return(check.m_n_open); + mutex_release(); } +} - /** Total size of file nodes visited so far */ - ulint m_size; - /** Total number of open files visited so far */ - ulint m_n_open; -}; - -/******************************************************************//** -Checks the consistency of the tablespace cache. -@return true if ok */ -bool -fil_validate(void) -/*==============*/ +/** Flush the redo log writes to disk, possibly cached by the OS. */ +void +Fil_system::flush_file_redo() { - ulint n_open = 0; - - mutex_enter(&fil_system->mutex); - - /* Look for spaces in the hash table */ - - for (auto& elem : fil_system->spaces) { - - n_open += Check::validate(elem.second); - } - - ut_a(fil_system->n_open == n_open); - - UT_LIST_CHECK(fil_system->LRU); + m_shards[REDO_SHARD]->flush_file_redo(); +} - for (auto fil_node = UT_LIST_GET_FIRST(fil_system->LRU); - fil_node != 0; - fil_node = UT_LIST_GET_NEXT(LRU, fil_node)) { +/** Flush to disk the writes in file spaces of the given type +possibly cached by the OS. +@param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG, + can be ORred */ +void +Fil_system::flush_file_spaces(uint8_t purpose) +{ + for (auto shard : m_shards) { - ut_a(fil_node->is_open); - ut_a(fil_node->n_pending == 0); - ut_a(fil_space_belongs_in_lru(fil_node->space)); + shard->flush_file_spaces(purpose); } +} - mutex_exit(&fil_system->mutex); +/** Flush to disk the writes in file spaces of the given type +possibly cached by the OS. +@param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG, can be ORred */ +void +fil_flush_file_spaces(uint8_t purpose) +{ + fil_system->flush_file_spaces(purpose); +} - return(true); +/** Flush to disk the writes in file spaces of the given type +possibly cached by the OS. */ +void +fil_flush_file_redo() +{ + fil_system->flush_file_redo(); } -/********************************************************************//** -Returns true if file address is undefined. +/** Returns true if file address is undefined. +@param[in] addr Address @return true if undefined */ bool -fil_addr_is_null( -/*=============*/ - fil_addr_t addr) /*!< in: address */ +fil_addr_is_null(const fil_addr_t& addr) { return(addr.page == FIL_NULL); } -/********************************************************************//** -Get the predecessor of a file page. +/** Get the predecessor of a file page. +@param[in] page File page @return FIL_PAGE_PREV */ page_no_t -fil_page_get_prev( -/*==============*/ - const byte* page) /*!< in: file page */ +fil_page_get_prev(const byte* page) { return(mach_read_from_4(page + FIL_PAGE_PREV)); } -/********************************************************************//** -Get the successor of a file page. +/** Get the successor of a file page. +@param[in] page File page @return FIL_PAGE_NEXT */ page_no_t -fil_page_get_next( -/*==============*/ - const byte* page) /*!< in: file page */ +fil_page_get_next(const byte* page) { return(mach_read_from_4(page + FIL_PAGE_NEXT)); } -/*********************************************************************//** -Sets the file page type. */ +/** Sets the file page type. +@param[in,out] page File page +@param[in] type Page type */ void -fil_page_set_type( -/*==============*/ - byte* page, /*!< in/out: file page */ - ulint type) /*!< in: type */ +fil_page_set_type(byte* page, ulint type) { - ut_ad(page); - mach_write_to_2(page + FIL_PAGE_TYPE, type); } @@ -6978,41 +8788,28 @@ fil_page_reset_type( mlog_write_ulint(page + FIL_PAGE_TYPE, type, MLOG_2BYTES, mtr); } -/****************************************************************//** -Closes the tablespace memory cache. */ +/** Closes the tablespace memory cache. */ void -fil_close(void) -/*===========*/ +fil_close() { - if (fil_system == NULL) { + if (fil_system == nullptr) { return; } - call_destructor(&fil_system->names); - - call_destructor(&fil_system->spaces); - - call_destructor(&fil_system->m_open); - - ut_a(UT_LIST_GET_LEN(fil_system->LRU) == 0); - ut_a(UT_LIST_GET_LEN(fil_system->unflushed_spaces) == 0); - ut_a(UT_LIST_GET_LEN(fil_system->space_list) == 0); + UT_DELETE(fil_system); - mutex_free(&fil_system->mutex); - - ut_free(fil_system); - fil_system = NULL; + fil_system = nullptr; } #ifndef UNIV_HOTBACKUP -/********************************************************************//** -Initializes a buffer control block when the buf_pool is created. */ +/** Initializes a buffer control block when the buf_pool is created. +@param[in] block Pointer to the control block +@param[in] frame Pointer to buffer frame */ static void fil_buf_block_init( -/*===============*/ - buf_block_t* block, /*!< in: pointer to control block */ - byte* frame) /*!< in: pointer to buffer frame */ + buf_block_t* block, + byte* frame) { UNIV_MEM_DESC(frame, UNIV_PAGE_SIZE); @@ -7026,44 +8823,62 @@ fil_buf_block_init( page_zip_des_init(&block->page.zip); } -struct fil_iterator_t { - pfs_os_file_t file; /*!< File handle */ - const char* filepath; /*!< File path name */ - os_offset_t start; /*!< From where to start */ - os_offset_t end; /*!< Where to stop */ - os_offset_t file_size; /*!< File size in bytes */ - ulint page_size; /*!< Page size */ - ulint n_io_buffers; /*!< Number of pages to use - for IO */ - byte* io_buffer; /*!< Buffer to use for IO */ - byte* encryption_key; /*!< Encryption key */ - byte* encryption_iv; /*!< Encryption iv */ +struct Fil_page_iterator { + /** File handle */ + pfs_os_file_t m_file; + + /** File path name */ + const char* m_filepath; + + /** From where to start */ + os_offset_t m_start; + + /** Where to stop */ + os_offset_t m_end; + + /* File size in bytes */ + os_offset_t m_file_size; + + /** Page size */ + size_t m_page_size; + + /** Number of pages to use for I/O */ + size_t m_n_io_buffers; + + /** Buffer to use for IO */ + byte* m_io_buffer; + + /** Encryption key */ + byte* m_encryption_key; + + /** Encruption iv */ + byte* m_encryption_iv; }; -/********************************************************************//** -TODO: This can be made parallel trivially by chunking up the file and creating -a callback per thread. Main benefit will be to use multiple CPUs for -checksums and compressed tables. We have to do compressed tables block by -block right now. Secondly we need to decompress/compress and copy too much -of data. These are CPU intensive. +/** TODO: This can be made parallel trivially by chunking up the file +and creating a callback per thread. Main benefit will be to use multiple +CPUs for checksums and compressed tables. We have to do compressed tables +block by block right now. Secondly we need to decompress/compress and copy +too much of data. These are CPU intensive. Iterate over all the pages in the tablespace. -@param iter Tablespace iterator -@param block block to use for IO -@param callback Callback to inspect and update page contents +@param[in] iter Tablespace iterator +@param[in,out] block Block to use for IO +@param[in] callback Callback to inspect and update page contents @retval DB_SUCCESS or error code */ static dberr_t fil_iterate( -/*========*/ - const fil_iterator_t& iter, + const Fil_page_iterator&iter, buf_block_t* block, PageCallback& callback) { os_offset_t offset; + size_t n_bytes; page_no_t page_no = 0; space_id_t space_id = callback.get_space_id(); - ulint n_bytes = iter.n_io_buffers * iter.page_size; + + n_bytes = iter.m_n_io_buffers * iter.m_page_size; ut_ad(!srv_read_only_mode); @@ -7074,30 +8889,30 @@ fil_iterate( ulint read_type = IORequest::READ; ulint write_type = IORequest::WRITE; - for (offset = iter.start; offset < iter.end; offset += n_bytes) { + for (offset = iter.m_start; offset < iter.m_end; offset += n_bytes) { - byte* io_buffer = iter.io_buffer; + byte* io_buffer = iter.m_io_buffer; block->frame = io_buffer; if (callback.get_page_size().is_compressed()) { page_zip_des_init(&block->page.zip); - page_zip_set_size(&block->page.zip, iter.page_size); + page_zip_set_size(&block->page.zip, iter.m_page_size); block->page.size.copy_from( - page_size_t(iter.page_size, + page_size_t(iter.m_page_size, univ_page_size.logical(), true)); block->page.zip.data = block->frame + UNIV_PAGE_SIZE; ut_d(block->page.zip.m_external = true); - ut_ad(iter.page_size + ut_ad(iter.m_page_size == callback.get_page_size().physical()); /* Zip IO is done in the compressed page buffer. */ io_buffer = block->page.zip.data; } else { - io_buffer = iter.io_buffer; + io_buffer = iter.m_io_buffer; } /* We have to read the exact number of bytes. Otherwise the @@ -7105,24 +8920,26 @@ fil_iterate( n_bytes = static_cast( ut_min(static_cast(n_bytes), - iter.end - offset)); + iter.m_end - offset)); ut_ad(n_bytes > 0); - ut_ad(!(n_bytes % iter.page_size)); + ut_ad(!(n_bytes % iter.m_page_size)); dberr_t err; IORequest read_request(read_type); /* For encrypted table, set encryption information. */ - if (iter.encryption_key != NULL && offset != 0) { - read_request.encryption_key(iter.encryption_key, - ENCRYPTION_KEY_LEN, - iter.encryption_iv); + if (iter.m_encryption_key != nullptr && offset != 0) { + + read_request.encryption_key( + iter.m_encryption_key, ENCRYPTION_KEY_LEN, + iter.m_encryption_iv); + read_request.encryption_algorithm(Encryption::AES); } err = os_file_read( - read_request, iter.file, io_buffer, offset, + read_request, iter.m_file, io_buffer, offset, (ulint) n_bytes); if (err != DB_SUCCESS) { @@ -7132,11 +8949,13 @@ fil_iterate( return(err); } + size_t n_pages_read; bool updated = false; os_offset_t page_off = offset; - ulint n_pages_read = (ulint) n_bytes / iter.page_size; - for (ulint i = 0; i < n_pages_read; ++i) { + n_pages_read = (ulint) n_bytes / iter.m_page_size; + + for (size_t i = 0; i < n_pages_read; ++i) { buf_block_set_file_page( block, page_id_t(space_id, page_no++)); @@ -7146,6 +8965,7 @@ fil_iterate( return(err); } else if (!updated) { + updated = buf_block_get_state(block) == BUF_BLOCK_FILE_PAGE; } @@ -7153,17 +8973,19 @@ fil_iterate( buf_block_set_state(block, BUF_BLOCK_NOT_USED); buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); - page_off += iter.page_size; - block->frame += iter.page_size; + page_off += iter.m_page_size; + block->frame += iter.m_page_size; } IORequest write_request(write_type); /* For encrypted table, set encryption information. */ - if (iter.encryption_key != NULL && offset != 0) { - write_request.encryption_key(iter.encryption_key, - ENCRYPTION_KEY_LEN, - iter.encryption_iv); + if (iter.m_encryption_key != nullptr && offset != 0) { + + write_request.encryption_key( + iter.m_encryption_key, + ENCRYPTION_KEY_LEN, iter.m_encryption_iv); + write_request.encryption_algorithm(Encryption::AES); } @@ -7174,7 +8996,7 @@ fil_iterate( if (updated && (err = os_file_write( write_request, - iter.filepath, iter.file, io_buffer, + iter.m_filepath, iter.m_file, io_buffer, offset, (ulint) n_bytes)) != DB_SUCCESS) { /* This is not a hard error */ @@ -7194,15 +9016,13 @@ fil_iterate( return(DB_SUCCESS); } -/********************************************************************//** -Iterate over all the pages in the tablespace. -@param table the table definiton in the server -@param n_io_buffers number of blocks to read and write together -@param callback functor that will do the page updates +/** Iterate over all the pages in the tablespace. +@param[in,out] table the table definiton in the server +@param[in] n_io_buffers number of blocks to read and write together +@param[in] callback functor that will do the page updates @return DB_SUCCESS or error code */ dberr_t fil_tablespace_iterate( -/*===================*/ dict_table_t* table, ulint n_io_buffers, PageCallback& callback) @@ -7219,19 +9039,13 @@ fil_tablespace_iterate( return(DB_CORRUPTION);); /* Make sure the data_dir_path is set. */ - dd_get_and_save_data_dir_path(table, NULL, false); + dd_get_and_save_data_dir_path(table, nullptr, false); - if (DICT_TF_HAS_DATA_DIR(table->flags)) { - ut_a(table->data_dir_path); + std::string path = dict_table_get_datadir(table); - filepath = fil_make_filepath( - table->data_dir_path, table->name.m_name, IBD, true); - } else { - filepath = fil_make_filepath( - NULL, table->name.m_name, IBD, false); - } + filepath = Fil_path::make(path, table->name.m_name, IBD, true); - if (filepath == NULL) { + if (filepath == nullptr) { return(DB_OUT_OF_MEMORY); } @@ -7298,24 +9112,25 @@ fil_tablespace_iterate( err = DB_IO_ERROR; } else if ((err = callback.init(file_size, block)) == DB_SUCCESS) { - fil_iterator_t iter; + Fil_page_iterator iter; - iter.file = file; - iter.start = 0; - iter.end = file_size; - iter.filepath = filepath; - iter.file_size = file_size; - iter.n_io_buffers = n_io_buffers; - iter.page_size = callback.get_page_size().physical(); + iter.m_file = file; + iter.m_start = 0; + iter.m_end = file_size; + iter.m_filepath = filepath; + iter.m_file_size = file_size; + iter.m_n_io_buffers = n_io_buffers; + iter.m_page_size = callback.get_page_size().physical(); /* Set encryption info. */ - iter.encryption_key = table->encryption_key; - iter.encryption_iv = table->encryption_iv; + iter.m_encryption_key = table->encryption_key; + iter.m_encryption_iv = table->encryption_iv; /* Check encryption is matched or not. */ ulint space_flags = callback.get_space_flags(); + if (FSP_FLAGS_GET_ENCRYPTION(space_flags)) { - ut_ad(table->encryption_key != NULL); + ut_ad(table->encryption_key != nullptr); if (!dict_table_is_encrypted(table)) { ib::error() << "Table is not in an encrypted" @@ -7332,17 +9147,17 @@ fil_tablespace_iterate( for now. We do the IMPORT page by page. */ if (callback.get_page_size().is_compressed()) { - iter.n_io_buffers = 1; - ut_a(iter.page_size + iter.m_n_io_buffers = 1; + ut_a(iter.m_page_size == callback.get_page_size().physical()); } /** Add an extra page for compressed page scratch area. */ void* io_buffer = ut_malloc_nokey( - (2 + iter.n_io_buffers) * UNIV_PAGE_SIZE); + (2 + iter.m_n_io_buffers) * UNIV_PAGE_SIZE); - iter.io_buffer = static_cast( + iter.m_io_buffer = static_cast( ut_align(io_buffer, UNIV_PAGE_SIZE)); err = fil_iterate(iter, block, callback); @@ -7379,165 +9194,49 @@ fil_tablespace_iterate( /** Set the tablespace table size. @param[in] page a page belonging to the tablespace */ void -PageCallback::set_page_size( - const buf_frame_t* page) UNIV_NOTHROW +PageCallback::set_page_size(const buf_frame_t* page) UNIV_NOTHROW { m_page_size.copy_from(fsp_header_get_page_size(page)); } -/********************************************************************//** -Delete the tablespace file and any related files like .cfg. +/** Delete the tablespace file and any related files like .cfg. This should not be called for temporary tables. -@param[in] ibd_filepath File path of the IBD tablespace */ -void -fil_delete_file( -/*============*/ - const char* ibd_filepath) +@param[in] path File path of the IBD tablespace +@return true on success */ +bool +fil_delete_file(const char* path) { - /* Force a delete of any stale .ibd files that are lying around. */ + bool success = true; - ib::info() << "Deleting " << ibd_filepath; + /* Force a delete of any stale .ibd files that are lying around. */ + success = os_file_delete_if_exists( + innodb_data_file_key, path, nullptr); -#ifdef _WIN32 - /* For Windows, we need to check the status of the file. - If there's a subdir with same name, we will skip to delete it. - Otherwise, it'll keep looping in os_file_delete_if_exists_func. */ - os_file_type_t type; - bool exists; - - os_file_status(ibd_filepath, &exists, &type); - if (type == OS_FILE_TYPE_DIR) { - ib::info() << "There is a directory with same name, skip deleting " - << ibd_filepath; - } else { - os_file_delete_if_exists(innodb_data_file_key, ibd_filepath, NULL); - } -#else - os_file_delete_if_exists(innodb_data_file_key, ibd_filepath, NULL); -#endif /* _WIN32 */ + char* cfg_filepath = Fil_path::make_cfg(path); - char* cfg_filepath = fil_make_filepath( - ibd_filepath, NULL, CFG, false); - if (cfg_filepath != NULL) { - os_file_delete_if_exists( - innodb_data_file_key, cfg_filepath, NULL); - ut_free(cfg_filepath); - } + if (cfg_filepath != nullptr) { - char* cfp_filepath = fil_make_filepath( - ibd_filepath, NULL, CFP, false); - if (cfp_filepath != NULL) { os_file_delete_if_exists( - innodb_data_file_key, cfp_filepath, NULL); - ut_free(cfp_filepath); - } -} - -/** -Iterate over all the spaces in the space list and fetch the -tablespace names. It will return a copy of the name that must be -freed by the caller using: delete[]. -@return DB_SUCCESS if all OK. */ -dberr_t -fil_get_space_names( -/*================*/ - space_name_list_t& space_name_list) - /*!< in/out: List to append to */ -{ - fil_space_t* space; - dberr_t err = DB_SUCCESS; - - mutex_enter(&fil_system->mutex); - - for (space = UT_LIST_GET_FIRST(fil_system->space_list); - space != NULL; - space = UT_LIST_GET_NEXT(space_list, space)) { - - if (space->purpose == FIL_TYPE_TABLESPACE) { - ulint len; - char* name; - - len = ::strlen(space->name); - name = UT_NEW_ARRAY_NOKEY(char, len + 1); - - if (name == 0) { - /* Caller to free elements allocated so far. */ - err = DB_OUT_OF_MEMORY; - break; - } + innodb_data_file_key, cfg_filepath, nullptr); - memcpy(name, space->name, len); - name[len] = 0; - - space_name_list.push_back(name); - } + ut_free(cfg_filepath); } - mutex_exit(&fil_system->mutex); - - return(err); -} - -/** Return the next fil_node_t in the current or next fil_space_t. -Once started, the caller must keep calling this until it returns NULL. -fil_space_acquire() and fil_space_release() are invoked here which -blocks a concurrent operation from dropping the tablespace. -@param[in] prev_node Pointer to the previous fil_node_t. -If NULL, use the first fil_space_t on fil_system->space_list. -@return pointer to the next fil_node_t. -@retval NULL if this was the last file node */ -const fil_node_t* -fil_node_next( - const fil_node_t* prev_node) -{ - fil_space_t* space; - const fil_node_t* node = prev_node; - - mutex_enter(&fil_system->mutex); - - if (node == NULL) { - space = UT_LIST_GET_FIRST(fil_system->space_list); + char* cfp_filepath = Fil_path::make_cfp(path); - /* We can trust that space is not NULL because at least the - system tablespace is always present and loaded first. */ - space->n_pending_ops++; + if (cfp_filepath != nullptr) { - node = UT_LIST_GET_FIRST(space->chain); - ut_ad(node != NULL); - } else { - space = node->space; - ut_ad(space->n_pending_ops > 0); - node = UT_LIST_GET_NEXT(chain, node); - - if (node == NULL) { - /* Move on to the next fil_space_t */ - space->n_pending_ops--; - space = UT_LIST_GET_NEXT(space_list, space); - - /* Skip spaces that are being - created by fil_ibd_create(), - or dropped. */ - while (space != NULL - && (UT_LIST_GET_LEN(space->chain) == 0 - || space->stop_new_ops)) { - space = UT_LIST_GET_NEXT(space_list, space); - } + os_file_delete_if_exists( + innodb_data_file_key, cfp_filepath, nullptr); - if (space != NULL) { - space->n_pending_ops++; - node = UT_LIST_GET_FIRST(space->chain); - ut_ad(node != NULL); - } - } + ut_free(cfp_filepath); } - mutex_exit(&fil_system->mutex); - - return(node); + return(success); } #ifndef UNIV_HOTBACKUP -/** Check if swapping two .ibd files can be done without failure +/** Check if swapping two .ibd files can be done without failure. @param[in] old_table old table @param[in] new_table new table @param[in] tmp_name temporary table name @@ -7562,20 +9261,21 @@ fil_rename_precheck( return(DB_SUCCESS); } - const char* old_dir = DICT_TF_HAS_DATA_DIR(old_table->flags) - ? old_table->data_dir_path - : NULL; + auto old_dir = dict_table_get_datadir(old_table); - char* old_path = fil_make_filepath( - old_dir, old_table->name.m_name, IBD, (old_dir != NULL)); - if (old_path == NULL) { + char* old_path = Fil_path::make( + old_dir, old_table->name.m_name, IBD, !old_dir.empty()); + + if (old_path == nullptr) { return(DB_OUT_OF_MEMORY); } if (old_is_file_per_table) { - char* tmp_path = fil_make_filepath( - old_dir, tmp_name, IBD, (old_dir != NULL)); - if (tmp_path == NULL) { + + char* tmp_path = Fil_path::make( + old_dir, tmp_name, IBD, !old_dir.empty()); + + if (tmp_path == nullptr) { ut_free(old_path); return(DB_OUT_OF_MEMORY); } @@ -7584,6 +9284,7 @@ fil_rename_precheck( err = fil_rename_tablespace_check( old_table->space, old_path, tmp_path, dict_table_is_discarded(old_table)); + if (err != DB_SUCCESS) { ut_free(old_path); ut_free(tmp_path); @@ -7594,13 +9295,13 @@ fil_rename_precheck( } if (new_is_file_per_table) { - const char* new_dir = DICT_TF_HAS_DATA_DIR(new_table->flags) - ? new_table->data_dir_path - : NULL; - char* new_path = fil_make_filepath( - new_dir, new_table->name.m_name, - IBD, (new_dir != NULL)); - if (new_path == NULL) { + + auto new_dir = dict_table_get_datadir(new_table); + + char* new_path = Fil_path::make( + new_dir, new_table->name.m_name, IBD, !new_dir.empty()); + + if (new_path == nullptr) { ut_free(old_path); return(DB_OUT_OF_MEMORY); } @@ -7608,9 +9309,11 @@ fil_rename_precheck( /* Destination filepath must not exist unless this ALTER TABLE starts and ends with a file_per-table tablespace. */ if (!old_is_file_per_table) { + err = fil_rename_tablespace_check( new_table->space, new_path, old_path, dict_table_is_discarded(new_table)); + if (err != DB_SUCCESS) { ut_free(old_path); ut_free(new_path); @@ -7629,11 +9332,11 @@ fil_rename_precheck( /** Note that the file system where the file resides doesn't support PUNCH HOLE. Called from AIO handlers when IO returns DB_IO_NO_PUNCH_HOLE -@param[in,out] node Node to set */ +@param[in,out] file file to set */ void -fil_no_punch_hole(fil_node_t* node) +fil_no_punch_hole(fil_node_t* file) { - node->punch_hole = false; + file->punch_hole = false; } /** Set the compression type for the tablespace of a table @@ -7645,7 +9348,7 @@ fil_set_compression( dict_table_t* table, const char* algorithm) { - ut_ad(table != NULL); + ut_ad(table != nullptr); /* We don't support Page Compression for the system tablespace, the temporary tablespace, or any general tablespace because @@ -7663,11 +9366,11 @@ fil_set_compression( dberr_t err; Compression compression; - if (algorithm == NULL || strlen(algorithm) == 0) { + if (algorithm == nullptr || strlen(algorithm) == 0) { #ifndef UNIV_DEBUG compression.m_type = Compression::NONE; -#else +#else /* UNIV_DEBUG */ /* This is a Debug tool for setting compression on all compressible tables not otherwise specified. */ switch (srv_debug_compress) { @@ -7695,7 +9398,7 @@ fil_set_compression( fil_space_t* space = fil_space_get(table->space); - if (space == NULL) { + if (space == nullptr) { return(DB_NOT_FOUND); } @@ -7703,11 +9406,7 @@ fil_set_compression( if (space->compression_type != Compression::NONE) { - const fil_node_t* node; - - node = UT_LIST_GET_FIRST(space->chain); - - if (!node->punch_hole) { + if (!space->files.front().punch_hole) { return(DB_IO_NO_PUNCH_HOLE_FS); } @@ -7724,7 +9423,7 @@ fil_get_compression(space_id_t space_id) { fil_space_t* space = fil_space_get(space_id); - return(space == NULL ? Compression::NONE : space->compression_type); + return(space == nullptr ? Compression::NONE : space->compression_type); } /** Set the encryption type for the tablespace @@ -7746,201 +9445,244 @@ fil_set_encryption( return(DB_IO_NO_ENCRYPT_TABLESPACE); } - mutex_enter(&fil_system->mutex); + auto shard = fil_system->shard_by_id(space_id); + + shard->mutex_acquire(); - fil_space_t* space = fil_space_get_by_id(space_id); + fil_space_t* space = shard->get_space_by_id(space_id); - if (space == NULL) { - mutex_exit(&fil_system->mutex); + if (space == nullptr) { + shard->mutex_release(); return(DB_NOT_FOUND); } - if (key == NULL) { + if (key == nullptr) { Encryption::random_value(space->encryption_key); } else { - memcpy(space->encryption_key, - key, ENCRYPTION_KEY_LEN); + memcpy(space->encryption_key, key, ENCRYPTION_KEY_LEN); } space->encryption_klen = ENCRYPTION_KEY_LEN; - if (iv == NULL) { + + if (iv == nullptr) { Encryption::random_value(space->encryption_iv); } else { - memcpy(space->encryption_iv, - iv, ENCRYPTION_KEY_LEN); + memcpy(space->encryption_iv, iv, ENCRYPTION_KEY_LEN); } ut_ad(algorithm != Encryption::NONE); space->encryption_type = algorithm; - mutex_exit(&fil_system->mutex); + shard->mutex_release(); return(DB_SUCCESS); } #ifndef UNIV_HOTBACKUP /** Rotate the tablespace keys by new master key. -@return true if the re-encrypt suceeds */ +@param[in,out] shard Rotate the keys in this shard +@return true if the re-encrypt succeeds */ bool -fil_encryption_rotate() +Fil_system::encryption_rotate_in_a_shard(Fil_shard* shard) { - fil_space_t* space; - mtr_t mtr; - byte encrypt_info[ENCRYPTION_INFO_SIZE_V2]; - - for (space = UT_LIST_GET_FIRST(fil_system->space_list); - space != NULL; ) { - /* Skip unencypted tablespaces. */ - /* Encrypted redo log tablespaces is handled in function - log_rotate_encryption. */ + byte encrypt_info[ENCRYPTION_INFO_SIZE_V2]; + + for (auto& elem : shard->m_spaces) { + + auto space = elem.second; + + /* Skip unencypted tablespaces. Encrypted redo log + tablespaces is handled in function log_rotate_encryption. */ + if (fsp_is_system_or_temp_tablespace(space->id) || space->purpose == FIL_TYPE_LOG) { - space = UT_LIST_GET_NEXT(space_list, space); + continue; } - /* Skip the undo tablespace when it's in default - key status, since it's the first server startup - after bootstrap, and the server uuid is not ready - yet. */ + /* Skip the undo tablespace when it's in default key status, + since it's the first server startup after bootstrap, and the + server uuid is not ready yet. */ + if (fsp_is_undo_tablespace(space->id) - && Encryption::master_key_id == - ENCRYPTION_DEFAULT_MASTER_KEY_ID) { - space = UT_LIST_GET_NEXT(space_list, space); + && Encryption::s_master_key_id + == ENCRYPTION_DEFAULT_MASTER_KEY_ID) { + continue; } /* Rotate the encrypted tablespaces. */ if (space->encryption_type != Encryption::NONE) { + mtr_t mtr; + mtr_start(&mtr); mtr_x_lock_space(space, &mtr); memset(encrypt_info, 0, ENCRYPTION_INFO_SIZE_V2); - if (!fsp_header_rotate_encryption(space, - encrypt_info, - &mtr)) { + if (!fsp_header_rotate_encryption( + space, encrypt_info, &mtr)) { + mtr_commit(&mtr); + return(false); } mtr_commit(&mtr); } - space = UT_LIST_GET_NEXT(space_list, space); DBUG_EXECUTE_IF("ib_crash_during_rotation_for_encryption", DBUG_SUICIDE();); } return(true); } -#endif /* !UNIV_HOTBACKUP */ -/** Build the basic folder name from the path and length provided -@param[in] path pathname (may also include the file basename) -@param[in] len length of the path, in bytes */ -void -Folder::make_path(const char* path, size_t len) +/** Rotate the tablespace keys by new master key. +@return true if the re-encrypt succeeds */ +bool +Fil_system::encryption_rotate_all() { - if (is_absolute_path(path)) { - m_folder = mem_strdupl(path, len); - m_folder_len = len; - } else { - size_t n = 2 + len + strlen(fil_path_to_mysql_datadir); - m_folder = static_cast(ut_malloc_nokey(n)); - m_folder_len = 0; - - if (path != fil_path_to_mysql_datadir) { - /* Put the mysqld datadir into m_folder first. */ - ut_ad(fil_path_to_mysql_datadir[0] != '\0'); - m_folder_len = strlen(fil_path_to_mysql_datadir); - memcpy(m_folder, fil_path_to_mysql_datadir, - m_folder_len); - if (m_folder[m_folder_len - 1] != OS_PATH_SEPARATOR) { - m_folder[m_folder_len++] = OS_PATH_SEPARATOR; - } - } + for (auto shard : m_shards) { + + // FIXME: We don't acquire the fil_sys::mutex here. Why? - /* Append the path. */ - memcpy(m_folder + m_folder_len, path, len); - m_folder_len += len; - m_folder[m_folder_len] = '\0'; + bool success = encryption_rotate_in_a_shard(shard); + + if (!success) { + return(false); + } } - os_normalize_path(m_folder); + return(true); } -/** Resolve a relative path in m_folder to an absolute path -in m_abs_path setting m_abs_len. */ -void -Folder::make_abs_path() +/** Rotate the tablespace keys by new master key. +@return true if the re-encrypt succeeds */ +bool +fil_encryption_rotate() { - my_realpath(m_abs_path, m_folder, MYF(0)); - m_abs_len = strlen(m_abs_path); + return(fil_system->encryption_rotate_all()); +} - ut_ad(m_abs_len + 1 < sizeof(m_abs_path)); +#endif /* !UNIV_HOTBACKUP */ - /* Folder::related_to() needs a trailing separator. */ - if (m_abs_path[m_abs_len - 1] != OS_PATH_SEPARATOR) { - m_abs_path[m_abs_len] = OS_PATH_SEPARATOR; - m_abs_path[++m_abs_len] = '\0'; - } +/** Constructor +@param[in] path pathname (may also include the file basename) + It's the callers responsibility to ensure that + the path is normalized. */ +Fil_path::Fil_path(const std::string& path) + : + m_path(path) +{ + m_abs_path = get_real_path(m_path); } /** Constructor -@param[in] path pathname (may also include the file basename) +@param[in] path pathname (may also include the file basename) + It's the callers responsibility to ensure that + the path is normalized. @param[in] len length of the path, in bytes */ -Folder::Folder(const char* path, size_t len) +Fil_path::Fil_path(const char* path, size_t len) + : + m_path(path, len) { - make_path(path, len); - make_abs_path(); + m_abs_path = get_real_path(m_path); } -/** Assignment operator -@param[in] path folder string provided */ -class Folder& -Folder::operator=(const char* path) +/** Default constructor. */ +Fil_path::Fil_path() + : + m_path(), + m_abs_path() { - ut_free(m_folder); - make_path(path, strlen(path)); - make_abs_path(); + /* No op */ +} - return(*this); +/** Destructor */ +Fil_path::~Fil_path() +{ + /* No op */ } -/** Determine if the directory referenced by m_folder exists. -@return whether the directory exists */ -bool -Folder::exists() +/** @return true if the path exists and is a file . */ +os_file_type_t +Fil_path::get_file_type(const std::string& path) { - bool exists; - os_file_type_t type; + const std::string* ptr; + os_file_type_t type; + bool exists; #ifdef _WIN32 /* Temporarily strip the trailing_separator since it will cause stat64() to fail on Windows unless the path is the root of some - drive; like "c:\". _stat64() will fail if it is "c:". */ - size_t len = strlen(m_abs_path); + drive; like "C:\". _stat64() will fail if it is "C:". */ + + std::string p{path}; - if (m_abs_path[m_abs_len - 1] == OS_PATH_SEPARATOR - && m_abs_path[m_abs_len - 2] != ':') { + if (path.length() > 3 + && is_separator(path.back()) + && path.at(p.length() - 2) != ':') { - m_abs_path[m_abs_len - 1] = '\0'; + p.pop_back(); } + + ptr = &p; +#else + ptr = &path; #endif /* WIN32 */ - bool ret = os_file_status(m_abs_path, &exists, &type); + os_file_status(ptr->c_str(), &exists, &type); + + return(type); +} + +/** @return true if the path exists and is a file . */ +bool +Fil_path::is_file_and_exists() const +{ + return(get_file_type(m_abs_path) == OS_FILE_TYPE_FILE); +} + +/** @return true if the path exists and is a directory. */ +bool +Fil_path::is_directory_and_exists() const +{ + return(get_file_type(m_abs_path) == OS_FILE_TYPE_DIR); +} + +/** This validation is only for ':'. +@return true if the path is valid. */ +bool +Fil_path::is_valid() const +{ + auto count = std::count(m_path.begin(), m_path.end(), ':'); + + if (count == 0) { + return(true); + } #ifdef _WIN32 - /* Put the separator back on. */ - if (m_abs_path[m_abs_len - 1] == '\0') { - m_abs_path[m_abs_len - 1] = OS_PATH_SEPARATOR; + /* Do not allow names like "C:name.ibd" because it + specifies the "C:" drive but allows a relative location. + It should be like "c:\". If a single colon is used it + must be the second byte and the third byte must be a + separator. */ + + /* 8 == strlen("c:\a,ibd") */ + if (count == 1 + && m_path.length() >= 8 + && isalpha(m_path.at(0)) + && m_path.at(1) == ':' + && (m_path.at(2) == '\\' || m_path.at(2) == '/')) { + + return(true); } -#endif /* WIN32 */ +#endif /* _WIN32 */ - return(ret && exists && type == OS_FILE_TYPE_DIR); + return(false); } /** Sets the flags of the tablespace. The tablespace must be locked @@ -7955,13 +9697,16 @@ fil_space_set_flags( ut_ad(fsp_flags_is_valid(flags)); rw_lock_x_lock(&space->latch); - space->flags = flags; + + ut_a(flags < std::numeric_limits::max()); + space->flags = (uint32_t) flags; + rw_lock_x_unlock(&space->latch); } /* Unit Tests */ #ifdef UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH -#define MF fil_make_filepath +#define MF Fil_path::make #define DISPLAY ib::info() << path void test_make_filepath() @@ -7980,33 +9725,32 @@ test_make_filepath() "oooooooooooooooooooooooooooooooooooooooooooooooooo" "oooooooooooooooooooooooooooooooooooooooooooooooong" "/folder/name"; - path = MF("/this/is/a/path/with/a/filename", NULL, IBD, false); DISPLAY; - path = MF("/this/is/a/path/with/a/filename", NULL, ISL, false); DISPLAY; - path = MF("/this/is/a/path/with/a/filename", NULL, CFG, false); DISPLAY; - path = MF("/this/is/a/path/with/a/filename", NULL, CFP, false); DISPLAY; - path = MF("/this/is/a/path/with/a/filename.ibd", NULL, IBD, false); DISPLAY; - path = MF("/this/is/a/path/with/a/filename.ibd", NULL, IBD, false); DISPLAY; - path = MF("/this/is/a/path/with/a/filename.dat", NULL, IBD, false); DISPLAY; - path = MF(NULL, "tablespacename", NO_EXT, false); DISPLAY; - path = MF(NULL, "tablespacename", IBD, false); DISPLAY; - path = MF(NULL, "dbname/tablespacename", NO_EXT, false); DISPLAY; - path = MF(NULL, "dbname/tablespacename", IBD, false); DISPLAY; - path = MF(NULL, "dbname/tablespacename", ISL, false); DISPLAY; - path = MF(NULL, "dbname/tablespacename", CFG, false); DISPLAY; - path = MF(NULL, "dbname/tablespacename", CFP, false); DISPLAY; - path = MF(NULL, "dbname\\tablespacename", NO_EXT, false); DISPLAY; - path = MF(NULL, "dbname\\tablespacename", IBD, false); DISPLAY; + path = MF("/this/is/a/path/with/a/filename", nullptr, IBD, false); DISPLAY; + path = MF("/this/is/a/path/with/a/filename", nullptr, ISL, false); DISPLAY; + path = MF("/this/is/a/path/with/a/filename", nullptr, CFG, false); DISPLAY; + path = MF("/this/is/a/path/with/a/filename", nullptr, CFP, false); DISPLAY; + path = MF("/this/is/a/path/with/a/filename.ibd", nullptr, IBD, false); DISPLAY; + path = MF("/this/is/a/path/with/a/filename.ibd", nullptr, IBD, false); DISPLAY; + path = MF("/this/is/a/path/with/a/filename.dat", nullptr, IBD, false); DISPLAY; + path = MF(nullptr, "tablespacename", NO_EXT, false); DISPLAY; + path = MF(nullptr, "tablespacename", IBD, false); DISPLAY; + path = MF(nullptr, "dbname/tablespacename", NO_EXT, false); DISPLAY; + path = MF(nullptr, "dbname/tablespacename", IBD, false); DISPLAY; + path = MF(nullptr, "dbname/tablespacename", ISL, false); DISPLAY; + path = MF(nullptr, "dbname/tablespacename", CFG, false); DISPLAY; + path = MF(nullptr, "dbname/tablespacename", CFP, false); DISPLAY; + path = MF(nullptr, "dbname\\tablespacename", NO_EXT, false); DISPLAY; + path = MF(nullptr, "dbname\\tablespacename", IBD, false); DISPLAY; path = MF("/this/is/a/path", "dbname/tablespacename", IBD, false); DISPLAY; path = MF("/this/is/a/path", "dbname/tablespacename", IBD, true); DISPLAY; path = MF("./this/is/a/path", "dbname/tablespacename.ibd", IBD, true); DISPLAY; path = MF("this\\is\\a\\path", "dbname/tablespacename", IBD, true); DISPLAY; path = MF("/this/is/a/path", "dbname\\tablespacename", IBD, true); DISPLAY; - path = MF(long_path, NULL, IBD, false); DISPLAY; + path = MF(long_path, nullptr, IBD, false); DISPLAY; path = MF(long_path, "tablespacename", IBD, false); DISPLAY; path = MF(long_path, "tablespacename", IBD, true); DISPLAY; } #endif /* UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH */ -/* @} */ /** Release the reserved free extents. @param[in] n_reserved number of reserved extents */ @@ -8017,12 +9761,16 @@ fil_space_t::release_free_extents(ulint n_reserved) ut_ad(rw_lock_own(&latch, RW_LOCK_X)); #endif /* !UNIV_HOTBACKUP */ + ut_a(n_reserved < std::numeric_limits::max()); ut_a(n_reserved_extents >= n_reserved); - n_reserved_extents -= n_reserved; + + n_reserved_extents -= (uint32_t) n_reserved; } #ifndef UNIV_HOTBACKUP -#ifdef UNIV_DEBUG + +# ifdef UNIV_DEBUG + /** Print the extent descriptor pages of this tablespace into the given file. @param[in] filename the output file name. */ @@ -8046,6 +9794,7 @@ fil_space_t::print_xdes_pages(std::ostream& out) const mtr_start(&mtr); for (page_no_t i = 0; i < 100; ++i) { + page_no_t xdes_page_no = i * UNIV_PAGE_SIZE; if (xdes_page_no >= size) { @@ -8062,2073 +9811,1912 @@ fil_space_t::print_xdes_pages(std::ostream& out) const switch (page_type) { case FIL_PAGE_TYPE_ALLOCATED: - ut_ad(xdes_page_no >= free_limit); - goto finish; - case FIL_PAGE_TYPE_FSP_HDR: - case FIL_PAGE_TYPE_XDES: - break; - default: - ut_error; - } - - xdes_page_print(out, page, xdes_page_no, &mtr); - } -finish: - mtr_commit(&mtr); - return(out); -} -#endif /* UNIV_DEBUG */ -#endif /* !UNIV_HOTBACKUP */ - -/** Redo log an open file request. -@param[in] space_id Tablespace ID being opened -@param[in] path Physical path of the file opened */ -void -Fil_Open::log(space_id_t space_id, const std::string& path) -{ -#ifndef UNIV_HOTBACKUP - if (!srv_read_only_mode && !recv_recovery_is_on()) { - lsn_t lsn; - - if (!srv_is_being_started) { - - DBUG_EXECUTE_IF( - "ib_crash_rename_log_1", - DBUG_SUICIDE();); - - mtr_t mtr; - mtr_start(&mtr); + ut_ad(xdes_page_no >= free_limit); - fil_op_write_log( - MLOG_FILE_OPEN, - space_id, 0, path.c_str(), NULL, 0, &mtr); - mtr_commit(&mtr); + return(out); - lsn = mtr.commit_lsn(); - } else { - lsn = log_get_lsn(); + case FIL_PAGE_TYPE_FSP_HDR: + case FIL_PAGE_TYPE_XDES: + break; + default: + ut_error; } - open(space_id, path, lsn); - } -#endif /* !UNIV_HOTBACKUP */ -} - -/** Get the first name a tablespace ID maps to. -@param[in] space_id Tablespace ID to lookup -@return first name in the mapping or empty string if not found */ -std::string -Fil_Open::fetch(space_id_t space_id) const -{ - const auto it = m_spaces.find(space_id); - - if (it == m_spaces.end() - || it->second.m_files.empty()) { - - return(""); + xdes_page_print(out, page, xdes_page_no, &mtr); } - return(it->second.m_files.front().m_name); -} - -/** Check if a space ID is known. -@param[in] space_id Tablespace ID to lookup -@return true if the space ID is known */ -bool -Fil_Open::lookup(space_id_t space_id) const -{ - const auto it = m_spaces.find(space_id); - - return(it != m_spaces.end()); + mtr_commit(&mtr); + return(out); } +# endif /* UNIV_DEBUG */ -/** Note that a tablespace file has been opened. -@param[in] space_id Tablespace ID -@param[in] path Physical path of the file opened -@param[in] state State of the file -@param[in] lsn Commit LSN of the mtr. */ +/** Initialize the table space encryption +@param[in,out] space Tablespace instance */ +static void -Fil_Open::load( - space_id_t space_id, - const std::string& path, - Fil_Open::Nodes::State state, - lsn_t lsn) +fil_tablespace_encryption_init(const fil_space_t* space) { - const auto it = m_spaces.find(space_id); + for (auto& key : *recv_sys->keys) { - if (it != m_spaces.end()) { + if (key.space_id != space->id) { + continue; + } - ut_a(space_id == it->second.m_id); + dberr_t err; - it->second.load(path, state, lsn); + err = fil_set_encryption( + space->id, Encryption::AES, key.ptr, key.iv); - } else { + if (err != DB_SUCCESS) { - using value_type = Spaces::value_type; + ib::error() + << "Can't set encryption information" + << " for tablespace" << space->name + << "!"; + } - Nodes nodes(space_id); + ut_free(key.iv); + ut_free(key.ptr); - nodes.load(path, state, lsn); + key.iv = nullptr; + key.ptr = nullptr; - m_spaces.insert(it, value_type(space_id, nodes)); + key.space_id = std::numeric_limits::max(); } } -/** Note that a tablespace file has been opened. -@param[in] space_id Tablespace ID -@param[in] path Physical path of the file opened -@param[in] lsn LSN of the mtr commit */ -void -Fil_Open::open(space_id_t space_id, const std::string& path, lsn_t lsn) +/** Update the DD if any files were moved to a new location. +Free the Tablespace_files instance. +@param[in] read_only_mode true if InnoDB is started in read only mode. +@return DB_SUCCESS if all OK */ +dberr_t +Fil_system::prepare_open_for_business(bool read_only_mode) { - load(space_id, path, Nodes::OPEN, lsn); + if (read_only_mode && !m_moved.empty()) { - m_needs_flush = true; -} + ib::error() + << m_moved.size() << " files have been relocated" + << " and the server has been started in read" + <<" only mode. Cannot update the data dictionary."; -/** Note that a tablespace file has been closed. -@param[in] space_id Tablespace ID -@param[in] path Physical path of the file closed */ -void -Fil_Open::close(space_id_t space_id, const std::string& path) -{ - ut_ad(!srv_read_only_mode); + return(DB_READ_ONLY); + } - const auto it = m_spaces.find(space_id); + trx_t* trx = check_trx_exists(current_thd); - if (it != m_spaces.end()) { + TrxInInnoDB trx_in_innodb(trx); - ut_a(it->second.m_id == space_id); + /* The transaction should not be active yet, start it */ - it->second.close(path); - } -} + trx->isolation_level = trx_t::READ_UNCOMMITTED; -/** Check if the file can be opened -@param[in] space_id Tablespace ID -@param[in] path Physical path of file to open -@param[in] lsn LSN of the redo log entry -@return true if file can be opened */ -bool -Fil_Open::can_load(space_id_t space_id, const std::string& path, lsn_t lsn) -{ - const auto it = m_spaces.find(space_id); + trx_start_if_not_started_xa(trx, false); - if (it == m_spaces.end()) { - return(true); - } + size_t count = 0; + size_t failed = 0; + size_t batch_size = 0; + bool print_msg = false; + auto start_time = ut_time(); - bool load = false; + /* If some file paths have changed then update the DD */ + for (auto& tablespace : m_moved) { - ut_a(it->second.m_id == space_id); + dberr_t err; - for (auto& file : it->second.m_files) { + auto old_path = std::get(tablespace); - /* If the file is already loaded then don't attempt - to re-load the file. */ + auto space_name = std::get(tablespace); - if (lsn < file.m_open_lsn) { - continue; - } + auto new_path = std::get(tablespace); + auto object_id = std::get(tablespace); - switch(file.m_state) { - case Nodes::INIT: - /* Called while scanning directories. */ - break; + err = dd_rename_tablespace( + object_id, space_name.c_str(), new_path.c_str()); - case Nodes::OPEN: + if (err != DB_SUCCESS) { - if (file.m_name.compare(path) != 0) { - load = true; - file.m_state = Nodes::CLOSED; - } - break; + ib::error() + << "Unable to update tablespace ID" + << " " << object_id << " " + << " '" << old_path << "' to" + << " '" << new_path << "'"; - case Nodes::CLOSED: - case Nodes::DELETED: - case Nodes::MISSING: - case Nodes::OPEN_FAILED: - load = true; + ++failed; } - } - return(load); -} + ++count; -/** Convert to a string */ -std::string -Fil_Open::to_string() const -{ - std::ostringstream str; + if (ut_time() - start_time >= PRINT_INTERVAL_SECS) { - str << std::endl << "open = {" << std::endl; + ib::info() + << "Processed " + << count << "/" << m_moved.size() + << " tablespace paths. Failures " << failed; - for (auto space : m_spaces) { + start_time = ut_time(); + print_msg = true; + } - str << '\t'; - str << "space = {" << std::endl; - str << '\t' << '\t'; - str << "id: " << space.first << "," << std::endl; - str << '\t' << '\t'; - str << "filenames = {" << std::endl; + ++batch_size; - for (auto file : space.second.m_files) { - str << '\t' << '\t' << '\t'; - str << "file = {" << std::endl; - str << '\t' << '\t' << '\t' << '\t'; - str << "name: " << file.m_name.c_str() << std::endl; - str << '\t' << '\t' << '\t' << '\t'; - str << "state: "; + if (batch_size > 10000) { - switch(file.m_state) { - case Nodes::INIT: - str << "INIT"; - break; + innobase_commit_low(trx); - case Nodes::OPEN: - str << "OPEN"; - break; + ib::info() << "Committed : " << batch_size; - case Nodes::MISSING: - str << "MISSING"; - break; + batch_size = 0; - case Nodes::CLOSED: - str << "CLOSED"; - break; + trx_start_if_not_started_xa(trx, false); + } + } - case Nodes::DELETED: - str << "DELETED"; - break; + if (batch_size > 0) { - case Nodes::OPEN_FAILED: - str << "OPEN_FAILED"; - break; - } + ib::info() << "Committed : " << batch_size; + } - str << std::endl; - str << '\t' << '\t' << '\t' << '\t'; - str << "lsn: " << file.m_open_lsn << std::endl; - str << '\t' << '\t' << '\t'; - str << "}" << std::endl; - } + innobase_commit_low(trx); - str << '\t' << '\t'; - str << "}" << std::endl; + if (print_msg) { - str << '\t'; - str << "}" << std::endl; + ib::info() + << "Updated " << count << " tablespace paths" + << ", failures " << failed; } - str << "}" << std::endl; - - return(str.str()); + return(failed == 0 ? DB_SUCCESS : DB_ERROR); } -/** Write the data to the file. -@param[in] version File format version -@param[in] original_len Uncompressed data len -@param[in] compressed_len Compressed data len -@param[in] data The data to write */ -void -Fil_Open::write( - size_t version, - size_t original_len, - size_t compressed_len, - const byte* data) +/** Free the Tablespace_files instance. +@param[in] read_only_mode true if InnoDB is started in read only mode. +@return DB_SUCCESS if all OK */ +dberr_t +fil_open_for_business(bool read_only_mode) { - pfs_os_file_t file = - m_handles.at(m_next % MAX_TABLESPACE_OPEN_FILES).handle; - std::string file_path = - m_handles.at(m_next % MAX_TABLESPACE_OPEN_FILES).path; - ++m_next; - - const char* path = file_path.c_str(); + return(fil_system->prepare_open_for_business(read_only_mode)); +} - ut_ad(file.m_file != OS_FILE_CLOSED); +/** Replay a file rename operation for ddl replay. +@param[in] page_id Space ID and first page number in the file +@param[in] old_name old file name +@param[in] new_name new file name +@return whether the operation was successfully applied (the name did not +exist, or new_name did not exist and name was successfully renamed to +new_name) */ +bool +fil_op_replay_rename_for_ddl( + const page_id_t& page_id, + const char* old_name, + const char* new_name) +{ + space_id_t space_id = page_id.space(); + fil_space_t* space = fil_space_get(space_id); - os_file_truncate(path, file, 0); + if (space == nullptr && !fil_system->open_for_recovery(space_id)) { - char header[sizeof(uint32_t) * 3]; - byte* ptr = reinterpret_cast(header); + ib::info() + << "Can not find space with space ID " + << space_id << " when replaying the DDL log " + << "rename from '" << old_name + << "' to '" << new_name << "'"; - /* Write out the version number. */ - mach_write_to_4(ptr, version); - ptr += sizeof(uint32_t); + return(true); + } - /* Write out the uncompressed length. */ - mach_write_to_4(ptr, original_len); - ptr += sizeof(uint32_t); + return(fil_op_replay_rename(page_id, old_name, new_name)); +} - /* Write out the compressed length. */ - mach_write_to_4(ptr, compressed_len); +/** Lookup the tablespace ID for recovery and DDL log apply. +@param[in] space_id Tablespace ID to lookup +@return true if the space ID is known. */ +bool +Fil_system::lookup_for_recovery(space_id_t space_id) +{ + ut_ad(recv_recovery_is_on() || Log_DDL::is_in_recovery()); + /* Single threaded code, no need to acquire mutex. */ + const auto result = get_scanned_files(space_id); - IORequest request(IORequest::WRITE); + if (recv_recovery_is_on()) { + const auto& end = recv_sys->deleted.end(); + const auto& it = recv_sys->deleted.find(space_id); - os_offset_t offset = 0; - ulint len = 0; - dberr_t err; + if (result.second == nullptr) { - /* Simulate a crash before having written anything - to disk at all. */ - DBUG_EXECUTE_IF( - "ib_tablespace_open_crash_before_write", - DBUG_SUICIDE();); - - /* Simulate a massive corruption of the data written to disk. - ~ temporary or permanent hardware failure. */ - DBUG_EXECUTE_IF( - "ib_tablespace_open_write_corrupt_0", - char buf[]="0123456789AB"; - len = strlen(buf); - err = os_file_write(request, path, file, buf, offset, len); - offset += len; - - char buf1[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; - len = strlen(buf1); - err = os_file_write(request, path, file, buf1, offset, len); - offset += len; - - os_file_flush(file); - DBUG_SUICIDE();); - - /* Simulate that a powerloss during write to disk causes - that the header is incomplete and what remains is lost - anyway. */ - DBUG_EXECUTE_IF( - "ib_tablespace_open_write_corrupt_1", - len = sizeof(header) - 1; - err = os_file_write(request, path, file, header, offset, len); - offset += len; - - os_file_flush(file); - DBUG_SUICIDE();); - - len = sizeof(header); - err = os_file_write(request, path, file, header, offset, len); - if (err != DB_SUCCESS) { - /* print msg to stderr */ - os_file_get_last_error(true); + /* If it wasn't deleted after finding it on disk then + we tag it as missing. */ - ib::fatal() << "write() failed to write header : " << path; - } else { - offset += len; - } + if (it == end) { - /* Simulate that a powerloss during write to disk causes - that the compressed data is incomplete. */ - DBUG_EXECUTE_IF( - "ib_tablespace_open_write_corrupt_2", - len = compressed_len - 1; - err = os_file_write(request, path, file, data, offset , len); - os_file_flush(file); - DBUG_SUICIDE();); + recv_sys->missing_ids.insert(space_id); + } - /* Write out the compressed data. */ - len = compressed_len; - err = os_file_write(request, path, file, data, offset, len); + return(false); + } - if (err != DB_SUCCESS) { - /* print msg to stderr */ - os_file_get_last_error(true); + /* Check that it wasn't deleted. */ - ib::fatal() << "write() failed to write header : " << path; - } else { - offset += len; + return(it == end); } - /* Crash before writing out the file. */ - DBUG_EXECUTE_IF( - "ib_tablespace_open_flush_crash", - DBUG_SUICIDE();); - - /** Flush the OS buffer to disk. */ - bool success = os_file_flush(file); - - if (!success) { + return(result.second != nullptr); +} - ib::fatal() - << "os_file_flush() failed: '" << path << "' : " - << strerror(errno); - } +/** Lookup the tablespace ID. +@param[in] space_id Tablespace ID to lookup +@return true if the space ID is known. */ +bool +fil_tablespace_lookup_for_recovery(space_id_t space_id) +{ + return(fil_system->lookup_for_recovery(space_id)); } -/** Absolute path of file. -@param[in] dir Parent directory (can be '.') -@param[in] filename Filename to read/write -@return absolute path name */ -std::string -Fil_Open::get_path(const char* dir, const std::string& filename) +/** Open a tablespace that has a redo/DDL log record to apply. +@param[in] space_id Tablespace ID +@return true if the open was successful */ +bool +Fil_system::open_for_recovery(space_id_t space_id) { - char abspath[FN_REFLEN + 2]; + ut_ad(recv_recovery_is_on() || Log_DDL::is_in_recovery()); - memset(abspath, 0x0, sizeof(abspath)); + if (!lookup_for_recovery(space_id)) { + return(false); + } - my_realpath(abspath, dir, MYF(0)); + const auto result = get_scanned_files(space_id); - size_t len = strlen(abspath); + /* Duplicates should have been sorted out before start of recovery. */ + ut_a(result.second->size() == 1); - ut_a(len < sizeof(abspath) - 10 - filename.length()); + const auto& filename = result.second->front(); + const std::string path = result.first + filename; - if (abspath[len - 1] != OS_PATH_SEPARATOR) { + fil_space_t* space; - abspath[len] = OS_PATH_SEPARATOR; - ++len; - } + auto status = ibd_open_for_recovery(space_id, path, space); - strncat(abspath, filename.c_str(), filename.length()); + if (status == FIL_LOAD_OK) { - return(std::string(abspath)); -} + /* For encrypted tablespace, set key and iv. */ + if (FSP_FLAGS_GET_ENCRYPTION(space->flags) + && recv_sys->keys!= nullptr) { -/** Write the state to disk. */ -void -Fil_Open::to_file() -{ - ut_ad(!srv_read_only_mode); - ut_ad(mutex_own(&m_mutex)); + fil_tablespace_encryption_init(space); + } - if (!m_needs_flush) { - return; + if (!recv_sys->dblwr.deferred.empty()) { + buf_dblwr_recover_pages(space); + } + + return(true); } - std::ostringstream os; + return(false); +} - char n_tablespaces[sizeof(uint32_t)]; - byte* ptr = reinterpret_cast(n_tablespaces); +/** Open a tablespace that has a redo log record to apply. +@param[in] space_id Tablespace ID +@return true if the open was successful */ +bool +fil_tablespace_open_for_recovery(space_id_t space_id) +{ + return(fil_system->open_for_recovery(space_id)); +} - /* Write the number of entries. */ - mach_write_to_4(ptr, m_spaces.size()); +/** Lookup the tablespace ID and return the path to the file. The filename +is ignored when testing for equality. Only the path up to the file name is +considered for matching: e.g. ./test/a.ibd == ./test/b.ibd. +@param[in] dd_object_id Server DD tablespace ID +@param[in] space_id Tablespace ID to lookup +@param[in] space_name Tablespace name +@param[in] old_path Path in the data dictionary +@param[out] new_path New path if scanned path not equal to path +@return status of the match. */ +Fil_state +fil_tablespace_path_equals( + dd::Object_id dd_object_id, + space_id_t space_id, + const char* space_name, + std::string old_path, + std::string* new_path) +{ + ut_ad(Fil_path::has_ibd_suffix(old_path)); - os.write(n_tablespaces, sizeof(n_tablespaces)); + /* Single threaded code, no need to acquire mutex. */ + const auto& end = recv_sys->deleted.end(); + const auto result = fil_system->get_scanned_files(space_id); + const auto& it = recv_sys->deleted.find(space_id); - lsn_t max_lsn = 0; + if (result.second == nullptr) { - for (auto space : m_spaces) { + /* If the DD has the path but --innodb-directories doesn't, + we need to check if the DD path is valid before we tag the + file as missing. */ - char space_id[sizeof(space_id_t)]; + if (Fil_path::get_file_type(old_path) == OS_FILE_TYPE_FILE) { - ut_ad(sizeof(space_id_t) == sizeof(uint32_t)); + ib::info() + << old_path << " found outside of" + << " --innodb-directories setting"; - ptr = reinterpret_cast(space_id); + return(Fil_state::MATCHES); + } - /* Write the tablespace ID. */ - mach_write_to_4(ptr, space.first); + /* If it wasn't deleted during redo apply, we tag it + as missing. */ - os.write(space_id, sizeof(space_id)); + if (it == end && recv_recovery_is_on()) { - char len[sizeof(uint32_t)]; + recv_sys->missing_ids.insert(space_id); + } - ptr = reinterpret_cast(len); + return(Fil_state::MISSING); + } - /* Write the number of files. */ - mach_write_to_4(ptr, space.second.m_files.size()); + /* Check that it wasn't deleted. */ + if (it != end) { + return(Fil_state::DELETED); + } - os.write(len, sizeof(len)); + /* A file with this space_id was found during scanning. + Validate its location and see if it was moved. - for (auto path : space.second.m_files) { + Don't compare the full filename, there can be a mismatch if + there was a DDL in progress and we will end up renaming the path + in the DD dictionary. Such renames should be handled by the + atomic DDL "ddl_log". */ - /* Write the */ - char name_len[sizeof(uint16_t)]; + std::string old_dir{old_path}; - ptr = reinterpret_cast(name_len); + /* Ignore the filename component of the old path. */ + auto pos = old_dir.find_last_of(Fil_path::SEPARATOR); + if (pos == std::string::npos) { + old_dir = MySQL_datadir_path; + } else { + old_dir.resize(pos + 1); + ut_ad(Fil_path::is_separator(old_dir.back())); + } + old_dir = Fil_path::get_real_path(old_dir); - /* Write the length of the string in bytes. */ - mach_write_to_2(ptr, path.m_name.length()); + /* Build the new path from the scan path and the found path. */ + std::string new_dir{result.first}; - os.write(name_len, sizeof(name_len)); + ut_ad(Fil_path::is_separator(new_dir.back())); - /* Write the name of the file. */ - os.write(path.m_name.c_str(), path.m_name.length()); + new_dir.append(result.second->front()); - /* Write the state of the file. */ - char state[sizeof(uint8_t)]; + new_dir = Fil_path::get_real_path(new_dir); - ptr = reinterpret_cast(state); + /* Do not use a datafile that is in the wrong place. */ + if (!Fil_path::is_valid_location(space_name, new_dir)) { - ut_ad(sizeof(state) == sizeof(uint8_t)); + ib::info() + << "Cannot use scanned file " << new_dir + << " for tablespace " << space_name + << " because it is not in a valid location."; - mach_write_to_1(ptr, path.m_state); + return(Fil_state::MISSING); + } - os.write(state, sizeof(state)); + /* Ignore the filename component of the new path. */ + pos = new_dir.find_last_of(Fil_path::SEPARATOR); - /* Write the latest OPEN/RENAME LSN. */ - char lsn[sizeof(lsn_t)]; + ut_ad(pos != std::string::npos); - ptr = reinterpret_cast(lsn); + new_dir.resize(pos + 1); - ut_ad(sizeof(lsn_t) == sizeof(uint64_t)); + if (old_dir.compare(new_dir) != 0) { - mach_write_to_8(ptr, path.m_open_lsn); + *new_path = result.first + result.second->front(); - os.write(lsn, sizeof(lsn)); + fil_system->moved( + dd_object_id, space_id, + space_name, old_path, *new_path); - if (path.m_open_lsn > max_lsn) { - max_lsn = path.m_open_lsn; - } - } + return(Fil_state::MOVED); } - const std::string& data = os.str(); - - /* See compress2() man page for the magic number 9. */ - uLongf zlen = compressBound(static_cast(data.length())); - auto dst = static_cast(ut_malloc_nokey(zlen)); - auto src = reinterpret_cast(data.c_str()); + *new_path = old_path; - switch(compress2(dst, &zlen, src, - static_cast(data.length()), 6)) { - case Z_BUF_ERROR: - ib::fatal() << "Compression failed, Z_BUF_ERROR"; - break; - - case Z_MEM_ERROR: - ib::fatal() << "Compression failed, Z_MEM_ERROR"; - break; + return(Fil_state::MATCHES); +} - case Z_STREAM_ERROR: - ib::fatal() << "Compression failed, Z_STREAM_ERROR"; - break; +#endif /* !UNIV_HOTBACKUP */ - case Z_OK: - break; +/** This function should be called after recovery has completed. +Check for tablespace files for which we did not see any MLOG_FILE_DELETE +or MLOG_FILE_RENAME record. These could not be recovered. +@return true if there were some filenames missing for which we had to + ignore redo log records during the apply phase */ +bool +Fil_system::check_missing_tablespaces() +{ + bool missing = false; + auto& dblwr = recv_sys->dblwr; + const auto end = recv_sys->deleted.end(); - default: - ib::fatal() << "Compression failed, UNKNOWN_ERROR"; - break; - } + /* Called in single threaded mode, no need to acquire the mutex. */ - write(VERSION_1, data.length(), zlen, dst); + /* First check if we were able to restore all the doublewrite + buffer pages. If not then print a warning. */ - ut_free(dst); + for (auto& page : dblwr.deferred) { - m_needs_flush = false; -} + space_id_t space_id; -/** Parse the input stream and populate the data structure. -@param[in,out] is Input stream to read -@param[out] max_lsn Maximum LSN read from the file -@param[in] recovery if called from recovery -@return true on success */ -bool -Fil_Open::parse(std::istream& is, lsn_t& max_lsn, bool recovery) -{ - max_lsn = 0; + space_id = page_get_space_id(page.m_page); - ut_ad(!m_needs_flush); - ut_ad(m_spaces.empty()); + /* If the tablespace was in the missing IDs then we + know that the problem is elsewhere. If a file deleted + record was not found in the redo log and the tablespace + doesn't exist in the SYS_TABLESPACES file then it is + an error or data corruption. The special case is an + undo truncate in progress. */ - /* Read the number of entries. */ + if (recv_sys->deleted.find(space_id) == end + && recv_sys->missing_ids.find(space_id) + != recv_sys->missing_ids.end()) { - char buf[sizeof(lsn_t)]; - byte* ptr = reinterpret_cast(buf); + page_no_t page_no; - ut_ad(sizeof(lsn_t) == sizeof(uint64_t)); - ut_ad(sizeof(space_id_t) == sizeof(uint32_t)); + page_no = page_get_page_no(page.m_page); - /* Read the total number of tablespace entries. */ - is.read(buf, sizeof(uint32_t)); + ib::warn() + << "Doublewrite page " << page.m_no + << " for {space: " << space_id << ", page_no:" + << page_no << "} could not be restored." + << " File name unknown for tablespace ID " + << space_id; + } - if (!is.good()) { - return(false); + /* Free the memory. */ + page.close(); } - uint32_t n_tablespaces = mach_read_from_4(ptr); - - for (size_t i = 0; i < n_tablespaces; ++i) { + dblwr.deferred.clear(); - /* Read the tablespace ID */ - is.read(buf, sizeof(uint32_t)); + for (auto space_id : recv_sys->missing_ids) { - if (!is.good()) { + if (recv_sys->deleted.find(space_id) != end) { - return(false); + continue; } - space_id_t space_id = mach_read_from_4(ptr); + const auto result = get_scanned_files(space_id); - /* Read the total number of files open for this tablespace. */ - is.read(buf, sizeof(uint32_t)); + if (result.second == nullptr) { - if (!is.good()) { + ib::error() + << "Could not find any file associated with" + << " the tablespace ID: " << space_id; - return(false); - } + if (!fsp_is_undo_tablespace(space_id)) { + + missing = true; + } else { + /* Could be an undo truncate in progress. */ + } - Nodes nodes(space_id); - uint32_t n_files = mach_read_from_4(ptr); + } else { + ut_a(!result.second->empty()); + } + } - for (size_t j = 0; j < n_files; ++j) { + return(missing); +} - /* Read the length of the filename in bytes. */ - is.read(buf, sizeof(uint16_t)); +/** This function should be called after recovery has completed. +Check for tablespace files for which we did not see any MLOG_FILE_DELETE +or MLOG_FILE_RENAME record. These could not be recovered +@return true if there were some filenames missing for which we had to + ignore redo log records during the apply phase */ +bool +fil_check_missing_tablespaces() +{ + return(fil_system->check_missing_tablespaces()); +} - if (!is.good()) { +/** Redo a tablespace create. +@param[in] ptr redo log record +@param[in] end end of the redo log buffer +@param[in] page_id Tablespace Id and first page in file +@param[in] parsed_bytes Number of bytes parsed so far +@return pointer to next redo log record +@retval nullptr if this log record was truncated */ +byte* +fil_tablespace_redo_create( + byte* ptr, + const byte* end, + const page_id_t& page_id, + ulint parsed_bytes) +{ + ut_a(page_id.page_no() == 0); - return(false); - } + /* We never recreate the system tablespace. */ + ut_a(page_id.space() != TRX_SYS_SPACE); - uint16_t len = mach_read_from_2(ptr); + ut_a(parsed_bytes != ULINT_UNDEFINED); - if (len == 0 || len >= OS_FILE_MAX_PATH) { + /* Where 6 = flags (uint32_t) + name len (uint16_t). */ + if (end <= ptr + 6) { + return(nullptr); + } - return(false); - } +#ifdef UNIV_HOTBACKUP + ulint flags = mach_read_from_4(ptr); +#else + /* Skip the flags, not used here. */ +#endif /* UNIV_HOTBACKUP */ - /* Read the filename */ - char name[OS_FILE_MAX_PATH + 1]; + ptr += 4; - name[len] = 0; + ulint len = mach_read_from_2(ptr); - /* Read the filename */ - is.read(name, len); + ptr += 2; - if (!is.good()) { + /* Do we have the full/valid file name. */ + if (end < ptr + len || len < 5) { - return(false); - } + if (len < 5) { - /* Read the state of the file */ - is.read(buf, sizeof(uint8_t)); + char name[6]; - if (!is.good()) { + snprintf(name, sizeof(name), "%.*s", (int) len, ptr); - return(false); - } + ib::error() + << "MLOG_FILE_CREATE : Invalid file name." + << " Length (" << len << ") must be >= 5" + << " and end in '.ibd'. File name in the" + << " redo log is '" << name << "'"; - Nodes::State state; + recv_sys->found_corrupt_log = true; + } - state = static_cast( - mach_read_from_1(ptr)); + return(nullptr); + } - /* Read the OPEN/RENAME redo LSN */ - is.read(buf, sizeof(lsn_t)); + char* name = reinterpret_cast(ptr); - if (!is.good()) { + Fil_path::normalize(name); - return(false); - } + ptr += len; - lsn_t lsn = mach_read_from_8(ptr); + if (!Fil_path::has_ibd_suffix(name)) { - if (lsn > max_lsn) { - max_lsn = lsn; - } + recv_sys->found_corrupt_log = true; - /* We always load the UNDO tablespace locations, even - if we are not in recovery mode. This is only done to - make some legacy tests pass WL#7806 related. See - fil_space_undo_check_if_opened() name check. */ - if (!recovery && !is_undo_tablespace_name(name, len)) { - continue; - } + return(nullptr); + } - nodes.load(name, state, lsn); - } +#ifdef UNIV_HOTBACKUP - const auto it = m_spaces.find(space_id); + meb_tablespace_redo_create(page_id, flags, name); - if (it != m_spaces.end()) { +#else /* !UNIV_HOTBACKUP */ - ib::error() - << "Duplicate entry for tablespace ID: " - << space_id; + const auto result = fil_system->get_scanned_files(page_id.space()); - return(false); - } + if (result.second == nullptr) { - using value_type = Spaces::value_type; + /* No file maps to this tablespace ID. It's possible that + the file was deleted later or is misisng. */ - m_spaces.insert(it, value_type(space_id, nodes)); + return(ptr); } - return(true); -} - -#ifndef UNIV_HOTBACKUP -/** Read tablespaces from data directory */ -void -Fil_Open::from_current_dir() -{ - /* Scan data directory to find all tablespaces */ - fil_scan_for_tablespaces("."); - - for (const auto& space : fil_scanned->m_spaces) { + auto abs_name = Fil_path::get_real_path(name); - if (space.first == TRX_SYS_SPACE){ + /* Duplicates should have been sorted out before we get here. */ + ut_a(result.second->size() == 1); - continue; - } + /* It's possible that the tablespace file was renamed later. */ + if (result.second->front().compare(abs_name) == 0) { + bool success; - for (const auto& file : space.second.m_files) { + success = fil_tablespace_open_for_recovery(page_id.space()); - load(space.first, file.m_name, - file.m_state, file.m_open_lsn); + if (!success) { + ib::info() << "Create '" << abs_name << "' failed!"; } } +#endif /* UNIV_HOTBACKUP */ - UT_DELETE(fil_scanned); - fil_scanned = nullptr; + return(ptr); } -#endif /* !UNIV_HOTBACKUP */ -/** Read the state from disk -@param[in] recovery true if called from crash recovery */ -void -Fil_Open::from_file(bool recovery) +/** Redo a tablespace rename. +This function doesn't do anything, simply parses the redo log record. +@param[in] ptr redo log record +@param[in] end end of the redo log buffer +@param[in] page_id Tablespace Id and first page in file +@param[in] parsed_bytes Number of bytes parsed so far +@return pointer to next redo log record +@retval nullptr if this log record was truncated */ +byte* +fil_tablespace_redo_rename( + byte* ptr, + const byte* end, + const page_id_t& page_id, + ulint parsed_bytes) { - ut_a(!srv_read_only_mode); + ut_a(page_id.page_no() == 0); - using Files = std::vector; + /* We never recreate the system tablespace. */ + ut_a(page_id.space() != TRX_SYS_SPACE); - Files files; - std::vector opened; - std::vector max_lsn; - - size_t n_errors = 0; + ut_a(parsed_bytes != ULINT_UNDEFINED); - for (const auto& filename : PATHS) { + /* Where 2 = from name len (uint16_t). */ + if (end <= ptr + 2) { + return(nullptr); + } - std::ifstream ifs; + /* Read and check the RENAME FROM_NAME. */ + ulint from_len = mach_read_from_2(ptr); - std::string abspath = get_path( - srv_log_group_home_dir, filename); + ptr += 2; - ifs.open(abspath.c_str(), std::ios::in | std::ios::binary); + /* Do we have the full/valid from and to file names. */ + if (end < ptr + from_len || from_len < 5) { - if (!ifs) { + if (from_len < 5) { - ++n_errors; + char name[6]; - ib::info() - << "Unable to read from '" - << abspath << "', the space ID to filename" - << " mapping file"; + snprintf(name, sizeof(name), "%.*s", + (int) from_len, ptr); - continue; + ib::error() + << "MLOG_FILE_RENAME: Invalid from file name." + << " Length (" << from_len << ") must be >= 5" + << " and end in '.ibd'. File name in the" + << " redo log is '" << name << "'"; } - /* Get the file size. */ - ifs.seekg (0, ifs.end); - - std::streampos n_bytes = ifs.tellg(); - - ifs.seekg (0, ifs.beg); - - if ((size_t) n_bytes < sizeof(uint32_t) * 3) { + return(nullptr); + } - ++n_errors; + char* from_name = reinterpret_cast(ptr); - continue; - } + Fil_path::normalize(from_name); - /* Read in the header (v1). */ - char header[sizeof(uint32_t) * 3]; - auto ptr = reinterpret_cast(header); + auto abs_from_name = Fil_path::get_real_path(from_name); - ifs.read(header, sizeof(header)); - std::streamsize n_bytes_read = ifs.gcount(); + ptr += from_len; - /* Check if the data read in matches our header. */ - if (!ifs.good() || (byte) n_bytes_read < sizeof(header)) { + if (!Fil_path::has_ibd_suffix(abs_from_name)) { - ib::fatal() - << "Failed to read " << sizeof(header) - << " bytes from '" << abspath << "'"; - } + ib::error() + << "MLOG_FILE_RENAME: From file name doesn't end in" + << " .ibd. File name in the redo log is '" + << from_name << "'"; - /* Deserialize the header. */ - size_t version = mach_read_from_4(ptr); - ptr += sizeof(uint32_t); + recv_sys->found_corrupt_log = true; - if (version != VERSION_1) { + return(nullptr); + } - ib::error() - << "Unsupported file format " << version - << " found in tablespace ID to filename" - << " mapping file: '" << abspath << "'." - << " You can use --innodb-scan-directories" - << " to recover if the tablespaces.open.*" - << " files are unreadable or corrupt."; + /* Read and check the RENAME TO_NAME. */ - ib::fatal() - << "Unable to read the space ID to filename" - << " mapping file(s)."; - } + ulint to_len = mach_read_from_2(ptr); - /* Read the original length of the data. */ - uLongf zlen = mach_read_from_4(ptr); - ptr += sizeof(uint32_t); + ptr += 2; - /* Read the compressed length from the header. */ - uLongf compressed_len = mach_read_from_4(ptr); - ptr += sizeof(uint32_t); + if (end < ptr + to_len || to_len < 5) { - /* Check that the compressed data was written out fully. */ - if ((size_t) n_bytes < sizeof(header) + compressed_len) { + if (to_len < 5) { - ++n_errors; + char name[6]; - ib::warn() - << "File '" << abspath << "' size is " - << n_bytes << " bytes, should be at least " - << sizeof(header) + compressed_len << " bytes"; + snprintf(name, sizeof(name), "%.*s", + (int) to_len, ptr); - continue; + ib::error() + << "MLOG_FILE_RENAME: Invalid to file name." + << " Length (" << to_len << ") must be >= 5" + << " and end in '.ibd'. File name in the" + << " redo log is '" << name << "'"; } - auto src = static_cast( - ut_malloc_nokey(compressed_len)); + return(nullptr); + } - ptr = reinterpret_cast(src); + char* to_name = reinterpret_cast(ptr); - /* Read the compressed data in one chunk. */ - ifs.read(src, compressed_len); + ptr += to_len; - /* We have checked the file size above, must succeed. */ - ut_a(ifs.good() && (size_t) ifs.gcount() == compressed_len); + Fil_path::normalize(to_name); - ifs.close(); +#ifdef UNIV_HOTBACKUP - uLongf original_len = zlen; - auto dst = static_cast(ut_malloc_nokey(zlen)); - auto dst_ptr = reinterpret_cast(dst); + meb_tablespace_redo_rename(page_id, from_name, to_name); - int ret; +#else /* !UNIV_HOTBACKUP */ - ret = uncompress(dst_ptr, &zlen, ptr, compressed_len); + auto abs_to_name = Fil_path::get_real_path(to_name); - if (ret != Z_OK) { + if (from_len == to_len && strncmp(to_name, from_name, to_len) == 0) { - ut_free(src); - ut_free(dst); + ib::error() + << "MLOG_FILE_RENAME: The from and to name are the" + << " same: '" << from_name << "', '" << to_name << "'"; - ib::error() - << "ZLIB uncompress() failed:" - << " '" << abspath << "'" - << " compressed len: " << compressed_len - << ", original_len: " << original_len; + recv_sys->found_corrupt_log = true; - ++n_errors; + return(nullptr); + } - switch(ret) { - case Z_BUF_ERROR: + if (!Fil_path::has_ibd_suffix(abs_to_name)) { - ib::error() << "retval = Z_BUF_ERROR"; - continue; + ib::error() + << "MLOG_FILE_RENAME: To file name doesn't end in" + << " .ibd. File name in the redo log is '" + << to_name << "'"; - case Z_MEM_ERROR: + recv_sys->found_corrupt_log = true; - ib::error() << "retval = Z_MEM_ERROR"; - continue; + return(nullptr); + } +#endif /* UNIV_HOTBACKUP */ - case Z_DATA_ERROR: + return(ptr); +} - ib::error() << "retval = Z_DATA_ERROR"; - continue; +/** Redo a tablespace delete. +@param[in] ptr redo log record +@param[in] end end of the redo log buffer +@param[in] page_id Tablespace Id and first page in file +@param[in] parsed_bytes Number of bytes parsed so far +@return pointer to next redo log record +@retval nullptr if this log record was truncated */ +byte* +fil_tablespace_redo_delete( + byte* ptr, + const byte* end, + const page_id_t& page_id, + ulint parsed_bytes) +{ + ut_a(page_id.page_no() == 0); - default: - ut_error; - } - } + /* We never recreate the system tablespace. */ + ut_a(page_id.space() != TRX_SYS_SPACE); + + ut_a(parsed_bytes != ULINT_UNDEFINED); - std::istringstream iss(std::string(dst, zlen)); + /* Where 2 = len (uint16_t). */ + if (end <= ptr + 2) { + return(nullptr); + } - ut_free(src); + ulint len = mach_read_from_2(ptr); - Fil_Open file; + ptr += 2; - max_lsn.push_back(0); + /* Do we have the full/valid file name. */ + if (end < ptr + len || len < 5) { - if (!file.parse(iss, max_lsn.back(), recovery)) { + if (len < 5) { - ib::error() << "Failed to parse : '" << abspath << "'"; + char name[6]; - ++n_errors; + snprintf(name, sizeof(name), "%.*s", (int) len, ptr); ib::error() - << "Tablespace open state file '" - << abspath << "' cannot be deserialized."; - - ib::warn() - << "A tablespace ID to filename mapping file" - << " is unreadable. It is possible that the" - << " readable copy doesn't contain all the" - << " tablespace ID to filename mappings." - << " This can potentially corrupt your data" - << " permanently during recovery."; + << "MLOG_FILE_DELETE : Invalid file name." + << " Length (" << len << ") must be >= 5" + << " and end in '.ibd'. File name in the" + << " redo log is '" << name << "'"; + } - if (n_errors == 2) { + return(nullptr); + } - ib::fatal() - << "Cannot continue, require at" - << " least one tablespace ID to" - << " file name mapping file."; - } + char* name = reinterpret_cast(ptr); - if (srv_force_recovery == 0) { + Fil_path::normalize(name); - ib::fatal() - << "Use --innodb-scan-directories" - << " to try and recover by scanning" - << " for .ibd files and creating a" - << " space ID to filename map by" - << " examining the tablespace header."; + ptr += len; - } else { - ib::info() - << "--innodb-force-recovery is set" - << " ignoring this error."; - } - } + if (!Fil_path::has_ibd_suffix(name)) { - ut_free(dst); + recv_sys->found_corrupt_log = true; - files.push_back(file); - opened.push_back(filename); + return(nullptr); } - if (files.empty()) { - - ib::error() << "No space ID to filename mapping file found"; +#ifdef UNIV_HOTBACKUP - return; - } + meb_tablespace_redo_delete(page_id, name); - size_t i; +#else /* !UNIV_HOTBACKUP */ - { - auto it = std::max_element(max_lsn.begin(), max_lsn.end()); - auto max = it != max_lsn.end() ? *it : 0; + const auto result = fil_system->get_scanned_files(page_id.space()); - if (max == 0) { + recv_sys->deleted.insert(page_id.space()); + recv_sys->missing_ids.erase(page_id.space()); - ib::warn() << "All the mapping files are empty!"; + if (result.second == nullptr) { - return; - } + /* No files map to this tablespace ID. The drop must + have succeeded. */ - i = std::distance(max_lsn.begin(), it); + return(ptr); } - /* File with the higher LSN is the latest. */ - - ib::info() << "Using '" << opened[i] << "' max LSN: " << max_lsn[i]; - - for (const auto& space : files[i].m_spaces) { - if (space.first == TRX_SYS_SPACE - || space.first == dict_sys_t::s_temp_space_id){ + /* Space_id_set should have been sorted out before we get here. */ - continue; - } + ut_a(result.second->size() == 1); - /* Setup the recovery state. These are the space ID to - filename mappings required to recover from the redo log. */ - for (const auto& file : space.second.m_files) { - - load(space.first, file.m_name, - file.m_state, file.m_open_lsn); - } - } + auto abs_name = Fil_path::get_real_path(name); - m_needs_flush = false; + ut_ad(!Fil_path::is_separator(abs_name.back())); - { - auto it = std::find( PATHS.begin(), PATHS.end(), opened[i]); + fil_space_free(page_id.space(), false); - /* Must find the filename. */ - ut_a(it != PATHS.end()); + bool success = fil_system->erase(page_id.space()); + ut_a(success); +#endif /* UNIV_HOTBACKUP */ - /* Ensure that we don't write to the file we just read in - when syncing next. */ - m_next = std::distance(PATHS.begin(), it) + 1; - } + return(ptr); } -#ifndef UNIV_HOTBACKUP -/** Initialize the table space encryption -@param[in,out] space Tablespace instance */ -static -void -fil_tablespace_encryption_init(fil_space_t* space) +/** Parse and process an encryption redo record. +@param[in] ptr redo log record +@param[in] end end of the redo log buffer +@param[in] space_id the tablespace ID +@return log record end, nullptr if not a complete record */ +byte* +fil_tablespace_redo_encryption( + byte* ptr, + const byte* end, + space_id_t space_id) { - for (auto& key : *recv_sys->keys) { + byte* iv = nullptr; + byte* key = nullptr; + bool is_new = false; - if (key.space_id == space->id) { + fil_space_t* space = fil_space_get(space_id); - dberr_t err; + if (space == nullptr) { - err = fil_set_encryption( - space->id, Encryption::AES, key.ptr, key.iv); + if (recv_sys->keys == nullptr) { - if (err != DB_SUCCESS) { + recv_sys->keys = UT_NEW_NOKEY( + recv_sys_t::Encryption_Keys()); + } - ib::error() - << "Can't set encryption information" - << " for tablespace" << space->name - << "!"; - } + for (auto& recv_key : *recv_sys->keys) { - ut_free(key.iv); - ut_free(key.ptr); + if (recv_key.space_id == space_id) { + iv = recv_key.iv; + key = recv_key.ptr; + } + } - key.iv = nullptr; - key.ptr = nullptr; + if (key == nullptr) { - key.space_id = std::numeric_limits::max(); - } - } -} + key = static_cast( + ut_malloc_nokey(ENCRYPTION_KEY_LEN)); -/** Register a tablespace that was open when the server crashed -Check if the tablespace file exists and contains the space_id. If not, -ignore the file after displaying a note. Abort if there are multiple -files with the same space_id. -@param[in] space_id Space id -@param[in] path Physical path to the filename -@param[in] lsn Commit LSN of redo log record -@return true on success */ -static -bool -fil_tablespace_open_for_recovery( - space_id_t space_id, - const std::string& path, - lsn_t lsn) -{ - fil_space_t* space; - fil_load_status status; + iv = static_cast( + ut_malloc_nokey(ENCRYPTION_KEY_LEN)); - ut_ad(recv_recovery_is_on() || Log_DDL::is_in_recovery()); + is_new = true; + } - status = fil_ibd_open_for_recovery(space_id, path.c_str(), space); + } else { + iv = space->encryption_iv; + key = space->encryption_key; + } - switch(status) { - case FIL_LOAD_OK: + ulint offset; - /* For encrypted tablespace, set key and iv. */ - if (FSP_FLAGS_GET_ENCRYPTION(space->flags) - && recv_sys->keys!= nullptr) { + offset = mach_read_from_2(ptr); + ptr += 2; - fil_tablespace_encryption_init(space); - } + ulint len; - /* Single threaded mode, no need to lock anything. */ - fil_system->m_open.open(space_id, path, lsn); + len = mach_read_from_2(ptr); + ptr += 2; - if (!recv_sys->dblwr.deferred.empty()) { - buf_dblwr_recover_pages(space); - } + if (end < ptr + len) { + return(nullptr); + } - break; + if (offset >= UNIV_PAGE_SIZE + || len + offset > UNIV_PAGE_SIZE + || (len != ENCRYPTION_INFO_SIZE_V1 + && len != ENCRYPTION_INFO_SIZE_V2)) { - case FIL_LOAD_ID_CHANGED: - /* File was renamed, or misconfiguration. */ - ut_ad(space == NULL); + recv_sys->found_corrupt_log = true; + return(nullptr); + } - if (space_id != TRX_SYS_SPACE) { +#ifdef UNIV_HOTBACKUP + if (fil_space_get(space_id) + && !meb_get_encryption_key(space_id, key, iv)) { - /* Single threaded mode, no need to lock anything. */ - fil_system->m_open.close(space_id, path); + recv_sys->found_corrupt_log = true; - } else if (srv_force_recovery == 0) { + ib::fatal() + << "Encryption informaton" + << " in the redo log of space " + << space_id << " is invalid" + << " MEB cannot proceed with the operation."; + } +#else /* UNIV_HOTBACKUP */ + if (!Encryption::decode_encryption_info(key, iv, ptr)) { - /* Can't operate without the system tablespace. */ + recv_sys->found_corrupt_log = true; - ib::error() - << "Redo log refers to a system tablespace" - << " file '" << path << "', which disagrees" - << " with innodb_data_file_path or the" - << " directory settings. Check the startup" - << " parameters or ignore this error by setting" - << " --innodb-force-recovery."; + ib::warn() + << "Encryption information" + << " in the redo log of space " + << space_id << " is invalid"; - recv_sys->found_corrupt_log = true; - } + return(nullptr); + } +#endif /* UNIV_HOTBACKUP */ - break; + ut_ad(len == ENCRYPTION_INFO_SIZE_V1 + || len == ENCRYPTION_INFO_SIZE_V2); - case FIL_LOAD_NOT_FOUND: + ptr += len; - /* No matching tablespace was found; maybe it - was renamed, and we will find a subsequent - MLOG_FILE_* record. */ + if (space == nullptr) { - ut_ad(space == NULL); + if (is_new) { - if (srv_force_recovery > 0) { + recv_sys_t::Encryption_Key new_key; - /* Without innodb_force_recovery, missing tablespaces - will only be reported in after applying all the redo - log records. Enable some more diagnostics when forcing - recovery. */ + new_key.iv = iv; + new_key.ptr = key; + new_key.space_id = space_id; - ib::info() - << "At LSN: " << recv_sys->recovered_lsn - << ": unable to open file " << path - << " for tablespace " << space_id; + recv_sys->keys->push_back(new_key); } - /* Single threaded mode, no need to lock anything. */ - fil_system->m_open.load( - space_id, path, Fil_Open::Nodes::MISSING, lsn); - - break; + } else { + ut_ad(FSP_FLAGS_GET_ENCRYPTION(space->flags)); - case FIL_LOAD_MISMATCH: + space->encryption_type = Encryption::AES; + space->encryption_klen = ENCRYPTION_KEY_LEN; + } - return(false); + return(ptr); +} - case FIL_LOAD_INVALID: - ut_ad(space == NULL); +/** Tokenize a path specification. Convert relative paths to absolute paths. +Check if the paths are valid and filter out invalid or unreadable directories. +Sort and filter out duplicates from dirs. +@param[in] str Path specification to tokenize +@param[in] delimiters Delimiters */ +void +Tablespace_dirs::tokenize_paths( + const std::string& str, + const std::string& delimiters) +{ + std::string::size_type start = str.find_first_not_of(delimiters); + std::string::size_type end = str.find_first_of(delimiters, start); - if (srv_force_recovery == 0) { + using Paths = std::vector>; - ib::warn() - << "We do not continue the crash" - " recovery, because the table may" - " become corrupt if we cannot apply" - " the log records in the InnoDB log to" - " it. To fix the problem and start" - " mysqld:"; + Paths dirs; - ib::info() - << "1) If there is a permission" - " problem in the file and mysqld" - " cannot open the file, you should" - " modify the permissions."; + /* Scan until 'end' and 'start' don't reach the end of string (npos) */ + while (std::string::npos != start || std::string::npos != end) { - ib::info() - << "2) If the tablespace is not" - " needed, or you can restore an older" - " version from a backup, then you can" - " remove the .ibd file, and use" - " --innodb_force_recovery=1 to force" - " startup without this file."; + std::array dir; - ib::info() - << "3) If the file system or the" - " disk is broken, and you cannot" - " remove the .ibd file, you can set" - " --innodb_force_recovery."; + dir.fill(0); - recv_sys->found_corrupt_fs = true; - return(false); - } + const auto path = str.substr(start, end - start); - ib::info() - << "innodb_force_recovery was set to " - << srv_force_recovery << ". Continuing crash" - " recovery even though we cannot access the" - " files for tablespace " << space_id << "."; - break; - } + ut_a(path.length() < dir.max_size()); - return(true); -} -#endif /* !UNIV_HOTBACKUP */ + std::copy(path.begin(), path.end(), dir.data()); -/** Write the open table (space_id -> name) mapping to disk */ -void -fil_tablespace_open_sync_to_disk() -{ - ut_ad(!srv_read_only_mode); + /* Filter out paths that contain '*'. */ + auto pos = path.find('*'); - fil_system->m_open.enter(); - fil_system->m_open.purge(); - fil_system->m_open.to_file(); - fil_system->m_open.exit(); -} + /* Filter out invalid path components. */ -/** Check if the name is an undo tablespace name. -@param[in] name Tablespace name -@param[in] len Tablespace name length in bytes -@return true if it is an undo tablespace name */ -bool -Fil_Open::is_undo_tablespace_name(const char* name, ulint len) -{ - if (len >= 8) { + if (path == "/") { - const char* end_ptr = name + len; - size_t u = (end_ptr[-4] == '_' ? 1 : 0); + ib::warn() << "Scan path '" << path << "' ignored"; - return(end_ptr[-8-u] == OS_PATH_SEPARATOR - && end_ptr[-7-u] == 'u' - && end_ptr[-6-u] == 'n' - && end_ptr[-5-u] == 'd' - && end_ptr[-4-u] == 'o' - && isdigit(end_ptr[-3]) - && isdigit(end_ptr[-2]) - && isdigit(end_ptr[-1])); - } + } else if (pos == std::string::npos) { - return(false); -} + Fil_path::normalize(dir.data()); -/** Process a file name from a MLOG_FILE_* record. -@param[in,out] path Path to file name -@param[in,out] path2 Set for rename only, new file name -@param[in] space_id the tablespace ID -@param[in] type type of the redo log record -@param[in] lsn LSN of the redo log record -@return true on success */ -static -bool -fil_name_process_for_recovery( - char* path, - char* path2, -#ifndef UNIV_HOTBACKUP - space_id_t space_id, -#else /* !UNIV_HOTBACKUP */ - const page_id_t& page_id, - ulint flags, -#endif /* !UNIV_HOTBACKUP */ - mlog_id_t type, - lsn_t lsn) -{ -#ifdef UNIV_HOTBACKUP - std::string abs_file_path; - std::string abs_file_path2; - std::string tablespace_name; - char* new_name = nullptr; - ulint new_len; - space_id_t space_id = page_id.space(); - meb_make_abs_file_path( - path, flags, space_id, abs_file_path, - tablespace_name); -#endif /* UNIV_HOTBACKUP */ + std::string cur_path; + std::string d{dir.data()}; - os_normalize_path(path); + if (Fil_path::get_file_type(dir.data()) + == OS_FILE_TYPE_DIR) { - if (path2 != nullptr) { - os_normalize_path(path2); - } + cur_path = Fil_path::get_real_path(d); - switch(type) { - case MLOG_FILE_DELETE: + } else { + cur_path = d; + } -#ifndef UNIV_HOTBACKUP - /* Single threaded mode, no need to acquire mutex. */ - fil_system->m_open.close(space_id, path); - fil_system->m_open.deleted(space_id); - fil_system->m_open.purge(); + if (!Fil_path::is_separator(d.back())) { + d.push_back(Fil_path::OS_SEPARATOR); + } - fil_space_free(space_id, false); + using value = Paths::value_type; - recv_sys->deleted.insert(space_id); - recv_sys->missing_ids.erase(space_id); -#else /* !UNIV_HOTBACKUP */ - meb_name_process(path, strlen(path), space_id, true); - if (meb_replay_file_ops - && fil_space_get(space_id)) { - dberr_t err = fil_delete_tablespace( - space_id, BUF_REMOVE_FLUSH_NO_WRITE); - ut_a(err == DB_SUCCESS); + dirs.push_back(value(d, cur_path)); + + } else { + ib::warn() + << "Scan path '" << path << "' ignored" + << " contains '*'"; } -#endif /* !UNIV_HOTBACKUP */ - break; - case MLOG_FILE_RENAME2: + start = str.find_first_not_of(delimiters, end); -#ifndef UNIV_HOTBACKUP - /* Best effort, the node may not exist. */ - fil_system->m_open.close(space_id, path); -#else /* !UNIV_HOTBACKUP */ - ut_a(path2 != nullptr); - meb_make_abs_file_path(path2, flags, space_id, abs_file_path2, - tablespace_name); - - if ((!meb_replay_file_ops) - || (meb_is_intermediate_file(path)) - || (meb_is_intermediate_file(path2)) - || (fil_space_get_id_by_name(tablespace_name.c_str()) - != SPACE_UNKNOWN) - || (meb_fil_space_get_rem_gen_ts_id_by_name( - tablespace_name) != SPACE_UNKNOWN) - || (NULL == fil_space_get(space_id))){ - /* Don't rename table while : - 1. Scanning the redo logs during backup - 2. Apply-log on a partial backup - 3. Either of old or new tables are intermediate table - 4. The new name is already loaded for recovery/apply-log - 5. The new name is a remote general tablespace which is - already loaded for recovery/apply-log from different - directory path - 6. Tablespace is not yet loaded in memory. - This will prevent unintended renames during recovery. */ - - ib::trace() << "Ignoring the log record. " - << "No need to rename tablespace"; - break; - } else { + end = str.find_first_of(delimiters, start); + } - ib::trace() << "Renaming space id : " << space_id - << ", old tablespace name : " << path - << " to new tablespace name : " << path2; + /* Remove duplicate paths by comparing the real paths. Note, this + will change the order of the directory scan because of the sort. */ - new_len = abs_file_path2.length() + 1; - new_name = static_cast( - ut_malloc_nokey(new_len)); - strcpy(new_name, abs_file_path2.c_str()); - } + using type = Paths::value_type; - meb_name_process(path, strlen(path), space_id, false); - meb_name_process(new_name, new_len, space_id, false); + std::sort(dirs.begin(), dirs.end(), [](const type& lhs, const type& rhs) + { + return(lhs.second < rhs.second); + }); - if (!meb_replay_file_ops) { - ut_free(new_name); - break; - } + dirs.erase( + std::unique( + dirs.begin(), dirs.end(), + [](const type& lhs, const type& rhs) + { + return(lhs.second == rhs.second); - if (!fil_op_replay_rename(page_id, abs_file_path.c_str(), - abs_file_path2.c_str())) { - recv_sys->found_corrupt_fs = true; - } + }), + dirs.end()); - ut_free(new_name); -#endif /* UNIV_HOTBACKUP */ + /* Eliminate sub-trees */ - path = path2; + Dirs scan_dirs; - /* Fall through */ + for (size_t i = 0; i < dirs.size(); ++i) { - case MLOG_FILE_OPEN: -#ifdef UNIV_HOTBACKUP - meb_name_process(path, strlen(path), space_id, false); - break; -#endif /* UNIV_HOTBACKUP */ - case MLOG_FILE_CREATE2: + const auto& path_i = dirs[i].second; -#ifndef UNIV_HOTBACKUP - /* Single threaded mode, no need to acquire mutex. */ - if (fil_system->m_open.can_load(space_id, path, lsn)) { + for (size_t j = i + 1; j < dirs.size(); ++j) { - if (!fil_tablespace_open_for_recovery( - space_id, path, lsn)) { + auto& path_j = dirs[j].second; - return(false); - } - } -#else /* !UNIV_HOTBACKUP */ - if ((!meb_replay_file_ops) - || (meb_is_intermediate_file(abs_file_path.c_str())) - || (fil_space_get(space_id)) - || (fil_space_get_id_by_name( - tablespace_name.c_str()) != SPACE_UNKNOWN) - || (meb_fil_space_get_rem_gen_ts_id_by_name( - tablespace_name) != SPACE_UNKNOWN)) { - /* Don't create table while :- - 1. scanning the redo logs during backup - 2. apply-log on a partial backup - 3. if it is intermediate file - 4. tablespace is already loaded in memory - 5. tablespace is a remote general tablespace which is - already loaded for recovery/apply-log from different - directory path */ - ib::trace() - << "Ignoring the log record. No need to " - << "create the tablespace : " << abs_file_path; - } else { - recv_spaces_t::iterator itr; - itr = recv_spaces.find(space_id); - if (itr == recv_spaces.end() - || (itr->second.name != abs_file_path)) { - - ib::trace() << "Creating the tablespace : " - << abs_file_path - << ", space_id : " << space_id; - dberr_t ret = fil_ibd_create( - space_id, tablespace_name.c_str(), - abs_file_path.c_str(), - flags, FIL_IBD_FILE_INITIAL_SIZE); - - if (ret != DB_SUCCESS) { - ib::fatal() << "Could not create the" - << " tablespace : " - << abs_file_path - << " with space Id : " - << space_id; - } + if (Fil_path::is_ancestor(path_i, path_j)) { + + path_j.resize(0); } } -#endif /* !UNIV_HOTBACKUP */ - break; - default: - ut_error; } - return(true); + for (auto& dir : dirs) { + + if (dir.second.length() == 0) { + continue; + } + + m_dirs.push_back(Tablespace_files{dir.first}); + } } -/** Parse or process a MLOG_FILE_* record. -@param[in] ptr redo log record -@param[in] end end of the redo log buffer -@param[in] page_id Tablespace Id and first page in file -@param[in] type MLOG_FILE_OPEN or MLOG_FILE_DELETE - or MLOG_FILE_CREATE2 or MLOG_FILE_RENAME2 -@param[in] parsed_bytes Number of bytes parsed so far -@return pointer to next redo log record -@retval NULL if this log record was truncated */ -byte* -fil_tablespace_name_recover( - byte* ptr, - const byte* end, - const page_id_t& page_id, - mlog_id_t type, - ulint parsed_bytes) +/** Check whether we can rename the file +@param[in] space Tablespace for which to rename +@param[in] name Source file name +@param[in] df Target file that exists on disk +@return DB_SUCCESS if all OK */ +static +dberr_t +fil_rename_validate(fil_space_t* space, const std::string& name, Datafile& df) { -#ifdef UNIV_HOTBACKUP - ulint flags = 0; - space_id_t space_id = page_id.space(); -#endif /* UNIV_HOTBACKUP */ + dberr_t err = df.validate_for_recovery(space->id); - if (type == MLOG_FILE_CREATE2) { + if (err == DB_TABLESPACE_NOT_FOUND) { - if (end < ptr + 4) { - return(nullptr); - } + /* Tablespace header doesn't contain the expected + tablespace ID. This is can happen during truncate. */ -#ifdef UNIV_HOTBACKUP - flags = mach_read_from_4(ptr); -#endif /* UNIV_HOTBACKUP */ - ptr += 4; - } + return(err); - if (end < ptr + 2) { - return(nullptr); - } + } else if (err != DB_SUCCESS) { - ulint len = mach_read_from_2(ptr); - ptr += 2; + ib::warn() + << "Failed to read the first page of the" + << " file '" << df.filepath() << "'." + << " You will need to verify and move the" + << " file out of the way retry recovery."; - if (end < ptr + len) { - return(nullptr); + return(err); } - bool corrupt = false; - byte* end_ptr = ptr + len; - char* name = reinterpret_cast(ptr); + auto file = &space->files.front(); - /* MLOG_FILE_* records should only be written for - user-created tablespaces. The name must end in .ibd. - Exception: MLOG_FILE_OPEN can be created for - predefined tablespaces. */ - os_normalize_path(name); + if (strcmp(df.filepath(), file->name) == 0) { - if (page_id.space() != TRX_SYS_SPACE - && !memcmp(end_ptr - 5, DOT_IBD, 5)) { + /* Check if already points to the correct file. + Must have the same space ID */ - /* User-defined tablespace (*.ibd file) */ + ib::info() + << "Tablespace ID already maps to: '" + << df.filepath() << "', rename ignored."; - if (page_id.page_no() != 0) { - corrupt = true; - } + ut_a(df.space_id() == space->id); - } else if (strcmp(name, dict_sys_t::s_dd_space_file_name) == 0) { - /* new dd tablespace (mysql.ibd) */ - if (page_id.page_no() != 0) { - corrupt = true; - } + return(DB_SUCCESS); - } else if (type != MLOG_FILE_OPEN) { + } else if (df.space_id() != space->id) { - } else if (Fil_Open::is_undo_tablespace_name(name, len - 1)) { + /* Target file exists on disk but has a different + tablespce ID. The user should manually delete it. */ - /* Undo tablespace */ - if (page_id.page_no() != 0) { - corrupt = true; - } + ib::error() + << "Cannot rename '" + << name << "' to '" + << df.filepath() << "'. File '" << df.filepath() + << "' tablespace ID " << df.space_id() + << " doesn't match the expected tablespace" + << " ID " << space->id + << ". You will need to verify and move '" + << df.filepath() << "' manually and retry recovery!"; - } else { - corrupt = true; + return(DB_ERROR); } -#ifdef UNIV_HOTBACKUP - if (corrupt) { - recv_sys->found_corrupt_log = true; - return(end_ptr); - } -#endif /* UNIV_HOTBACKUP */ + /* Target file exists on disk and has the same ID. */ - lsn_t commit_lsn; + ib::error() + << "Cannot rename '" << name << "' to '" << df.filepath() + << "'. The File '" << df.filepath() << " already exists on" + << " disk. You will need to verify and move either file" + << " manually and retry recovery!"; - ut_a(parsed_bytes != ULINT_UNDEFINED); + return(DB_ERROR); +} - switch (type) { - default: - ut_ad(0); // the caller checked this +/** Replay a file rename operation if possible. +@param[in] page_id Space ID and first page number in the file +@param[in] old_name old file name +@param[in] new_name new file name +@return whether the operation was successfully applied (the name did not exist, +or new_name did not exist and name was successfully renamed to new_name) */ +static +bool +fil_op_replay_rename( + const page_id_t& page_id, + const std::string& old_name, + const std::string& new_name) +{ +#ifdef UNIV_HOTBACKUP + ut_ad(meb_replay_file_ops); +#endif /* UNIV_HOTBACKUP */ - case MLOG_FILE_DELETE: - case MLOG_FILE_CREATE2: + ut_ad(page_id.page_no() == 0); + ut_ad(old_name.compare(new_name) != 0); + ut_ad(Fil_path::has_ibd_suffix(new_name)); + ut_ad(page_id.space() != TRX_SYS_SPACE); - ut_a(page_id.space() != TRX_SYS_SPACE); - // Fall through. - case MLOG_FILE_OPEN: + /* In order to replay the rename, the following must hold: + 1. The new name is not already used. + 2. A tablespace exists with the old name. + 3. The space ID for that tablepace matches this log entry. + This will prevent unintended renames during recovery. */ -#ifndef UNIV_HOTBACKUP - if (corrupt) { - recv_sys->found_corrupt_log = true; - break; - } -#else - /* Don't validate tablespaces while copying redo logs - because backup process might keep some tablespace handles - open in server datadir. */ - if (recv_is_making_a_backup) { - break; - } - ib::trace() - << get_mlog_string(type) - << " record for " - << "space_id : " << space_id - << ", name : " << name; -#endif /* !UNIV_HOTBACKUP */ + space_id_t space_id = page_id.space(); + fil_space_t* space = fil_space_get(space_id); - /* Length of the filename + 2 bytes for the length and - bytes parsed so far. This should give us the commit LSN. */ + if (space == nullptr) { + return(true); + } - commit_lsn = recv_calc_lsn_on_data_add( - recv_sys->recovered_lsn, len + 2 + parsed_bytes); + Datafile df; + std::string name{new_name}; - fil_name_process_for_recovery( - name, nullptr, -#ifndef UNIV_HOTBACKUP - page_id.space(), -#else /* !UNIV_HOTBACKUP */ - page_id, - flags, -#endif /* !UNIV_HOTBACKUP */ - type, commit_lsn); + df.set_filepath(name.c_str()); - break; + if (df.open_read_only(false) == DB_SUCCESS) { + dberr_t err = fil_rename_validate(space, old_name, df); - case MLOG_FILE_RENAME2: + if (err == DB_TABLESPACE_NOT_FOUND) { -#ifndef UNIV_HOTBACKUP - if (corrupt) { - recv_sys->found_corrupt_log = true; + /* This can happend during truncate. */ + ib::info() + << "Tablespace ID mismatch in '" << name << "'"; } -#endif /* !UNIV_HOTBACKUP */ - /* The new name follows the old name. */ - byte* new_name = end_ptr + 2; + df.close(); - if (end < new_name) { - return(NULL); - } + return(err == DB_SUCCESS); + } - ulint new_len = mach_read_from_2(end_ptr); + auto path_sep_pos = name.find_last_of(Fil_path::SEPARATOR); - if (end < end_ptr + 2 + new_len) { - return(NULL); - } + ut_a(path_sep_pos != std::string::npos); - end_ptr += 2 + new_len; + /* Create the database directory for the new name, if + it does not exist yet */ - corrupt = corrupt - || new_len < sizeof "/a.ibd\0" - || memcmp(new_name + new_len - 5, DOT_IBD, 5) != 0 - || !memchr(new_name, OS_PATH_SEPARATOR, new_len); + name.resize(path_sep_pos); - if (corrupt) { - recv_sys->found_corrupt_log = true; - break; - } + bool success = os_file_create_directory(name.c_str(), false); + ut_a(success); -#ifdef UNIV_HOTBACKUP - /* Don't validate tablespaces while copying redo logs - because backup process might keep some tablespace handles - open in server datadir. */ - if (recv_is_making_a_backup) { - break; - } + auto datadir_pos = name.find_last_of(Fil_path::SEPARATOR); - ib::trace() - << get_mlog_string(type) - << " record for " - << "space_id : " << space_id - << ", old table name : " << name - << ", new table name " << new_name; -#endif /* UNIV_HOTBACKUP */ + ut_ad(datadir_pos != std::string::npos); - /* Length of the (filename + 2 bytes) * 2 for the length and - bytes parsed so far. This should give us the commit LSN. */ + name.erase(0, datadir_pos + 1); - commit_lsn = recv_calc_lsn_on_data_add( - recv_sys->recovered_lsn, - new_len + len + 4 + parsed_bytes); + ut_ad(!Fil_path::is_separator(name.back())); - if (fil_name_process_for_recovery( - name, - reinterpret_cast(new_name), -#ifndef UNIV_HOTBACKUP - page_id.space(), -#else /* !UNIV_HOTBACKUP */ - page_id, - flags, -#endif /* !UNIV_HOTBACKUP */ - type, commit_lsn)) { -#ifndef UNIV_HOTBACKUP - std::string from( - reinterpret_cast(ptr)); + /* schema/table separator is always a '/'. */ + name.push_back('/'); - std::string to( - reinterpret_cast(new_name)); + /* Strip the '.ibd' suffix. */ + name.append(new_name.begin() + path_sep_pos + 1, new_name.end() - 4); - if (!fil_op_replay_rename( - page_id, from.c_str(), to.c_str())) { + ut_ad(!Fil_path::has_ibd_suffix(name)); - recv_sys->found_corrupt_fs = true; + clone_mark_abort(true); - ib::error() << "Cannot replay the rename '" - << from << "' to '" << to << "'" - << " for tablespace with ID " - << page_id.space() << "." - << " Please try to increase" - << " innodb_buffer_pool_size to" - << " see if it helps the redo" - << " recovery."; - } else { + const auto ptr = name.c_str(); - fil_system->m_open.rename( - page_id.space(), from, to); - } -#endif /* !UNIV_HOTBACKUP */ - } - } + success = fil_rename_tablespace( + space_id, old_name.c_str(), ptr, new_name.c_str()); + + ut_a(success); + + clone_mark_active(); - return(end_ptr); + return(true); } -/** Read the tablespace id to path mapping from the file -@param[in] recovery true if called from crash recovery */ -void -fil_tablespace_open_init_for_recovery(bool recovery) +/** Get the tablespace ID from an .ibd and/or an undo tablespace. If the ID +is == 0 on the first page then check for at least MAX_PAGES_TO_CHECK pages +with the same tablespace ID. Do a Light weight check before trying with +Datafile::find_space_id(). +@param[in] filename File name to check +@return ULINT32_UNDEFINED if not found, otherwise the space ID */ +space_id_t +Fil_system::get_tablespace_id(const std::string& filename) { - /* Single threaded mode, no need to acquire mutex. */ - if (recv_sys->is_cloned_db) { - -#ifndef UNIV_HOTBACKUP - fil_system->m_open.from_current_dir(); -#endif /* !UNIV_HOTBACKUP */ - } else { + dberr_t err = DB_CORRUPTION; + char buf[sizeof(space_id_t)]; + space_id_t space_id = ULINT32_UNDEFINED; + space_id_t prev_space_id = ULINT32_UNDEFINED; + std::ifstream ifs(filename, std::ios::binary); - fil_system->m_open.from_file(recovery); + if (!ifs) { + ib::warn() << "Unable to open '" << filename << "'"; + return(ULINT32_UNDEFINED); } -} -/** Lookup the space ID. -@param[in] space_id Tablespace ID to lookup -@return true if space ID is known. */ -bool -fil_tablespace_lookup_for_recovery(space_id_t space_id) -{ - ut_ad(recv_recovery_is_on()); + auto page_size = srv_page_size; - /* Single threaded mode, no need to acquire mutex. */ - if (fil_system->m_open.lookup(space_id)) { + for (page_no_t page_no = 0; page_no < MAX_PAGES_TO_CHECK; ++page_no) { - const auto end = recv_sys->deleted.end(); + off_t off; - return(recv_sys->deleted.find(space_id) == end); - } + off = page_no * page_size + FIL_PAGE_SPACE_ID; - return(false); -} + if (off == FIL_PAGE_SPACE_ID) { -#ifndef UNIV_HOTBACKUP -/** This function should be called after recovery has completed. -Check for tablespace files for which we did not see any MLOG_FILE_DELETE -or MLOG_FILE_RENAME record. These could not be recovered -@return true if there were some filenames missing for which we had to -ignore redo log records during the apply phase */ -bool -fil_check_missing_tablespaces() -{ - bool missing = false; - auto& dblwr = recv_sys->dblwr; - const auto end = recv_sys->deleted.end(); + /* Figure out the page size of the tablespace. If it's + a compressed tablespace. */ + ifs.seekg(FSP_SPACE_FLAGS, ifs.beg); - /* Called in single threaded mode, no need to acquire the mutex. */ + if ((ifs.rdstate() & std::ifstream::eofbit) != 0 + || (ifs.rdstate() & std::ifstream::failbit) != 0 + || (ifs.rdstate() & std::ifstream::badbit) != 0) { - /* First check if we were able to restore all the doublewrite - buffer pages. If not then print a warning. */ + return(ULINT32_UNDEFINED);; + } - for (auto& page : dblwr.deferred) { + ifs.read(buf, sizeof(buf)); - space_id_t space_id; + if (!ifs.good() + || (size_t) ifs.gcount() < sizeof(buf)) { - space_id = page_get_space_id(page.m_page); + return(ULINT32_UNDEFINED); + } - /* If the tablespace was in the missing IDs then we - know that the problem is elsewhere. If a file deleted - record was not found in the redo log and the tablespace - doesn't exist in the SYS_TABLESPACES file then it is - an error or data corruption. */ + ulint flags; - if (recv_sys->deleted.find(space_id) == end - && recv_sys->missing_ids.find(space_id) - != recv_sys->missing_ids.end()) { - page_no_t page_no; + flags = mach_read_from_4(reinterpret_cast(buf)); - page_no = page_get_page_no(page.m_page); + const page_size_t space_page_size(flags); - ib::warn() - << "Doublewrite page " << page.m_no - << " for {space: " << space_id << ", page_no:" - << page_no << "} could not be restored." - << " File name unknown for tablespace ID " - << space_id; + page_size = space_page_size.physical(); } - /* Free the memory. */ - page.close(); - } + ifs.seekg(off, ifs.beg); - dblwr.deferred.clear(); + if ((ifs.rdstate() & std::ifstream::eofbit) != 0 + || (ifs.rdstate() & std::ifstream::failbit) != 0 + || (ifs.rdstate() & std::ifstream::badbit) != 0) { - for (auto space_id : recv_sys->missing_ids) { + /* Trucated files can be a single page */ + return(page_no > 0 ? space_id : ULINT32_UNDEFINED); + } - if (recv_sys->deleted.find(space_id) != end) { + ifs.read(buf, sizeof(buf)); - continue; + if (!ifs.good() || (size_t) ifs.gcount() < sizeof(buf)) { + + /* Trucated files can be a single page */ + return(page_no > 0 ? space_id : ULINT32_UNDEFINED); } - const auto spaces = fil_system->m_open.m_spaces; - const auto space = spaces.find(space_id); + space_id = mach_read_from_4(reinterpret_cast(buf)); - if (space == spaces.end()) { + if (space_id == 0 || space_id == ULINT32_UNDEFINED) { - ib::error() - << "Could not find any file associated with" - << " the tablespace " << space_id; + /* We don't write the space ID to new pages. */ + if (prev_space_id != ULINT32_UNDEFINED + && prev_space_id != 0) { + + err = DB_SUCCESS; + space_id = prev_space_id; + + break; + } - missing = true; + continue; - } else { + } else if (space_id > 0 && prev_space_id == space_id) { - ut_ad(space_id == space->second.m_id); + ut_a(prev_space_id != ULINT32_UNDEFINED); - ib::error() - << "Some files associated with the tablespace" - << " " << space_id << " were not found:" - << " " << space->second.to_string(); + err = DB_SUCCESS; + break; + + } else if (space_id > 0) { - missing = true; + prev_space_id = space_id; } } - return(missing); -} + ifs.close(); -/** Open the tablespace for recovery. Get the tablespace filenames, -space_id must already be known. -@param[in] space_id Tablespace ID to open */ -void -Fil_Open::open_for_recovery(space_id_t space_id) const -{ - const auto it = m_spaces.find(space_id); + /* Try the more heavy duty method, as a last resort. */ + if (err != DB_SUCCESS) { + + /* The ifstream will work for all file formats compressed or + otherwise because the header of the page is not compressed. + Where it will fail is if the first page is corrupt. Then for + compressed tablespaces we don't know where the page boundary + starts because we don't know the page size. */ - if (it != m_spaces.end()) { + Datafile file; - for (const auto& file : it->second.m_files) { - fil_tablespace_open_for_recovery( - space_id, file.m_name, file.m_open_lsn); + file.set_filepath(filename.c_str()); + + err = file.open_read_only(false); + + ut_a(file.is_open()); + ut_a(err == DB_SUCCESS); + + /* Read and validate the first page of the tablespace. + Assign a tablespace name based on the tablespace type. */ + err = file.find_space_id(); + + if (err == DB_SUCCESS) { + space_id = file.space_id(); } + + file.close(); } -} -/** Open the tablespace and also get the tablespace filenames, space_id must -already be known. -@param[in] space_id Tablespace ID to lookup */ -void -fil_tablespace_open_for_recovery(space_id_t space_id) -{ - fil_system->m_open.open_for_recovery(space_id); + return(err != DB_SUCCESS) ? ULINT32_UNDEFINED : space_id; } -/** Clear the tablspace ID to filename mapping. */ +/** Check for duplicate tablespace IDs. +@param[in] start Slice start +@param[in] end Slice end +@param[in] thread_id Thread ID +@param[in,out] mutex Mutex that covers the global state +@param[in,out] unique To check for duplciates +@param[in,out] duplicates Duplicate space IDs found */ void -fil_tablespace_open_clear() +Tablespace_dirs::duplicate_check( + const Const_iter& start, + const Const_iter& end, + size_t thread_id, + std::mutex* mutex, + Space_id_set* unique, + Space_id_set* duplicates) { - if (srv_read_only_mode) { - return; - } + size_t count = 0; + bool printed_msg = false; + auto start_time = ut_time(); - ut_a(srv_shutdown_state == SRV_SHUTDOWN_LAST_PHASE); + for (auto it = start; it != end; ++it, ++m_checked) { - fil_system->m_open.enter(); + const std::string filename = it->second; + auto& files = m_dirs[it->first]; + const std::string phy_filename = files.path() + filename; - fil_system->m_open.clear(); + space_id_t space_id; - /* This will truncate all the files on disk too. */ - for (auto path : PATHS) { - fil_system->m_open.to_file(); - } + space_id = Fil_system::get_tablespace_id(phy_filename); - fil_system->m_open.exit(); -} + ut_a(space_id != 0); -/** Tokenize a path specification. Convert relative paths to absolute paths. -Check if the paths are valid and filter out invalid or unreadable directories. -Sort and filter out duplicates from dirs. -@param[in] str Path specification to tokenize -@param[out] dirs Parsed directory paths -@param[in] delimiters Delimiters */ -static -void -fil_tokenize_paths( - const std::string& str, - std::vector& dirs, - const std::string& delimiters) -{ - std::string::size_type start = str.find_first_not_of(delimiters); - std::string::size_type end = str.find_first_of(delimiters, start); + if (space_id != ULINT32_UNDEFINED) { - /* Scan until 'end' and 'start' don't reach the end of string (npos) */ - while (std::string::npos != start || std::string::npos != end) { + std::lock_guard guard(*mutex); - std::array dir; + auto ret = unique->insert(space_id); - dir.fill(0); + size_t n_files; - const auto path = str.substr(start, end - start); + n_files = files.add(space_id, filename); - ut_a(path.length() < dir.max_size()); + if (n_files > 1 || !ret.second) { - std::copy(path.begin(), path.end(), dir.data()); + duplicates->insert(space_id); + } - /* Filter out paths that contain '*'. */ - auto pos = path.find('*'); + } else if (Fil_path::is_undo_tablespace_name(phy_filename)) { - /* Filter out invalid path components. */ - if (pos == std::string::npos) { + ib::info() + << "Can't determine the undo file tablespace" + << " ID for '" << phy_filename << "', could be" + << " an undo truncate in progress"; + + } else { + + ib::warn() + << "Ignoring '" << phy_filename << "' invalid" + << " tablespace ID in the header"; + } - os_normalize_path(dir.data()); + ++count; - /* Don't convert to absolute path as the same would - be loaded to innodb tablespace node. Otherwise, - it could mistmatch with DD node path. - abs_path = Fil_Open::get_path(dir.data(), ""); */ + if (ut_time() - start_time >= PRINT_INTERVAL_SECS) { + + ib::info() + << "Thread# " << thread_id + << " - Checked " + << count << "/" << (end - start) + << " files"; - std::string cur_path(dir.data()); + start_time = ut_time(); - os_file_type_t type; - bool exists; + printed_msg = true; + } + } - if (os_file_status( - cur_path.c_str(), &exists, &type) && exists) { + if (printed_msg) { + ib::info() << "Checked " << count << " files"; + } +} - if (type == OS_FILE_TYPE_DIR) { +/** Print the duplicate filenames for a tablespce ID to the log +@param[in] duplicates Duplicate tablespace IDs*/ +void +Tablespace_dirs::print_duplicates(const Space_id_set& duplicates) +{ + /* Print the duplicate names to the error log. */ + for (auto space_id : duplicates) { + Dirs files; - dirs.push_back(cur_path); + for (auto& dir : m_dirs) { - } else { - ib::warn() - << "'" << path << "' ignored, " - << " not a directory"; - } + const auto names = dir.find(space_id); - } else { - ib::warn() - << "'" << path << "' ignored" - << " os_file_status() failed."; + if (names == nullptr) { + continue; } - } else { - ib::warn() - << "Scan path '" << path << "' ignored" - << " contains '*'"; + + files.insert(files.end(), names->begin(), names->end()); } - start = str.find_first_not_of(delimiters, end); + /* Fixes the order in the mtr tests. */ + std::sort(files.begin(), files.end()); - end = str.find_first_of(delimiters, start); - } + ut_a(files.size() > 1); + + std::ostringstream oss; + + oss << "Tablespace ID: " << space_id << " = ["; + + for (size_t i = 0; i < files.size(); ++i) { + + oss << "'" << files[i] << "'"; - /* Remove duplicate paths. Note, this will change the order of - the directory scan because of the sort. */ + if (i < files.size() - 1) { + oss << ", "; + } + } - std::sort(dirs.begin(), dirs.end()); - dirs.erase(std::unique(dirs.begin(), dirs.end() ), dirs.end()); + oss << "]" << std::endl; + + ib::error() << oss.str(); + } } /** Discover tablespaces by reading the header from .ibd files. -@param[in] directories Directories to scan +@param[in] in_directories Directories to scan @return DB_SUCCESS if all goes well */ dberr_t -fil_scan_for_tablespaces(const std::string& directories) +Tablespace_dirs::scan(const std::string& in_directories) { - using Dirs = std::vector; + std::string directories(in_directories); + + Fil_path::normalize(directories); ib::info() << "Directories to scan '" << directories << "'"; - Dirs dirs; - Dirs filenames; + Scanned_files ibd_files; + Scanned_files undo_files; + + { + std::string separators; + + separators.push_back(FIL_PATH_SEPARATOR); + + tokenize_paths(directories, separators); + } - fil_tokenize_paths(directories, dirs, ";"); + uint16_t count = 0; + bool print_msg = false; + auto start_time = ut_time(); /* Should be trivial to parallelize the scan and ID check. */ - for (const auto& dir : dirs) { + for (const auto& dir : m_dirs) { - ib::info() << "Scanning '" << dir << "'"; + const auto& real_path_dir = dir.real_path(); + + ut_a(Fil_path::is_separator(dir.path().back())); + + ib::info() << "Scanning '" << dir.path() << "'"; /* Walk the sub-tree of dir. */ - Dir_Walker::walk( - dir, - [&](const std::string& path) - { - /* If it is a file and the suffix - matches ".ibd" then store it for reading. */ - - if (!Dir_Walker::is_directory(path) - && path.size() >= 4 - && (path.compare(path.length() - 4, 4, - ".ibd") == 0 - || Fil_Open::is_undo_tablespace_name( - path.c_str(), - path.length()))) { - - filenames.push_back(path); - } - }); - } + Dir_Walker::walk(real_path_dir, [&](const std::string& path) + { + /* If it is a file and the suffix matches ".ibd" + or the undo file name format then store it for + determining the space ID. */ - ib::info() << "Found " << filenames.size() << " '.ibd' file(s)"; + ut_a(path.length() > real_path_dir.length()); + ut_a(Fil_path::get_file_type(path) != OS_FILE_TYPE_DIR); - ut_a(fil_scanned == nullptr); - fil_scanned = UT_NEW_NOKEY(Fil_Open()); + /* Make the filename relative to the directory that + was scanned. */ - using Duplicates = std::set; + std::string file = path.substr( + real_path_dir.length(), path.length()); - Duplicates duplicates; + if (file.size() <= 4) { - for (const auto& filename : filenames) { + return; + } - std::ifstream ifs(filename, std::ios::binary); + using value = Scanned_files::value_type; - if (!ifs) { - ib::warn() << "Unable to open '" << filename << "'"; - continue; - } + if (Fil_path::has_ibd_suffix(file.c_str())) { - ifs.seekg(FIL_PAGE_SPACE_ID, ifs.beg); + ibd_files.push_back(value{count, file}); - char buf[sizeof(space_id_t)]; + } else if (Fil_path::is_undo_tablespace_name(file)) { - ifs.read(buf, sizeof(buf)); + undo_files.push_back(value{count, file}); + } - if (!ifs.good() || (size_t) ifs.gcount() < sizeof(buf)) { + if (ut_time() - start_time >= PRINT_INTERVAL_SECS) { - ib::warn() - << "Unable to read tablespace ID from" - << " '" << filename << "'"; - } else { + ib::info() + << "Files found so far: " + << ibd_files.size() << " data files" + << " and " + << undo_files.size() << " undo files"; - space_id_t space_id; + start_time = ut_time(); + print_msg = true; + } + }); - space_id = mach_read_from_4( - reinterpret_cast(buf)); + ++count; + } - const auto it = fil_scanned->m_spaces.find(space_id); + if (print_msg) { + ib::info() + << "Found " << ibd_files.size() + << " '.ibd' and " + << undo_files.size() << " undo files"; + } - if (it != fil_scanned->m_spaces.end()) { + Space_id_set unique; + Space_id_set duplicates; - ut_a(it->second.m_id == space_id); + size_t n_threads = (ibd_files.size() / 50000); - duplicates.insert(space_id); - } + if (n_threads > 0) { - fil_scanned->load( - space_id, filename, Fil_Open::Nodes::INIT, 0); + if (n_threads > MAX_SCAN_THREADS) { + n_threads = MAX_SCAN_THREADS; } - ifs.close(); + ib::info() + << "Using " << (n_threads + 1) << " threads to" + << " scan the tablespace files"; } + std::mutex m; + + using std::placeholders::_1; + using std::placeholders::_2; + using std::placeholders::_3; + using std::placeholders::_4; + using std::placeholders::_5; + using std::placeholders::_6; + + std::function + check = std::bind( + &Tablespace_dirs::duplicate_check, this, + _1, _2, _3, _4, _5, _6); + + par_for(PFS_NOT_INSTRUMENTED, + ibd_files, n_threads, check, &m, &unique, &duplicates); + + duplicate_check( + undo_files.begin(), undo_files.end(), n_threads, + &m, &unique, &duplicates); + + ut_a(m_checked == ibd_files.size() + undo_files.size()); + + ib::info() << "Completed space ID check of " << m_checked << " files."; + dberr_t err; if (!duplicates.empty()) { - ib::warn() << "Multiple files found for tablespace ID(s)"; + ib::error() + << "Multiple files found for the same tablespace ID:"; + + print_duplicates(duplicates); err = DB_FAIL; } else { err = DB_SUCCESS; } - for (auto space_id : duplicates) { + return(err); +} + +/** Discover tablespaces by reading the header from .ibd files. +@param[in] directories Directories to scan +@return DB_SUCCESS if all goes well */ +dberr_t +fil_scan_for_tablespaces(const std::string& directories) +{ + return(fil_system->scan(directories)); +} - const auto it = fil_scanned->m_spaces.find(space_id); +/** Callback to check tablespace size with space header size and extend. +Caller must own the Fil_shard mutex that the file belongs to. +@param[in] file Tablespace file +@return error code */ +dberr_t +fil_check_extend_space(fil_node_t* file) +{ + dberr_t err = DB_SUCCESS; + bool open_node = !file->is_open; - ut_a(it != fil_scanned->m_spaces.end()); + if (recv_sys == nullptr || !recv_sys->is_cloned_db) { - std::ostringstream oss; + return(DB_SUCCESS); + } + + fil_space_t* space = file->space; + + auto shard = fil_system->shard_by_id(space->id); + + if (open_node && !shard->open_file(file, false)) { + + return(DB_CANNOT_OPEN_FILE); + } + + shard->mutex_release(); + + if (space->size < space->size_in_header) { - oss << "Tablespace ID: " << space_id; + ib::info() + << "Extending space: " << space->name + << " from size " << space->size + << " pages to " << space->size_in_header + << " pages as stored in space header."; + + if (!shard->space_extend(space, space->size_in_header)) { + + ib::error() + << "Failed to extend tablespace." + << " Check for free space in disk" + << " and try again."; - for (const auto& path : it->second.m_files) { - oss << std::endl << '\t' << "'" << path.m_name << "'"; + err = DB_OUT_OF_FILE_SPACE; } + } + + shard->mutex_acquire(); - ib::warn() << oss.str(); + /* Close the file if it was opened by current function */ + if (open_node) { + + shard->close_file(file, true); } return(err); } -#endif /* !UNIV_HOTBACKUP */ -/** Create tablespaces.open.* files. */ +/** Check if a path is known to InnoDB. +@param[in] path Path to check +@return true if path is known to InnoDB */ +bool +fil_check_path(const std::string& path) +{ + return(fil_system->check_path(path)); +} + +/** Get the list of directories that datafiles can reside in. +@return the list of directories 'dir1;dir2;....;dirN' */ +std::string +fil_get_dirs() +{ + return(fil_system->get_dirs()); +} + +/** Free the data structures required for recovery. */ void -fil_tablespace_open_create() +fil_free_scanned_files() { - if (!srv_read_only_mode) { - fil_system->m_open.create_open_files(); - } + fil_system->free_scanned_files(); } /** Update the tablespace name. Incase, the new name and old name are same, no update done. @param[in,out] space tablespace object on which name will be updated -@param[in] name new name for tablespace -@param[in] has_fil_sys true if fil_system mutex is - acquired */ +@param[in] name new name for tablespace */ void -fil_space_update_name( - fil_space_t* space, - const char* name, - bool has_fil_sys) +fil_space_update_name(fil_space_t* space, const char* name) { - if (space == nullptr || name == nullptr + if (space == nullptr + || name == nullptr || space->name == nullptr || strcmp(space->name, name) == 0) { + return; } - if (!has_fil_sys) { - mutex_enter(&fil_system->mutex); + dberr_t err = fil_rename_tablespace_by_name(space->name, name); + + if (err != DB_SUCCESS) { + ib::warn() + << "Tablespace rename '" << space->name << "' to" + << " '" << name << "' failed!"; + + } +} + +#ifndef UNIV_HOTBACKUP +/** Check if the filepath provided is in a valid placement. +1) File-per-table must be in a dir named for the schema. +2) File-per-table must not be in the datadir. +3) General tablespace must not be under the datadir. +@param[in] space_name tablespace name +@param[in] path filepath to validate +@retval true if the filepath is a valid datafile location */ +bool +Fil_path::is_valid_location( + const char* space_name, + const std::string& path) +{ + ut_ad(!path.empty()); + ut_ad(space_name != nullptr); + + std::string name{space_name}; + + /* The path is a realpath to a file. Make sure it is not an + undo tablespace filename. Undo datafiles can be located anywhere. */ + if (Fil_path::is_undo_tablespace_name(path)) { + return(true); + } + + /* Strip off the filename to reduce the path to a directory. */ + std::string dirpath{path}; + auto pos = dirpath.find_last_of(SEPARATOR); + + dirpath.resize(pos); + + pos = name.find_last_of(SEPARATOR); + + if (pos == std::string::npos) { + + /* This is a general or system tablespace. */ + + if (MySQL_datadir_path.is_ancestor(dirpath)) { + ib::error() + << "A general tablespace cannot" + << " be located under the datadir." + << " Cannot open file '" << path << "'."; + return(false); + } + } else { - ut_ad(mutex_own(&fil_system->mutex)); + /* This is a file-per-table datafile. + Reduce the name to just the db name. */ + + name.resize(pos); + + if (MySQL_datadir_path.is_same_as(dirpath)) { + ib::error() + << "A file-per-table tablespace cannot" + << " be located in the datadir." + << " Cannot open file" << path << "'."; + return(false); + } + + /* Get the subdir that the file is in. */ + pos = dirpath.find_last_of(SEPARATOR); + + std::string subdir = + (pos == std::string::npos) + ? dirpath + : dirpath.substr(pos + 1, dirpath.length()); + + if (name != subdir) { + + Fil_path::convert_to_filename_charset(name); + + if (name != subdir) { + + return(false); + } + } } - /* Update the name */ - fil_system->names.erase(space->name); - ut_free(space->name); - space->name = mem_strdup(name); -#ifdef UNIV_DEBUG - auto it = -#endif /* UNIV_DEBUG */ - fil_system->names.insert( - Names::value_type(space->name, space)); - ut_ad(it.second); + return(true); +} - if (!has_fil_sys) { - mutex_exit(&fil_system->mutex); +/** Convert filename to the file system charset format. +@param[in,out] name Filename to convert */ +void +Fil_path::convert_to_filename_charset(std::string& name) +{ + uint errors = 0; + char old_name[MAX_TABLE_NAME_LEN + 20]; + char filename[MAX_TABLE_NAME_LEN + 20]; + + strncpy(filename, name.c_str(), sizeof(filename)); + strncpy(old_name, filename, sizeof(old_name)); + + ut_ad(strchr(filename, '/') == nullptr); + + innobase_convert_to_filename_charset( + filename, old_name, MAX_TABLE_NAME_LEN); + + if (errors == 0) { + name.assign(filename); } } + +#endif /* !UNIV_HOTBACKUP */ diff --git a/storage/innobase/fsp/fsp0file.cc b/storage/innobase/fsp/fsp0file.cc index 697c0a6374b0..7da1a0381c7e 100644 --- a/storage/innobase/fsp/fsp0file.cc +++ b/storage/innobase/fsp/fsp0file.cc @@ -78,12 +78,6 @@ Datafile::shutdown() ut_free(m_encryption_iv); m_encryption_iv = NULL; } -#ifdef UNIV_HOTBACKUP - if (m_dirpath != NULL) { - ut_free(m_dirpath); - m_dirpath = NULL; - } -#endif /* UNIV_HOTBACKUP */ } /** Create/open a data file. @@ -218,13 +212,22 @@ void Datafile::make_filepath( const char* dirpath, const char* filename, - ib_extention ext) + ib_file_suffix ext) { - ut_ad(dirpath != NULL || filename != NULL); - free_filepath(); - m_filepath = fil_make_filepath(dirpath, filename, ext, false); + std::string path; + std::string name; + + if (dirpath != nullptr) { + path.assign(dirpath); + } + + if (filename != nullptr) { + name.assign(filename); + } + + m_filepath = Fil_path::make(path, name, ext); ut_ad(m_filepath != NULL); @@ -428,7 +431,7 @@ Datafile::validate_to_dd( dberr_t err; if (!is_open()) { - return DB_ERROR; + return(DB_ERROR); } /* Validate this single-table-tablespace with the data dictionary, @@ -543,7 +546,7 @@ m_is_valid is set true on success, else false. @param[out] flush_lsn contents of FIL_PAGE_FILE_FLUSH_LSN @param[in] for_import if it is for importing (only valid for the first file of the system tablespace) -@retval DB_TABLESPACE_NOT_FOUND tablespace in file header doesn't match +@retval DB_WRONG_FILE_NAME tablespace in file header doesn't match expected value @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @@ -641,7 +644,7 @@ Datafile::validate_first_page( << m_space_id; #endif /* !UNIV_HOTBACKUP */ - return(DB_TABLESPACE_NOT_FOUND); + return(DB_WRONG_FILE_NAME); } else { BlockReporter reporter( @@ -688,7 +691,7 @@ Datafile::validate_first_page( m_encryption_iv = static_cast( ut_zalloc_nokey(ENCRYPTION_KEY_LEN)); #ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "Got from file %lu:", m_space_id); + fprintf(stderr, "Got from file %lu:", m_space_id); #endif #ifdef UNIV_HOTBACKUP diff --git a/storage/innobase/fsp/fsp0fsp.cc b/storage/innobase/fsp/fsp0fsp.cc index d36a3bbd8536..c37464f5f63c 100644 --- a/storage/innobase/fsp/fsp0fsp.cc +++ b/storage/innobase/fsp/fsp0fsp.cc @@ -710,8 +710,9 @@ xdes_lst_get_descriptor( xdes_t* descr; ut_ad(mtr); - ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space, NULL), - MTR_MEMO_X_LOCK)); + ut_ad(mtr_memo_contains( + mtr, fil_space_get_latch(space), MTR_MEMO_X_LOCK)); + descr = fut_get_ptr(space, page_size, lst_node, RW_SX_LATCH, mtr) - XDES_FLST_NODE; @@ -851,22 +852,21 @@ fsp_parse_init_file_page( return(ptr); } -/**********************************************************************//** -Initializes the fsp system. */ +/** Initializes the fsp system. */ void -fsp_init(void) -/*==========*/ +fsp_init() { /* FSP_EXTENT_SIZE must be a multiple of page & zip size */ + ut_a(UNIV_PAGE_SIZE > 0); ut_a(0 == (UNIV_PAGE_SIZE % FSP_EXTENT_SIZE)); - ut_a(UNIV_PAGE_SIZE); -#if UNIV_PAGE_SIZE_MAX % FSP_EXTENT_SIZE_MAX -# error "UNIV_PAGE_SIZE_MAX % FSP_EXTENT_SIZE_MAX != 0" -#endif -#if UNIV_ZIP_SIZE_MIN % FSP_EXTENT_SIZE_MIN -# error "UNIV_ZIP_SIZE_MIN % FSP_EXTENT_SIZE_MIN != 0" -#endif + static_assert( + !(UNIV_PAGE_SIZE_MAX % FSP_EXTENT_SIZE_MAX), + "UNIV_PAGE_SIZE_MAX % FSP_EXTENT_SIZE_MAX != 0"); + + static_assert( + !(UNIV_ZIP_SIZE_MIN % FSP_EXTENT_SIZE_MIN), + "UNIV_ZIP_SIZE_MIN % FSP_EXTENT_SIZE_MIN != 0"); /* Does nothing at the moment */ } @@ -953,7 +953,7 @@ fsp_header_write_encryption( tablespaces. */ master_key_id = mach_read_from_4( page + offset + ENCRYPTION_MAGIC_SIZE); - if (master_key_id == Encryption::master_key_id) { + if (master_key_id == Encryption::s_master_key_id) { ut_ad(memcmp(page + offset, ENCRYPTION_KEY_MAGIC_V1, ENCRYPTION_MAGIC_SIZE) == 0 @@ -1259,19 +1259,18 @@ fsp_try_extend_data_file_with_pages( fsp_header_t* header, mtr_t* mtr) { - bool success; - ulint size; DBUG_ENTER("fsp_try_extend_data_file_with_pages"); ut_a(!fsp_is_system_or_temp_tablespace(space->id)); ut_d(fsp_space_modify_check(space->id, mtr)); - size = mach_read_from_4(header + FSP_SIZE); + page_no_t size = mach_read_from_4(header + FSP_SIZE); ut_ad(size == space->size_in_header); ut_a(page_no >= size); - success = fil_space_extend(space, page_no + 1); + bool success = fil_space_extend(space, page_no + 1); + /* The size may be less than we wanted if we ran out of disk space. */ fsp_header_size_update(header, space->size, mtr); space->size_in_header = space->size; @@ -1593,8 +1592,8 @@ fsp_fill_free_list( i += FSP_EXTENT_SIZE; } - - space->free_len += count; + ut_a(count < std::numeric_limits::max()); + space->free_len += (uint32_t) count; } /** Allocates a new free extent. @@ -2830,7 +2829,7 @@ fsp_alloc_xdes_free_frag( ulint n_used; ut_ad(mtr); - ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space, NULL), + ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space), MTR_MEMO_X_LOCK)); fsp_header_t* header = fsp_get_space_header(space, page_size, mtr); @@ -3476,12 +3475,18 @@ tablespace without running out of space. uintmax_t fsp_get_available_space_in_free_extents(space_id_t space_id) { - FilSpace space(space_id); - if (space() == NULL) { + fil_space_t* space = fil_space_acquire(space_id); + + if (space == nullptr) { + return(UINTMAX_MAX); } - return(fsp_get_available_space_in_free_extents(space)); + auto n_free_extents = fsp_get_available_space_in_free_extents(space); + + fil_space_release(space); + + return(n_free_extents); } /** Calculate how many KiB of new data we will be able to insert to the diff --git a/storage/innobase/fsp/fsp0space.cc b/storage/innobase/fsp/fsp0space.cc index f6a6415c282d..f45ffd00496b 100644 --- a/storage/innobase/fsp/fsp0space.cc +++ b/storage/innobase/fsp/fsp0space.cc @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -151,7 +151,7 @@ Tablespace::open_or_create(bool is_temp) ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE); } - ut_a(fil_validate()); + ut_ad(fil_validate()); /* Create the tablespace node entry for this data file. */ if (!fil_node_create( @@ -220,36 +220,38 @@ Tablespace::add_datafile( /* The path provided ends in ".ibd". This was assured by validate_create_tablespace_info() */ ut_d(const char* dot = strrchr(datafile_added, '.')); - ut_ad(dot != NULL && 0 == strcmp(dot, DOT_IBD)); + ut_ad(dot != NULL && Fil_path::has_ibd_suffix(dot)); - char* filepath = mem_strdup(datafile_added); - if (filepath == NULL) { - return(DB_OUT_OF_MEMORY); - } - os_normalize_path(filepath); + std::string filepath{datafile_added}; + + Fil_path::normalize(filepath); /* If the path is an absolute path, separate it onto m_path and a basename. For relative paths, make the whole thing a basename so that it can be appended to the datadir. */ - bool is_abs_path = is_absolute_path(filepath); - size_t dirlen = (is_abs_path ? dirname_length(filepath) : 0); - const char* basename = filepath + dirlen; + bool is_abs_path = Fil_path::is_absolute_path(filepath); + size_t dirlen = (is_abs_path ? dirname_length(filepath.c_str()) : 0); /* If the pathname contains a directory separator, fill the m_path member which is the default directory for files in this tablespace. Leave it null otherwise. */ if (dirlen > 0) { - set_path(filepath, dirlen); + set_path(filepath.c_str(), dirlen); } - /* Now add a new Datafile and set the filepath - using the m_path created above. */ - m_files.push_back(Datafile(m_name, m_flags, - FIL_IBD_FILE_INITIAL_SIZE, 0)); - Datafile* datafile = &m_files.back(); - datafile->make_filepath(m_path, basename, IBD); + Datafile value(m_name, m_flags, FIL_IBD_FILE_INITIAL_SIZE, 0); + + /* Now add a new Datafile and set the filepath using the m_path + created above. */ + m_files.push_back(value); + + Datafile* datafile = &m_files.back(); + + if (dirlen > 0) { + filepath.erase(0, dirlen); + } - ut_free(filepath); + datafile->make_filepath(m_path, filepath.c_str(), IBD); return(DB_SUCCESS); } diff --git a/storage/innobase/fsp/fsp0sysspace.cc b/storage/innobase/fsp/fsp0sysspace.cc index e5d729516e40..a7ed836a302e 100644 --- a/storage/innobase/fsp/fsp0sysspace.cc +++ b/storage/innobase/fsp/fsp0sysspace.cc @@ -458,7 +458,6 @@ SysTablespace::create_file( break; } - if (err == DB_SUCCESS && file.m_type != SRV_OLD_RAW) { err = set_size(file); } @@ -949,7 +948,7 @@ SysTablespace::open_or_create( ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE); } - ut_a(fil_validate()); + ut_ad(fil_validate()); page_no_t max_size = (++node_counter == m_files.size() ? (m_last_file_size_max == 0 diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 1c66d7d6f414..f1dd53bae49f 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -36,20 +36,28 @@ this program; if not, write to the Free Software Foundation, Inc., #ifndef UNIV_HOTBACKUP # include "my_config.h" - -# include -# include -# include #endif /* !UNIV_HOTBACKUP */ + +#include +#include +#include +#include #include #include #include #include #include #include -#include -#include + +#include +#include "mysql/components/services/system_variable_source.h" + #ifndef UNIV_HOTBACKUP +# include +# include +# include +# include +# include # include # include # include @@ -57,22 +65,11 @@ this program; if not, write to the Free Software Foundation, Inc., # include # include # include -#endif /* !UNIV_HOTBACKUP */ -#include -#ifndef UNIV_HOTBACKUP # include # include -#endif /* !UNIV_HOTBACKUP */ -#include -#include -#include -#include - -#ifndef UNIV_HOTBACKUP # include "api0api.h" # include "api0misc.h" -/* Include necessary InnoDB headers */ -#include "auth_acls.h" +# include "auth_acls.h" # include "btr0btr.h" # include "btr0bulk.h" # include "btr0cur.h" @@ -82,14 +79,14 @@ this program; if not, write to the Free Software Foundation, Inc., # include "buf0flu.h" # include "buf0lru.h" # include "buf0stats.h" -#include "clone0api.h" -#include "dd/dd.h" -#include "dd/dictionary.h" -#include "dd/properties.h" -#include "dd/types/index.h" -#include "dd/types/partition.h" -#include "dd/types/table.h" -#include "dd/types/tablespace.h" +# include "clone0api.h" +# include "dd/dd.h" +# include "dd/dictionary.h" +# include "dd/properties.h" +# include "dd/types/index.h" +# include "dd/types/partition.h" +# include "dd/types/table.h" +# include "dd/types/tablespace.h" # include "dict0boot.h" # include "dict0crea.h" # include "dict0dd.h" @@ -113,9 +110,10 @@ this program; if not, write to the Free Software Foundation, Inc., # include "lex_string.h" # include "lob0lob.h" # include "lock0lock.h" -#endif /* !UNIV_HOTBACKUP */ -#include "log0log.h" -#ifndef UNIV_HOTBACKUP +# include "pars0pars.h" +# include "row0import.h" +# include "row0merge.h" +# include "os0thread-create.h" # include "mem0mem.h" # include "mtr0mtr.h" # include "my_dbug.h" @@ -125,19 +123,13 @@ this program; if not, write to the Free Software Foundation, Inc., # include "my_psi_config.h" # include "mysql/components/services/log_builtins.h" # include "mysql/psi/mysql_data_lock.h" -#endif /* !UNIV_HOTBACKUP */ -#include "mysys_err.h" -#include "os0file.h" -#ifndef UNIV_HOTBACKUP +# include "mysys_err.h" # include "os0thread.h" # include "p_s.h" # include "page0zip.h" -# include "pars0pars.h" # include "rem0types.h" # include "row0ext.h" -# include "row0import.h" # include "row0ins.h" -# include "row0merge.h" # include "row0mysql.h" # include "row0quiesce.h" # include "row0sel.h" @@ -147,70 +139,50 @@ this program; if not, write to the Free Software Foundation, Inc., # include "srv0srv.h" # include "srv0start.h" # include "sync0sync.h" -#ifdef UNIV_DEBUG -# include "trx0purge.h" -#endif /* UNIV_DEBUG */ -#include "dict0priv.h" -#include "dict0sdi.h" -#include "dict0upgrade.h" -#include "sql_base.h" // OPEN_FRM_FILE_ONLY -#include "sql/item.h" +# ifdef UNIV_DEBUG +# include "trx0purge.h" +# endif /* UNIV_DEBUG */ +# include "dict0priv.h" +# include "dict0sdi.h" +# include "dict0upgrade.h" +# include "sql_base.h" +# include "sql/item.h" # include "trx0roll.h" -#include "trx0rseg.h" +# include "trx0rseg.h" # include "trx0sys.h" # include "trx0trx.h" # include "trx0xa.h" -# include "univ.i" # include "ut0mem.h" -#endif /* !UNIV_HOTBACKUP */ - -#ifdef UNIV_HOTBACKUP +#else +# include "univ.i" # include # include "buf0types.h" -#endif /* UNIV_HOTBACKUP */ - -#ifndef UNIV_HOTBACKUP -#include +#endif /* !UNIV_HOTBACKUP */ -SERVICE_TYPE(registry) *reg_svc = nullptr; -my_h_service h_ret_sysvar_source_svc = nullptr; -SERVICE_TYPE(system_variable_source) *sysvar_source_svc = nullptr; +#include "os0file.h" +#include "log0log.h" -/* -------------------- SYSTEM MEMORY ----------------------------- */ -static const uint64_t MB = 1048576; //1024 * 1024 -static const uint64_t GB = 1073741824; //1024 * 1024 * 1024 +#include +#include #ifdef HAVE_UNISTD_H -#include +# include #endif /* HAVE_UNISTD_H */ -#if defined(_WIN32) || defined(_WIN64) -#include -static -double -get_mem_GlobalMemoryStatus() -{ - MEMORYSTATUSEX ms; - ms.dwLength = sizeof(ms); - GlobalMemoryStatusEx(&ms); - return(((double) ms.ullTotalPhys)/GB); -} -#undef get_sys_mem -#define get_sys_mem get_mem_GlobalMemoryStatus -#else -static -double -get_mem_sysconf() -{ - return(((double)sysconf(_SC_PHYS_PAGES)) * - ((double)sysconf(_SC_PAGESIZE)/GB)); -} -#undef get_sys_mem -#define get_sys_mem get_mem_sysconf -#endif /* defined(_WIN32) || defined(_WIN64) */ +#ifndef UNIV_HOTBACKUP +/** Stop printing warnings, if the count exceeds this threshold. */ +static const size_t MOVED_FILES_PRINT_THRESHOLD = 32; + +SERVICE_TYPE(registry)* reg_svc = nullptr; +my_h_service h_ret_sysvar_source_svc = nullptr; +SERVICE_TYPE(system_variable_source)* sysvar_source_svc = nullptr; + +static const uint64_t KB = 1024; +static const uint64_t MB = KB * 1024; +static const uint64_t GB = MB * 1024; /** fil_space_t::flags for hard-coded tablespaces */ -ulint predefined_flags; +ulint predefined_flags; /** to protect innobase_open_files */ static mysql_mutex_t innobase_share_mutex; @@ -255,7 +227,7 @@ static char* innobase_enable_monitor_counter = NULL; static char* innobase_disable_monitor_counter = NULL; static char* innobase_reset_monitor_counter = NULL; static char* innobase_reset_all_monitor_counter = NULL; -static char* innobase_scan_directories = NULL; +static char* innobase_directories = NULL; static ulong innodb_flush_method; @@ -284,6 +256,31 @@ enum default_row_format_enum { DEFAULT_ROW_FORMAT_DYNAMIC = 2, }; +#if defined(_WIN32) || defined(_WIN64) +#include +static +double +get_mem_GlobalMemoryStatus() +{ + MEMORYSTATUSEX ms; + ms.dwLength = sizeof(ms); + GlobalMemoryStatusEx(&ms); + return(((double) ms.ullTotalPhys)/GB); +} +#undef get_sys_mem +#define get_sys_mem get_mem_GlobalMemoryStatus +#else +static +double +get_mem_sysconf() +{ + return(((double)sysconf(_SC_PHYS_PAGES)) * + ((double)sysconf(_SC_PAGESIZE)/GB)); +} +#undef get_sys_mem +#define get_sys_mem get_mem_sysconf +#endif /* defined(_WIN32) || defined(_WIN64) */ + static void release_sysvar_source_service() @@ -306,22 +303,26 @@ void acquire_sysvar_source_service() { /* Acquire mysql_server's registry service */ + reg_svc = mysql_plugin_registry_acquire(); - if (!reg_svc) { - goto error; - } /* Acquire system_variable_source service */ - if (reg_svc->acquire("system_variable_source", - &h_ret_sysvar_source_svc)) { - goto error; + + if (!reg_svc + || reg_svc->acquire( + "system_variable_source", &h_ret_sysvar_source_svc)) { + + release_sysvar_source_service(); + + } else { + + /* Type cast this handler to proper service handle */ + + sysvar_source_svc = + reinterpret_cast( + h_ret_sysvar_source_svc); } - /* Type cast this handler to proper service handle */ - sysvar_source_svc = - reinterpret_cast(h_ret_sysvar_source_svc); - return; -error: - release_sysvar_source_service(); } /** Return the InnoDB ROW_FORMAT enum value @@ -386,6 +387,7 @@ static TYPELIB innodb_stats_method_typelib = { }; #endif /* UNIV_HOTBACKUP */ + /** Possible values of the parameter innodb_checksum_algorithm */ static const char* innodb_checksum_algorithm_names[] = { "crc32", @@ -447,9 +449,9 @@ static TYPELIB innodb_default_row_format_typelib = { innodb_default_row_format_names, NULL }; -#endif /* !UNIV_HOTBACKUP */ -#ifdef UNIV_HOTBACKUP +#else /* !UNIV_HOTBACKUP */ + /** Returns the name of the checksum algorithm corresponding to the algorithm id given by "algo_enum" parameter. @param[in] algo_enum algorithm enumerator @@ -484,7 +486,7 @@ meb_get_checksum_algorithm_enum( return(TRUE); } -#endif /* UNIV_HOTBACKUP */ +#endif /* !UNIV_HOTBACKUP */ #ifndef UNIV_HOTBACKUP /* The following counter is used to convey information to InnoDB @@ -549,7 +551,7 @@ srv_debug_loop(void) os_thread_sleep(100); } } -#endif +#endif /* UNIV_DEBUG */ /** Find and Retrieve the FTS Relevance Ranking result for doc with doc_id of m_prebuilt->fts_doc_id @@ -930,7 +932,7 @@ innodb_tmpdir_validate( return(1); } - os_normalize_path(alter_tmp_dir); + Fil_path::normalize(alter_tmp_dir); my_realpath(tmp_abs_path, alter_tmp_dir, 0); size_t tmp_abs_len = strlen(tmp_abs_path); @@ -1325,23 +1327,21 @@ innobase_get_index_column_cardinality( dd::Object_id se_private_id, ulonglong* cardinality); -/** - Retrieve ha_tablespace_statistics for the tablespace. +/** Retrieve ha_tablespace_statistics for the tablespace. - @param tablespace_name Tablespace_name - @param file_name Data file name. - @param ts_se_private_data Tablespace SE private data. - @param[out] stats Contains tablespace - statistics read from SE. - - @returns false on success, true on failure -*/ +@param tablespace_name Tablespace_name +@param file_name Data file name. +@param ts_se_private_data Tablespace SE private data. +@param[out] stats Contains tablespace + statistics read from SE. +@return false on success, true on failure */ static bool -innobase_get_tablespace_statistics(const char *tablespace_name, - const char *file_name, - const dd::Properties &ts_se_private_data, - ha_tablespace_statistics *stats); +innobase_get_tablespace_statistics( + const char *tablespace_name, + const char *file_name, + const dd::Properties &ts_se_private_data, + ha_tablespace_statistics *stats); /** Perform post-commit/rollback cleanup after DDL statement. @param[in,out] thd connection thread */ @@ -1496,8 +1496,7 @@ and the rest of InnoDB have been shut down. @see innodb_shutdown() */ static void -innodb_pre_dd_shutdown( - handlerton*) +innodb_pre_dd_shutdown(handlerton*) { if (innodb_inited) { srv_pre_dd_shutdown(); @@ -1583,9 +1582,9 @@ innobase_fill_i_s_table( Item*, enum_schema_tables idx) { - DBUG_ASSERT(idx == SCH_TABLESPACES); + DBUG_ASSERT(idx == SCH_TABLESPACES); - /** InnoDB does not implement I_S.TABLESPACES */ + /** InnoDB does not implement I_S.TABLESPACES */ return(0); } @@ -3460,112 +3459,393 @@ innobase_dict_init( List* tables, List* tablespaces); -/** Discover all InnoDB tablespaces. -@param[in,out] thd thread handle -@retval true on error -@retval false on success */ -static MY_ATTRIBUTE((warn_unused_result)) -bool -boot_tablespaces(THD* thd) -{ - dd::cache::Dictionary_client* dc - = dd::get_dd_client(thd); - dd::cache::Dictionary_client::Auto_releaser releaser(dc); - std::vector tablespaces; +/** Validate the DD tablespace data against what's read during the +directory scan on startup. */ +class Validate_files { - /* Initialize the max space_id from sys header */ - mutex_enter(&dict_sys->mutex); - mtr_t mtr; - mtr_start(&mtr); - space_id_t max_id = mtr_read_ulint( - dict_hdr_get(&mtr) + DICT_HDR_MAX_SPACE_ID, - MLOG_4BYTES, &mtr); - mtr_commit(&mtr); - fil_set_max_space_id_if_bigger(max_id); - mutex_exit(&dict_sys->mutex); + using Tablespaces = std::vector; + using Const_iter = Tablespaces::const_iterator; - if (dc->fetch_global_components(&tablespaces)) { - return(true); +public: + /** Constructor */ + Validate_files() + : + m_mutex(), + m_space_max_id(), + m_n_threads() +#if !defined(__SUNPRO_CC) + ,m_checked() + ,m_n_errors() +#endif /* !__SUNPRO_CC */ + { +#if defined(__SUNPRO_CC) + m_checked = ATOMIC_VAR_INIT(0); + m_n_errors = ATOMIC_VAR_INIT(0); +#endif /* __SUNPRO_CC */ + } + + /** Validate the tablespaces against the DD. + @param[in] tablespaces Tablespace files read from the DD + @param[out] moved_count Number of tablespaces that have moved + @return DB_SUCCESS if all OK */ + dberr_t validate(const Tablespaces& tablespaces, size_t* moved_count) + MY_ATTRIBUTE((warn_unused_result)); + +private: + /** Validate the tablespace filenames. + @param[in] begin Start of the slice + @param[in] end End of the slice + @param[in] thread_id Thread ID + @param[in] moved_count Number of files that were moved */ + void check( + const Const_iter& begin, + const Const_iter& end, + size_t thread_id, + size_t* moved_count); + + /** @return true if there were failures. */ + bool failed() const + { + return(m_n_errors != 0); } + /** @return the number of tablespaces checked. */ + size_t checked() const + { + return(m_checked); + } + + /** @return the maximum tablespace ID found. */ + space_id_t get_space_max_id() const + { + return(m_space_max_id); + } + +private: + /** Mutex protecting the parallel check. */ + std::mutex m_mutex; + + /** Maximum tablespace ID found. */ + space_id_t m_space_max_id; + + /** Number of threads used in the parallel for. */ + size_t m_n_threads; + + /** Number of tablespaces checked. */ + std::atomic_size_t m_checked; + + /** Number of threads that failed. */ + std::atomic_size_t m_n_errors; +}; + +/** Validate the tablespace filenames. +@param[in] begin Start of the slice +@param[in] end End of the slice +@param[in] thread_id Thread ID +@param[in] moved_count Number of files that were moved */ +void +Validate_files::check( + const Const_iter& begin, + const Const_iter& end, + size_t thread_id, + size_t* moved_count) +{ + const auto sys_space_name = dict_sys_t::s_sys_space_name; + + size_t count = 0; + bool print_msg = false; + auto start_time = ut_time(); + auto heap = mem_heap_create(FN_REFLEN * 2 + 1); + const bool validate = recv_needed_recovery && srv_force_recovery == 0; - bool fail = false; - mem_heap_t* heap = mem_heap_create(FN_REFLEN * 2 + 1); - max_id = 0; - for (const dd::Tablespace* t : tablespaces) { - ut_ad(!fail); + std::string prefix; + + if (m_n_threads > 0) { + std::ostringstream msg; + + msg << "Thread# " << thread_id << " - "; + + prefix = msg.str(); + } + + for (auto it = begin; it != end; ++it, ++m_checked, ++count) { + + const auto& tablespace = *it; + + if (ut_time() - start_time >= PRINT_INTERVAL_SECS) { + + std::ostringstream msg; + + msg << prefix + << "Checked " << count << "/" << (end - begin) + << " tablespaces"; + + if (*moved_count > 0) { + msg << ", moved count " << moved_count; + } + + ib::info() << msg.str(); + + start_time = ut_time(); + + print_msg = true; + } - if (t->engine() != innobase_hton_name) { + if (tablespace->engine() != innobase_hton_name) { continue; } - const dd::Properties& p = t->se_private_data(); - uint32 id; - uint32 flags = 0; + space_id_t space_id; + uint32_t flags = 0; + const auto& p = tablespace->se_private_data(); + const char* space_name = tablespace->name().c_str(); + const auto se_key_value = dd_space_key_strings; /* There should be exactly one file name associated with each InnoDB tablespace, except innodb_system */ - fail = p.get_uint32(dd_space_key_strings[DD_SPACE_ID], &id) - || p.get_uint32(dd_space_key_strings[DD_SPACE_FLAGS], - &flags) - || (t->files().size() != 1 && - strcmp(t->name().c_str(), - dict_sys_t::s_sys_space_name) != 0); - - if (fail) { + + if (p.get_uint32(se_key_value[DD_SPACE_ID], &space_id)) { + /* Failed to fetch the tablespace ID */ + ++m_n_errors; break; + } - if (!dict_sys_t::is_reserved(id) && id > max_id) { - /* Currently try to find the max one only, it should - be able to reuse the deleted smaller ones later */ - max_id = id; + if (p.get_uint32(se_key_value[DD_SPACE_FLAGS], &flags)) { + /* Failed to fetch the tablespace flags. */ + ++m_n_errors; + break; } - const dd::Tablespace_file* f = *t->files().begin(); - fail = f == nullptr; - if (fail) { + if (tablespace->files().size() != 1 + && strcmp(space_name, sys_space_name) != 0) { + + /* Only the InnoDB system tablespace has support for + multiple files per tablespace. For historial reasons. */ + ++m_n_errors; break; } - const char* space_name = t->name().c_str(); - fil_type_t purpose = fsp_is_system_temporary(id) - ? FIL_TYPE_TEMPORARY : FIL_TYPE_TABLESPACE; - const char* filename = f->filename().c_str(); + { + std::lock_guard guard(m_mutex); + + if (!dict_sys_t::is_reserved(space_id) + && space_id > m_space_max_id) { - if (fsp_is_system_or_temp_tablespace(id) - || fsp_is_undo_tablespace(id) - || fil_space_for_table_exists_in_mem( - id, space_name, false, true, heap, 0)) { + /* Currently try to find the max one only, + it should be able to reuse the deleted smaller + ones later */ + m_space_max_id = space_id; + } + } + + /* Non-IBD datafiles are tracked and opened separately. */ + if (!fsp_is_ibd_tablespace(space_id)) { continue; } + /* If this IBD tablespace exists in memory correctly, + we can continue. */ + if (fil_space_exists_in_mem(space_id, space_name, + false, true, heap, 0)) { + continue; + } + + /* Check if any IBD files are moved, deleted or missing. */ + + const auto file = *tablespace->files().begin(); + std::string dd_path{file->filename().c_str()}; + const char* filename = dd_path.c_str(); + std::string new_path; + + /* Just in case this dictionary was ported between + Windows and POSIX. */ + Fil_path::normalize(dd_path); + Fil_state state = Fil_state::MATCHES; + + std::lock_guard guard(m_mutex); + + state = fil_tablespace_path_equals( + tablespace->id(), space_id, space_name, + dd_path, &new_path); + + switch(state) { + case Fil_state::MATCHES: + break; + + case Fil_state::MISSING: + + ib::warn() + << prefix + << "Tablespace " << space_id << "," + << " name '" << space_name << "'," + << " file '" << dd_path << "'" + << " is missing!"; + + continue; + + case Fil_state::DELETED: + + ib::warn() + << prefix + << "Tablespace " << space_id << "," + << " name '" << space_name << "'," + << " file '" << dd_path << "'" + << " was deleted!"; + + continue; + + case Fil_state::MOVED: + + ++*moved_count; + + if (*moved_count > MOVED_FILES_PRINT_THRESHOLD) { + + filename = new_path.c_str(); + + break; + } + + ib::info() + << prefix + << "DD ID: " << tablespace->id() + << " - " + << "Tablespace " << space_id << "," + << " name '" << space_name << "'," + << " file '" << dd_path << "'" + << " has been moved to" + << " '" << new_path << "'"; + + filename = new_path.c_str(); + + if (*moved_count == MOVED_FILES_PRINT_THRESHOLD) { + + ib::info() + << prefix + << "Too many files have" + << " have been moved, disabling" + << " logging of detailed" + << " messages"; + } + + case Fil_state::RENAMED: + break; + } + /* It's safe to pass space_name in tablename charset because filename is already in filename charset. */ dberr_t err = fil_ibd_open( - validate, purpose, id, flags, space_name, - nullptr, filename, false, false); + validate, FIL_TYPE_TABLESPACE, space_id, flags, + space_name, nullptr, filename, false, false); + switch (err) { case DB_SUCCESS: - case DB_CANNOT_OPEN_FILE: break; + case DB_CANNOT_OPEN_FILE: + case DB_WRONG_FILE_NAME: default: - ib::info() << "Unable to open tablespace " << id - << " (flags=" << flags - << ", filename=" << filename << ")." - << " This should be fixed after data" - << " dictionary and DDL recovery later."; - ut_strerr(err); + ib::info() + << prefix + << "Tablespace " << space_id << "," + << " name '" << space_name << "'," + << " unable to open file" + << " '" << filename << "' - " + << ut_strerr(err); } } - fil_set_max_space_id_if_bigger(max_id); + if (!print_msg) { + ib::info() + << prefix + << "Validated " + << count << "/" << (end - begin) << " tablespaces"; + } mem_heap_free(heap); +} - return(fail); +/** Validate the tablespaces against the DD. +@param[in] tablespaces Tablespace files read from the DD +@return DB_SUCCESS if all OK */ +dberr_t +Validate_files::validate(const Tablespaces& tablespaces, size_t* moved_count) +{ + m_n_threads = tablespaces.size() / 50000; + + if (m_n_threads > 8) { + m_n_threads = 8; + } + + using std::placeholders::_1; + using std::placeholders::_2; + using std::placeholders::_3; + using std::placeholders::_4; + + std::function check = std::bind( + &Validate_files::check, this, _1, _2, _3, _4); + + par_for(PFS_NOT_INSTRUMENTED, + tablespaces, m_n_threads, check, moved_count); + + if (failed()) { + return(DB_ERROR); + } + + fil_set_max_space_id_if_bigger(get_space_max_id()); + + return(DB_SUCCESS); +} + +/** Discover all InnoDB tablespaces. +@param[in,out] thd thread handle +@param[out] moved_count Number of files that have been moved +@retval true on error +@retval false on success */ +static MY_ATTRIBUTE((warn_unused_result)) +bool +boot_tablespaces(THD* thd, size_t* moved_count) +{ + auto dc = dd::get_dd_client(thd); + + using DD_tablespaces = std::vector; + using Releaser = dd::cache::Dictionary_client::Auto_releaser; + + DD_tablespaces tablespaces; + Releaser releaser(dc); + + /* Initialize the max space_id from sys header */ + mutex_enter(&dict_sys->mutex); + + mtr_t mtr; + + mtr_start(&mtr); + + space_id_t space_max_id = mtr_read_ulint( + dict_hdr_get(&mtr) + DICT_HDR_MAX_SPACE_ID, MLOG_4BYTES, &mtr); + + mtr_commit(&mtr); + + fil_set_max_space_id_if_bigger(space_max_id); + + mutex_exit(&dict_sys->mutex); + + ib::info() << "Reading DD tablespace files"; + + if (dc->fetch_global_components(&tablespaces)) { + + /* Failed to fetch the tablespaces from the DD. */ + + return(true); + } + + Validate_files validator; + + return(validator.validate(tablespaces, moved_count) != DB_SUCCESS); } /** Create metadata for a predefined tablespace at server initialization. @@ -3661,8 +3941,8 @@ innobase_dict_cache_reset_tables_and_tablespaces() in LRU list */ while (table) { /* Make sure table->is_dd_table is set */ - char db_buf[NAME_LEN + 1]; - char tbl_buf[NAME_LEN + 1]; + char db_buf[NAME_LEN + 1]; + char tbl_buf[NAME_LEN + 1]; dict_table_t* next_table = UT_LIST_GET_NEXT(table_LRU, table); @@ -3704,6 +3984,7 @@ innobase_dict_recover( dict_recovery_mode_t dict_recovery_mode, uint version) { + size_t moved_count = 0; THD* thd = current_thd; switch (dict_recovery_mode) { @@ -3727,8 +4008,8 @@ innobase_dict_recover( thd, NULL, "mysql/innodb_index_stats", false, DICT_ERR_IGNORE_NONE); dict_sys->ddl_log = dd_table_open_on_name( - thd, NULL, "mysql/innodb_ddl_log", - false, DICT_ERR_IGNORE_NONE); + thd, NULL, "mysql/innodb_ddl_log", + false, DICT_ERR_IGNORE_NONE); log_ddl = UT_NEW_NOKEY(Log_DDL()); } @@ -3749,20 +4030,20 @@ innobase_dict_recover( return(true); } - goto success; + break; } case DICT_RECOVERY_RESTART_SERVER: - if (boot_tablespaces(thd)) { + if (boot_tablespaces(thd, &moved_count)) { return(true); } srv_dict_recover_on_restart(); -success: - srv_start_threads( - dict_recovery_mode != DICT_RECOVERY_RESTART_SERVER); + } - return(false); + srv_start_threads(dict_recovery_mode != DICT_RECOVERY_RESTART_SERVER); + + return(fil_open_for_business(srv_read_only_mode) != DB_SUCCESS); } /** DDL crash recovery: process the records recovered from "log_ddl" table */ @@ -3774,6 +4055,8 @@ innobase_post_recover() log_ddl->recover(); } + fil_free_scanned_files(); + if (srv_read_only_mode || srv_force_recovery >= SRV_FORCE_NO_BACKGROUND) { purge_sys->state = PURGE_STATE_DISABLED; @@ -3848,7 +4131,7 @@ static bool innobase_is_supported_system_table( "role_edges", "default_roles", "global_grants", - "password_history"}; + "password_history"}; static const char*const*const end = tables + UT_ARR_SIZE(tables); @@ -3877,13 +4160,13 @@ innobase_encryption_key_rotation() /* Check if keyring loaded and the currently master key can be fetched. */ - if (Encryption::master_key_id != 0) { + if (Encryption::s_master_key_id != 0) { ulint master_key_id; Encryption::Version version; - Encryption::get_master_key(&master_key_id, - &master_key, - &version); + Encryption::get_master_key( + &master_key_id, &master_key, &version); + if (master_key == NULL) { mutex_exit(&master_key_id_mutex); my_error(ER_CANNOT_FIND_KEY_IN_KEYRING, MYF(0)); @@ -3975,33 +4258,30 @@ innodb_buffer_pool_size_init() if (!sysvar_source_svc->get( variable_name, static_cast(strlen(variable_name)), - &source)) - { + &source)) { - /* If innodb_buffer_pool_size is not specified explicitly, - then set it according to server memory as follow: - 1) server_memory < 1 GB then - innodb_buffer_pool_size = default. - 2) server_memory <= 4 GB then - innodb_buffer_pool_size = 50% of server memory. - 2) server_memory > 4 GB then - innodb_buffer_pool_size = 75% of server memory.*/ if (source == COMPILED) { + double server_mem = get_sys_mem(); + if (server_mem < 1.0) { - /* Do nothing. Current default is considered. */ + ; } else if (server_mem <= 4.0) { srv_buf_pool_size = - static_cast(server_mem * 0.5 * GB); + static_cast( + server_mem * 0.5 * GB); } else srv_buf_pool_size = - static_cast(server_mem * 0.75 * GB); + static_cast( + server_mem * 0.75 * GB); } else { - ib::warn() << "Option innodb_dedicated_server" - " is ignored for innodb_buffer_pool_size because" + ib::warn() + << "Option innodb_dedicated_server" + " is ignored for" + " innodb_buffer_pool_size because" " innodb_buffer_pool_size=" << srv_buf_pool_curr_size - <<" is specified explicitly."; + <<" is specified explicitly."; } } } @@ -4092,27 +4372,30 @@ innodb_init_params() current_dir[1] = FN_LIBCHAR; current_dir[2] = 0; default_path = current_dir; - ut_a(default_path); - fil_path_to_mysql_datadir = default_path; - folder_mysql_datadir = default_path; + + std::string mysqld_datadir{default_path}; + + MySQL_datadir_path = Fil_path{mysqld_datadir}; /* Validate, normalize and interpret the InnoDB start-up parameters. */ /* The default dir for data files is the datadir of MySQL */ - srv_data_home = innobase_data_home_dir + srv_data_home = innobase_data_home_dir != nullptr ? innobase_data_home_dir : default_path; + Fil_path::normalize(srv_data_home); if (srv_undo_dir == nullptr) { srv_undo_dir = default_path; } - os_normalize_path(srv_undo_dir); + Fil_path::normalize(srv_undo_dir); /* The default dir for log files is the datadir of MySQL */ if (srv_log_group_home_dir == nullptr) { srv_log_group_home_dir = default_path; } + Fil_path::normalize(srv_log_group_home_dir); if (strchr(srv_log_group_home_dir, ';')) { log_errlog(ERROR_LEVEL, ER_INNODB_INVALID_LOG_GROUP_HOME_DIR); @@ -4149,19 +4432,11 @@ innodb_init_params() if (!sysvar_source_svc->get( variable_name, static_cast(strlen(variable_name)), - &source)) - { - /* If innodb_log_file_size is not specified explicitly, - then set it according to server memory as follow: - 1) server_memory < 1 GB then innodb_log_file_size = default. - 2) server_memory <= 4 GB then innodb_log_file_size = 128 MB. - 3) server_memory <= 8 GB then innodb_log_file_size = 512 MB. - 4) server_memory <= 16 GB then innodb_log_file_size = 1024 MB. - 5) server_memory > 16 GB then innodb_log_file_size = 2048 MB.*/ + &source)) { if (source == COMPILED) { double server_mem = get_sys_mem(); if (server_mem < 1.0) { - /* Do nothing. Current default is considered. */ + ; } else if (server_mem <= 4.0) { innodb_log_file_size = 128ULL * MB; } else if (server_mem <= 8.0) { @@ -4172,11 +4447,12 @@ innodb_init_params() innodb_log_file_size = 2048ULL * MB; } } else { - ib::warn() << "Option innodb_dedicated_server" - " is ignored for innodb_log_file_size because" - " innodb_log_file_size=" + ib::warn() + << "Option innodb_dedicated_server" + " is ignored for innodb_log_file_size" + " because innodb_log_file_size=" << innodb_log_file_size - <<" is specified explicitly."; + <<" is specified explicitly."; } } } @@ -4326,16 +4602,6 @@ innodb_init_params() srv_use_doublewrite_buf = FALSE; } -#ifdef HAVE_LZO1X - if (lzo_init() != LZO_E_OK) { - ib::warn() << "lzo_init() failed, support disabled"; - srv_lzo_disabled = true; - } else { - ib::info() << "LZO1X support available"; - srv_lzo_disabled = false; - } -#endif /* HAVE_LZO1X */ - #ifdef LINUX_NATIVE_AIO if (srv_use_native_aio) { ib::info() << "Using Linux native AIO"; @@ -4350,23 +4616,26 @@ innodb_init_params() #ifndef _WIN32 acquire_sysvar_source_service(); /* Check if innodb_dedicated_server == ON and O_DIRECT is supported */ - if (srv_dedicated_server && - sysvar_source_svc != nullptr && - os_is_o_direct_supported()) { + if (srv_dedicated_server + && sysvar_source_svc != nullptr + && os_is_o_direct_supported()) { + static const char* variable_name = "innodb_flush_method"; enum enum_variable_source source; if (!sysvar_source_svc->get( - variable_name, strlen(variable_name), &source)) - { + variable_name, strlen(variable_name), &source)) { + /* If innodb_flush_method is not specified explicitly */ if (source == COMPILED) { innodb_flush_method = static_cast( SRV_UNIX_O_DIRECT_NO_FSYNC); } else { - ib::warn() << "Option innodb_dedicated_server" - " is ignored for innodb_flush_method because" - " innodb_flush_method=" - << innodb_flush_method_names[innodb_flush_method] + ib::warn() + << "Option innodb_dedicated_server" + " is ignored for innodb_flush_method" + "because innodb_flush_method=" + << innodb_flush_method_names[ + innodb_flush_method] <<" is specified explicitly."; } } @@ -4779,24 +5048,35 @@ dd_open_hardcoded(space_id_t space_id, const char* filename) bool fail = false; fil_space_t* space = fil_space_acquire_silent(space_id); - if (space != NULL) { + if (space != nullptr) { + + /* ADD SDI flag presence in predefined flags of mysql + tablespace. */ + /* The tablespace was already opened up by redo log apply. */ ut_ad(space->flags == predefined_flags); - if (strstr(UT_LIST_GET_FIRST(space->chain)->name, filename) != 0 + + if (strstr(space->files.front().name, filename) != 0 && space->flags == predefined_flags) { + fil_space_open_if_needed(space); + } else { fail = true; } fil_space_release(space); + } else if (fil_ibd_open(true, FIL_TYPE_TABLESPACE, space_id, predefined_flags, dict_sys_t::s_dd_space_name, dict_sys_t::s_dd_space_name, filename, true, false) == DB_SUCCESS) { + /* Set fil_space_t::size, which is 0 initially. */ - fil_space_get_size(space_id); + ulint size = fil_space_get_size(space_id); + ut_a(size != ULINT_UNDEFINED); + } else { fail = true; } @@ -4836,7 +5116,29 @@ innobase_init_files( srv_is_upgrade_mode = (dict_init_mode == DICT_INIT_UPGRADE_FILES); - err = srv_start(create, innobase_scan_directories); + /* InnoDB files should be found in the following locations only. */ + std::string directories; + + if (innobase_directories != nullptr && *innobase_directories != 0) { + + Fil_path::normalize(innobase_directories); + directories.append(Fil_path::parse(innobase_directories)); + directories.push_back(FIL_PATH_SEPARATOR); + } + + directories.append(srv_data_home); + + if (srv_undo_dir != nullptr && *srv_undo_dir != 0) { + + directories.push_back(FIL_PATH_SEPARATOR); + directories.append(srv_undo_dir); + } + + /* This is the default directory for .ibd files. */ + directories.push_back(FIL_PATH_SEPARATOR); + directories.append(MySQL_datadir_path.path()); + + err = srv_start(create, directories); if (err != DB_SUCCESS) { DBUG_RETURN(innodb_init_abort()); @@ -6651,9 +6953,6 @@ ha_innobase::innobase_initialize_autoinc() dict_table_autoinc_initialize(m_prebuilt->table, auto_inc); } -/** partition separator */ -extern const char* part_sep; - /** Open an InnoDB table. @param[in] name table name @param[in] open_flags flags for opening table from SQL-layer. @@ -6693,7 +6992,7 @@ ha_innobase::open( /* We look for pattern #P# to see if the table is partitioned MySQL table. */ - is_part = strstr(norm_name, part_sep); + is_part = strstr(norm_name, PART_SEPARATOR); /* Get pointer to a table object in InnoDB dictionary cache. For intrinsic table, get it from session private data */ @@ -10730,7 +11029,7 @@ create_table_info_t::create_table_def( } if (DICT_TF_HAS_DATA_DIR(m_flags)) { - ut_a(strlen(m_remote_path)); + ut_a(strlen(m_remote_path) != 0); table->data_dir_path = mem_heap_strdup( table->heap, m_remote_path); @@ -11121,13 +11420,13 @@ create_table_info_t::create_table_def( template const dd::Index* get_my_dd_index(const Index* index); template<> const dd::Index* get_my_dd_index(const dd::Index* dd_index) { - return dd_index; + return dd_index; } template<> const dd::Index* get_my_dd_index( - const dd::Partition_index* dd_index) { - return (dd_index != nullptr) ? &dd_index->index() : nullptr; + const dd::Partition_index* dd_index) { + return (dd_index != nullptr) ? &dd_index->index() : nullptr; } /*****************************************************************//** @@ -11179,7 +11478,7 @@ create_index( const auto* dd_index_auto = dd_table->indexes()[dd_index_num]; - const dd::Index* dd_index = get_my_dd_index(dd_index_auto); + const dd::Index* dd_index = get_my_dd_index(dd_index_auto); ut_ad(dd_index->name() == key->name); size_t geom_col_idx; @@ -11410,8 +11709,8 @@ create_table_info_t::create_option_data_directory_is_valid() { bool is_valid = true; - ut_ad(m_create_info->data_file_name - && m_create_info->data_file_name[0] != '\0'); + ut_ad(m_create_info->data_file_name != nullptr + && *m_create_info->data_file_name != '\0'); /* Use DATA DIRECTORY only with file-per-table. */ if (!m_use_shared_space && !m_allow_file_per_table) { @@ -11433,8 +11732,36 @@ create_table_info_t::create_option_data_directory_is_valid() is_valid = false; } - /* We check for a DATA DIRECTORY mixed with TABLESPACE in - create_option_tablespace_is_valid(), no need to here. */ + /* We checked previously for a conflicting DATA DIRECTORY mixed + with TABLESPACE in create_option_tablespace_is_valid(). + An ALTER TABLE statement might have both if it is being moved. + So if m_tablespace is set, don't check the existing data_file_name. */ + if (m_create_info->tablespace != nullptr) { + return(is_valid); + } + +#ifndef UNIV_HOTBACKUP + /* Do not allow a datafile outside the known directories. */ + char* file_path = Fil_path::make( + m_create_info->data_file_name, m_table_name, IBD, true); + + if (!Fil_path::is_valid_location(m_table_name, file_path)) { + + push_warning( + m_thd, Sql_condition::SL_WARNING, + ER_ILLEGAL_HA_CREATE_OPTION, + "InnoDB: DATA DIRECTORY is not in a valid location." + " It is not found in innodb_directories."); + + ib::error() << "Cannot create table " << m_table_name + << " in directory " << file_path + << " because it is not in a valid location."; + + is_valid = false; + } + + ut_free(file_path); +#endif /* UNIV_HOTBACKUP */ return(is_valid); } @@ -11451,11 +11778,12 @@ validate_tablespace_name( { int err = 0; - /* This prefix is reserved by InnoDB for use in internal tablespace names. */ + /* This prefix is reserved by InnoDB for use in internal tablespace + names. */ const char reserved_space_name_prefix[] = "innodb_"; - /* Validation at the SQL layer should already be completed at this stage. - Re-assert that the length is valid. */ + /* Validation at the SQL layer should already be completed at this + stage. Re-assert that the length is valid. */ ut_ad(!validate_tablespace_name_length(name)); /* The tablespace name cannot start with `innodb_`. */ @@ -11561,10 +11889,13 @@ create_table_info_t::create_option_tablespace_is_valid() needed for CREATE TABLE only. ALTER TABLE may be moving remote file-per-table table to a general tablespace, in which case the create_info->data_file_name is not null. */ - bool is_create_table = (thd_sql_command(m_thd) == SQLCOM_CREATE_TABLE); + bool is_create_table; + + is_create_table = (thd_sql_command(m_thd) == SQLCOM_CREATE_TABLE); + if (is_create_table - && m_create_info->data_file_name != NULL - && m_create_info->data_file_name[0] != '\0') { + && m_create_info->data_file_name != nullptr + && *m_create_info->data_file_name != '\0') { my_printf_error(ER_ILLEGAL_HA_CREATE_OPTION, "InnoDB: DATA DIRECTORY cannot be used" " with a TABLESPACE assignment.", MYF(0)); @@ -11854,8 +12185,9 @@ create_table_info_t::create_options_are_invalid() break; } - if (m_create_info->data_file_name - && m_create_info->data_file_name[0] != '\0' + if (m_create_info->data_file_name != nullptr + && *m_create_info->data_file_name != '\0' + && m_table_name != nullptr && !create_option_data_directory_is_valid()) { ret = "DATA DIRECTORY"; } @@ -11918,9 +12250,11 @@ ha_innobase::update_create_info( } /* Update the DATA DIRECTORY name. */ - dd_get_and_save_data_dir_path(m_prebuilt->table, NULL, false); + dd_get_and_save_data_dir_path( + m_prebuilt->table, NULL, false); + + if (m_prebuilt->table->data_dir_path != nullptr) { - if (m_prebuilt->table->data_dir_path) { create_info->data_file_name = m_prebuilt->table->data_dir_path; } @@ -12017,7 +12351,7 @@ innobase_dict_init( " ENGINE=INNODB ROW_FORMAT=DYNAMIC " "DEFAULT CHARSET=utf8 COLLATE=utf8_bin " "STATS_PERSISTENT=0", - /* Tablespace */ + /* Tablespace */ MYSQL_TABLESPACE_NAME.str); static Plugin_table innodb_index_stats( @@ -12048,8 +12382,8 @@ innobase_dict_init( " ENGINE=INNODB ROW_FORMAT=DYNAMIC " "DEFAULT CHARSET=utf8 COLLATE=utf8_bin " "STATS_PERSISTENT=0", - /* Tablespace */ - MYSQL_TABLESPACE_NAME.str); + /* Tablespace */ + MYSQL_TABLESPACE_NAME.str); static const Plugin_table innodb_dynamic_metadata( /* Schema Name */ @@ -12063,8 +12397,8 @@ innobase_dict_init( /* Options */ " ENGINE=INNODB ROW_FORMAT=DYNAMIC " " STATS_PERSISTENT=0", - /* Tablespace */ - MYSQL_TABLESPACE_NAME.str); + /* Tablespace */ + MYSQL_TABLESPACE_NAME.str); static const Plugin_table innodb_ddl_log( /* Schema Name */ @@ -12120,26 +12454,25 @@ create_table_info_t::parse_table_name( if (m_innodb_file_per_table && !(m_create_info->options & HA_LEX_CREATE_TMP_TABLE)) { - if ((name[1] == ':') + if (name[1] == ':' || (name[0] == '\\' && name[1] == '\\')) { log_errlog(ERROR_LEVEL, ER_INNODB_CANNOT_CREATE_TABLE, name); DBUG_RETURN(HA_ERR_GENERIC); } } -#endif +#endif /* _WIN32 */ normalize_table_name(m_table_name, name); m_remote_path[0] = '\0'; m_tablespace[0] = '\0'; - /* Make sure DATA DIRECTORY is compatible with other options - and set the remote path. In the case of either; - CREATE TEMPORARY TABLE ... DATA DIRECTORY={path} ... ; - CREATE TABLE ... DATA DIRECTORY={path} TABLESPACE={name}... ; - we ignore the DATA DIRECTORY. */ - if (m_create_info->data_file_name - && m_create_info->data_file_name[0] != '\0') { + /* Set the remote path if DATA DIRECTORY is valid. If not, + we ignore the DATA DIRECTORY. In strict mode, a non-valid + value would have already been rejected. */ + if (m_create_info->data_file_name != nullptr + && *m_create_info->data_file_name != '\0' + && m_table_name != nullptr) { if (!create_option_data_directory_is_valid()) { push_warning_printf( m_thd, Sql_condition::SL_WARNING, @@ -12148,6 +12481,7 @@ create_table_info_t::parse_table_name( "DATA DIRECTORY"); m_flags &= ~DICT_TF_MASK_DATA_DIR; } else { + strncpy(m_remote_path, m_create_info->data_file_name, FN_REFLEN - 1); @@ -12166,6 +12500,7 @@ create_table_info_t::parse_table_name( create_option_tablespace_is_valid() irregardless of strict-mode. So it only needs to be copied now. */ if (m_use_shared_space) { + strncpy(m_tablespace, m_create_info->tablespace, NAME_LEN - 1); } @@ -12794,8 +13129,6 @@ create_table_info_t::prepare_create_table( DBUG_ENTER("prepare_create_table"); ut_ad(m_thd != NULL); - ut_ad(m_create_info != NULL); - ut_ad(m_form->s->row_type == m_create_info->row_type); normalize_table_name(m_table_name, name); @@ -13365,7 +13698,6 @@ innobase_basic_ddl::create_impl( ulint old_flags, ulint old_flags2) { - int error; char norm_name[FN_REFLEN]; /* {database}/{tablename} */ char remote_path[FN_REFLEN]; /* Absolute path of table */ char tablespace[NAME_LEN]; /* Tablespace name identifier */ @@ -13379,24 +13711,22 @@ innobase_basic_ddl::create_impl( trx_start_if_not_started(trx, true); } - create_table_info_t info(thd, - form, - create_info, - norm_name, - remote_path, - tablespace, - file_per_table, - skip_strict, - old_flags, - old_flags2); + create_table_info_t info( + thd, form, create_info, norm_name, remote_path, + tablespace, file_per_table, skip_strict, old_flags, + old_flags2); /* Initialize the object. */ - if ((error = info.initialize())) { + int error = info.initialize(); + + if (error != 0) { return(error); } /* Prepare for create and validate options. */ - if ((error = info.prepare_create_table(name))) { + error = info.prepare_create_table(name); + + if (error != 0) { return(error); } @@ -13723,14 +14053,11 @@ innobase_basic_ddl::rename_impl( row_mysql_unlock_data_dictionary(trx); if (error == DB_SUCCESS && rename_file) { - char* new_path = fil_space_get_first_path(space); + char* new_path = fil_space_get_first_path(space); - dd::Object_id dd_space_id = - dd_first_index(to_table)->tablespace_id(); + auto dd_space_id = dd_first_index(to_table)->tablespace_id(); - if (dd_rename_tablespace(dd_space_id, norm_to, new_path)) { - error = DB_ERROR; - } + error = dd_rename_tablespace(dd_space_id, norm_to, new_path); if (new_path != nullptr) { ut_free(new_path); @@ -14317,20 +14644,28 @@ ha_innobase::truncate_rename_tablespace( dict_sys->size += new_size - old_size; mutex_exit(&dict_sys->mutex); - char* new_path = nullptr; char* old_path = nullptr; old_path = fil_space_get_first_path(table->space); + std::string new_path; + if (DICT_TF_HAS_DATA_DIR(table->flags)) { - new_path = os_file_make_new_pathname(old_path, temp_name); + + new_path = Fil_path::make_new_ibd(old_path, temp_name); + } else { - new_path = fil_make_filepath(nullptr, temp_name, IBD, false); + char* ptr = Fil_path::make_ibd_from_table_name(temp_name); + + new_path.assign(ptr); + + ut_free(ptr); } /* New filepath must not exist. */ dberr_t err = fil_rename_tablespace_check( - table->space, old_path, new_path, false); + table->space, old_path, new_path.c_str(), false); + if (err == DB_SUCCESS) { mutex_enter(&dict_sys->mutex); @@ -14338,7 +14673,7 @@ ha_innobase::truncate_rename_tablespace( clone_mark_abort(true); bool success = fil_rename_tablespace( - table->space, old_path, temp_name, new_path); + table->space, old_path, temp_name, new_path.c_str()); clone_mark_active(); @@ -14354,7 +14689,6 @@ ha_innobase::truncate_rename_tablespace( } ut_free(old_path); - ut_free(new_path); dd_table_close(table, nullptr, nullptr, false); @@ -14424,9 +14758,9 @@ ha_innobase::truncate(dd::Table *table_def) info.tablespace = tsname; info.key_block_size = table->s->key_block_size; - char* data_file_name = m_prebuilt->table->data_dir_path; + char* data_file_name = m_prebuilt->table->data_dir_path; - if (data_file_name) { + if (data_file_name != nullptr) { info.data_file_name = data_file_name = mem_strdup(data_file_name); } @@ -14569,11 +14903,9 @@ validate_create_tablespace_info( THD* thd, st_alter_tablespace* alter_info) { - space_id_t space_id; - /* The parser ensures that these fields are provided. */ - ut_a(alter_info->tablespace_name); - ut_a(alter_info->data_file_name); + ut_a(alter_info->data_file_name != nullptr); + ut_a(alter_info->tablespace_name != nullptr); if (high_level_read_only) { return(HA_ERR_INNODB_READ_ONLY); @@ -14588,7 +14920,11 @@ validate_create_tablespace_info( int error = 0; /* Make sure the tablespace is not already open. */ + + space_id_t space_id; + space_id = fil_space_get_id_by_name(alter_info->tablespace_name); + if (space_id != SPACE_UNKNOWN) { my_printf_error(ER_TABLESPACE_EXISTS, "InnoDB: A tablespace named `%s`" @@ -14630,82 +14966,94 @@ validate_create_tablespace_info( } /* Validate the ADD DATAFILE name. */ - char* filepath = mem_strdup(alter_info->data_file_name); - os_normalize_path(filepath); + Fil_path filepath{alter_info->data_file_name}; /* It must end with '.ibd' and contain a basename of at least 1 character before the.ibd extension. */ - ulint dirname_len = dirname_length(filepath); - const char* basename = filepath + dirname_len; - ulint basename_len = strlen(basename); - if (basename_len < 5) { - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - ut_free(filepath); - return(HA_WRONG_CREATE_OPTION); - } - if (memcmp(&basename[basename_len - 4], DOT_IBD, 5)) { - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - my_printf_error(ER_WRONG_FILE_NAME, + + ulint dirname_len = dirname_length(filepath); + const char* basename = filepath + dirname_len; + auto basename_len = strlen(basename); + + if (basename_len <= 4 || !Fil_path::has_ibd_suffix(basename)) { + + if (basename_len <= 4) { + + my_error(ER_WRONG_FILE_NAME, + MYF(0), filepath.path().c_str()); + } else { + + my_printf_error( + ER_WRONG_FILE_NAME, "An IBD filepath must end with `.ibd`.", MYF(0)); - ut_free(filepath); + } + return(HA_WRONG_CREATE_OPTION); } - /* Do not allow an invalid colon in the file name. */ - const char* colon = strchr(filepath, ':'); - if (colon != NULL) { -#ifdef _WIN32 - /* Do not allow names like "C:name.ibd" because it - specifies the "C:" drive but allows a relative location. - It should be like "c:\". If a single colon is used it must - be the second byte the the third byte must be a separator. */ - if (colon != &filepath[1] - || (colon[1] != OS_PATH_SEPARATOR) - || NULL != strchr(&colon[1], ':')) { -#endif /* _WIN32 */ - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - my_printf_error(ER_WRONG_FILE_NAME, - "Invalid use of ':'.", MYF(0)); - ut_free(filepath); - return(HA_WRONG_CREATE_OPTION); -#ifdef _WIN32 - } -#endif /* _WIN32 */ + if (!filepath.is_valid()) { + + my_error(ER_WRONG_FILE_NAME, MYF(0), filepath.path().c_str()); + + my_printf_error( + ER_WRONG_FILE_NAME, "Invalid use of ':'.", MYF(0)); + + return(HA_WRONG_CREATE_OPTION); } #ifndef _WIN32 /* On Non-Windows platforms, '\\' is a valid file name character. But for InnoDB datafiles, we always assume it is a directory separator and convert these to '/' */ - if (strchr(alter_info->data_file_name, '\\') != NULL) { - ib::warn() << "Converting backslash to forward slash in" - " ADD DATAFILE " << alter_info->data_file_name; + if (strchr(filepath, '\\') != nullptr) { + + ib::warn() + << "Converting backslash to forward slash in" + " ADD DATAFILE " << filepath.path(); } #endif /* _WIN32 */ - /* The directory path must be pre-existing. */ - Folder folder(filepath, dirname_len); - ut_free(filepath); - if (!folder.exists()) { - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - my_printf_error(ER_WRONG_FILE_NAME, - "The directory does not exist.", MYF(0)); + Fil_path dirpath{alter_info->data_file_name, dirname_len}; + + if (dirpath.len() > 0 && !dirpath.is_directory_and_exists()) { + + my_error(ER_WRONG_FILE_NAME, MYF(0), filepath.path().c_str()); + + my_printf_error( + ER_WRONG_FILE_NAME, + "The directory does not exist.", MYF(0)); + return(HA_WRONG_CREATE_OPTION); } + /* CREATE TABLESPACE...ADD DATAFILE must be under a path that InnoDB + knows about. */ + if (dirpath.len() > 0 && !fil_check_path(dirpath.path())) { + + std::string paths = fil_get_dirs(); + + my_error(ER_WRONG_FILE_NAME, MYF(0), filepath.path().c_str()); + + my_printf_error( + ER_WRONG_FILE_NAME, + "CREATE TABLESPACE data file must be in one of" + " these directories '%s'.", MYF(0), paths.c_str()); + + error = HA_WRONG_CREATE_OPTION; + } + /* CREATE TABLESPACE...ADD DATAFILE can be inside but not under the datadir.*/ - if (folder_mysql_datadir > folder) { - my_error(ER_WRONG_FILE_NAME, MYF(0), - alter_info->data_file_name); - my_printf_error(ER_WRONG_FILE_NAME, - "CREATE TABLESPACE data file" - " cannot be under the datadir.", MYF(0)); + if (MySQL_datadir_path.is_ancestor(dirpath)) { + + my_error(ER_WRONG_FILE_NAME, MYF(0), filepath.path().c_str()); + + my_printf_error( + ER_WRONG_FILE_NAME, + "CREATE TABLESPACE data file cannot be under the" + " datadir.", MYF(0)); + error = HA_WRONG_CREATE_OPTION; } @@ -14898,7 +15246,7 @@ innobase_alter_tablespace( break; case DROP_TABLESPACE: - ut_ad(old_ts_def != NULL); + ut_ad(old_ts_def != NULL); error = innobase_drop_tablespace(hton, thd, alter_info, old_ts_def); break; @@ -14906,7 +15254,7 @@ innobase_alter_tablespace( case ALTER_TABLESPACE: if (alter_info->ts_alter_tablespace_type == ALTER_TABLESPACE_RENAME) - { + { /* Only support rename tablespace at the time being */ from = old_ts_def->name().c_str(); to = new_ts_def->name().c_str(); @@ -16184,7 +16532,7 @@ innodb_get_table_statistics_for_uncached( const char* norm_name, dd::Object_id se_private_id, const dd::Properties& ts_se_private_data, - const dd::Properties& tbl_se_private_data, + const dd::Properties& tbl_se_private_data, ulint flags, ha_statistics* stats) { @@ -16307,11 +16655,11 @@ innobase_get_table_statistics( if (innodb_get_table_statistics_for_uncached( db_name, table_name, - norm_name, + norm_name, se_private_id, ts_se_private_data, - tbl_se_private_data, - flags, stats)) { + tbl_se_private_data, + flags, stats)) { return(false); } @@ -16461,146 +16809,146 @@ innobase_get_index_column_cardinality( /** Retrieve ha_tablespace_statistics for the tablespace */ static bool -innobase_get_tablespace_statistics(const char *tablespace_name, - const char *file_name, - const dd::Properties &ts_se_private_data, - ha_tablespace_statistics *stats) -{ - /** Tablespace does not have space id stored. */ - if (!ts_se_private_data.exists(dd_space_key_strings[DD_SPACE_ID])) - { - my_error(ER_TABLESPACE_MISSING, MYF(0), tablespace_name); - return(true); - } +innobase_get_tablespace_statistics( + const char* tablespace_name, + const char* file_name, + const dd::Properties& ts_se_private_data, + ha_tablespace_statistics* stats) +{ + /* Tablespace does not have space id stored. */ + if (!ts_se_private_data.exists(dd_space_key_strings[DD_SPACE_ID])) { + my_error(ER_TABLESPACE_MISSING, MYF(0), tablespace_name); + return(true); + } space_id_t space_id; - ts_se_private_data.get_uint32( - dd_space_key_strings[DD_SPACE_ID], - &space_id); - fil_space_t* space; - space = fil_space_acquire(space_id); + ts_se_private_data.get_uint32( + dd_space_key_strings[DD_SPACE_ID], &space_id); - /** Tablespace is missing in this case. */ + auto space = fil_space_acquire(space_id); + + /* Tablespace is missing in this case. */ if (space == nullptr) { - my_error(ER_TABLESPACE_MISSING, MYF(0), tablespace_name); - return(true); - } - - /** Store space id */ - stats->m_id= space->id; - - /** Store space type */ - const char* type = "TABLESPACE"; - fil_type_t purpose = space->purpose; - switch (purpose) { - case FIL_TYPE_LOG: - /* Do not report REDO LOGs to I_S.FILES */ - space = nullptr; - return(false); - case FIL_TYPE_TABLESPACE: - if (fsp_is_undo_tablespace(space->id)) { - type = "UNDO LOG"; - break; - } /* else fall through for TABLESPACE */ - case FIL_TYPE_IMPORT: - /* 'IMPORTING'is a status. The type is TABLESPACE. */ - break; - case FIL_TYPE_TEMPORARY: - type = "TEMPORARY"; - break; - }; - stats->m_type= type; - - /** Store free_extents */ - stats->m_free_extents= space->free_len; - - /** Store total extents */ - page_no_t extent_pages; - page_size_t page_size(space->flags); - extent_pages = fsp_get_extent_size_in_pages(page_size); - stats->m_total_extents= space->size_in_header / extent_pages; - - /** Store extent size. */ - stats->m_extent_size= extent_pages * page_size.physical(); - - /** Store initial size */ - - /** Pick correct file_node_t as per the file name */ - fil_node_t* node = nullptr; - if (UT_LIST_GET_LEN(space->chain) == 1) - { - node = UT_LIST_GET_FIRST(space->chain); - } - else /** In case a single tablespace has multiple data files */ - { - char tmp_file_name[OS_FILE_MAX_PATH]; - strcpy(tmp_file_name, file_name); - os_normalize_path(tmp_file_name); - - for (node = UT_LIST_GET_FIRST(space->chain); - node != nullptr; - node = UT_LIST_GET_NEXT(chain, node)) { - - /** Skip ./ .// .\\ path. - Fix for Bug#26518545 might change below code. - Mainly because the difference in DD not - containing './' and node->name here containing - './' might change after the fix. - */ - const char *file_name_ptr1= node->name; - const char *file_name_ptr2= tmp_file_name; - if (*file_name_ptr1 == '.') { - file_name_ptr1++; - while (*file_name_ptr1 == OS_PATH_SEPARATOR || - *file_name_ptr1 == OS_PATH_SEPARATOR_ALT) - file_name_ptr1++; - } - if (*file_name_ptr2 == '.') { - file_name_ptr2++; - while (*file_name_ptr2 == OS_PATH_SEPARATOR || - *file_name_ptr2 == OS_PATH_SEPARATOR_ALT) - file_name_ptr2++; - } + my_error(ER_TABLESPACE_MISSING, MYF(0), tablespace_name); + return(true); + } - /** Compare the file name */ - if (!strcmp(file_name_ptr1, file_name_ptr2)) - break; + stats->m_id = space->id; + + const char* type = "TABLESPACE"; + + fil_type_t purpose = space->purpose; + + switch (purpose) { + case FIL_TYPE_LOG: + /* Do not report REDO LOGs to I_S.FILES */ + space = nullptr; + return(false); + case FIL_TYPE_TABLESPACE: + if (fsp_is_undo_tablespace(space->id)) { + type = "UNDO LOG"; + break; + } /* else fall through for TABLESPACE */ + case FIL_TYPE_IMPORT: + /* 'IMPORTING'is a status. The type is TABLESPACE. */ + break; + case FIL_TYPE_TEMPORARY: + type = "TEMPORARY"; + break; + } + + stats->m_type = type; + + stats->m_free_extents = space->free_len; + + page_size_t page_size{space->flags}; + + page_no_t extent_pages = fsp_get_extent_size_in_pages(page_size); + + stats->m_total_extents = space->size_in_header / extent_pages; + + stats->m_extent_size = extent_pages * page_size.physical(); + + const fil_node_t* file = nullptr; + + /* Find the fil_node_t that matches the filename. */ + for (const auto& f : space->files) { + + char name[OS_FILE_MAX_PATH + 1]; + + strncpy(name, file_name, sizeof(name) - 1); + + /* Ensure that it's always NUL terminated. */ + name[OS_FILE_MAX_PATH] = 0; + + Fil_path::normalize(name); + + if (Fil_path::equal(f.name, name)) { + + file = &f; + break; + + } else if (space->files.size() == 1) { + + file = &f; + + ib::info() + << "Tablespace '" << tablespace_name << "'" + << " DD filename '" << name << "' doesn't " + << " match the InnoDB filename '" + << f.name << "'"; } - } - ut_a(node); - stats->m_initial_size= node->init_size * page_size.physical(); + } - /** Store maximum size */ - if (node->max_size >= PAGE_NO_MAX) { - stats->m_maximum_size= -1; - } else { - stats->m_maximum_size= node->max_size * page_size.physical(); - } + if (file == nullptr) { - /** Store autoextend size */ - page_no_t extend_pages; - if (space->id == TRX_SYS_SPACE) { - extend_pages = srv_sys_space.get_increment(); - } else if (fsp_is_system_temporary(space->id)) { - extend_pages = srv_tmp_space.get_increment(); - } else { - extend_pages = fsp_get_pages_to_extend_ibd( - page_size, node->size); - } - stats->m_autoextend_size= extend_pages * page_size.physical(); + ib::warn() + << "Tablespace '" << tablespace_name << "'" + << " filename is unknown. Use --innodb-directories" + << " to locate of the file."; + + my_error(ER_TABLESPACE_MISSING, MYF(0), tablespace_name); + + return(true); + } + + stats->m_initial_size = file->init_size * page_size.physical(); + + /** Store maximum size */ + if (file->max_size >= PAGE_NO_MAX) { + stats->m_maximum_size = -1; + } else { + stats->m_maximum_size = file->max_size * page_size.physical(); + } + + /** Store autoextend size */ + page_no_t extend_pages; + + if (space->id == TRX_SYS_SPACE) { - /** Store data free */ - uintmax_t avail_space; - avail_space = fsp_get_available_space_in_free_extents(space); - stats->m_data_free= avail_space * 1024; + extend_pages = srv_sys_space.get_increment(); - stats->m_status= ((purpose == FIL_TYPE_IMPORT) - ? "IMPORTING" : "NORMAL"); + } else if (fsp_is_system_temporary(space->id)) { + + extend_pages = srv_tmp_space.get_increment(); + + } else { + extend_pages = fsp_get_pages_to_extend_ibd( + page_size, file->size); + } + + stats->m_autoextend_size = extend_pages * page_size.physical(); + + auto avail_space = fsp_get_available_space_in_free_extents(space); + + stats->m_data_free = avail_space * 1024; + + stats->m_status = (purpose == FIL_TYPE_IMPORT ? "IMPORTING" : "NORMAL"); fil_space_release(space); - return(false); + return(false); } @@ -16670,7 +17018,7 @@ ha_innobase::disable_indexes( return(error); } -/* +/** Updates index cardinalities of the table, based on random dives into each index tree. This does NOT calculate exact statistics on the table. @return HA_ADMIN_* error code or HA_ADMIN_OK */ @@ -18272,7 +18620,7 @@ struct ShowStatus { spins=N,waits=N,calls=N" - The user has to parse the dataunfortunately + The user has to parse the data unfortunately @param[in,out] hton the innodb handlerton @param[in,out] thd the MySQL query thread of the caller @param[in,out] stat_print function for printing statistics @@ -18284,7 +18632,7 @@ struct ShowStatus { UNIV_NOTHROW; /** For collecting the active mutex stats. */ - Values m_values; + Values m_values; }; /** Implements the SHOW MUTEX STATUS command, for mutexes. @@ -18293,7 +18641,7 @@ We store the metrics in the "Status" column as: spins=N,waits=N,calls=N" -The user has to parse the dataunfortunately +The user has to parse the data unfortunately @param[in,out] hton the innodb handlerton @param[in,out] thd the MySQL query thread of the caller @param[in,out] stat_print function for printing statistics @@ -20872,7 +21220,9 @@ checkpoint_now_set( fil_flush_file_spaces(to_int(FIL_TYPE_LOG)); } - fil_write_flushed_lsn(log_sys->lsn); + dberr_t err = fil_write_flushed_lsn(log_sys->lsn); + + ut_a(err == DB_SUCCESS); } } @@ -22049,10 +22399,10 @@ static MYSQL_SYSVAR_BOOL(ddl_log_crash_reset_debug, NULL, ddl_log_crash_reset, FALSE); #endif /* UNIV_DEBUG */ -static MYSQL_SYSVAR_STR(scan_directories, innobase_scan_directories, +static MYSQL_SYSVAR_STR(directories, innobase_directories, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY | PLUGIN_VAR_NOPERSIST, - "List of directories to scan for missing tablespace files.", - NULL, NULL, ""); + "List of directories 'dir1;dir2;..;dirN' to scan for tablespace files. Default is to scan 'innodb-data-home-dir;innodb-undo-directory;datadir'", + NULL, NULL, NULL); static SYS_VAR* innobase_system_variables[]= { MYSQL_SYSVAR(api_trx_level), @@ -22148,7 +22498,7 @@ static SYS_VAR* innobase_system_variables[]= { MYSQL_SYSVAR(strict_mode), MYSQL_SYSVAR(sort_buffer_size), MYSQL_SYSVAR(online_alter_log_max_size), - MYSQL_SYSVAR(scan_directories), + MYSQL_SYSVAR(directories), MYSQL_SYSVAR(sync_spin_loops), MYSQL_SYSVAR(spin_wait_delay), MYSQL_SYSVAR(table_locks), @@ -22999,6 +23349,7 @@ innobase_convert_to_filename_charset( cs_from, from, cs_to, to, static_cast(len), &errors))); } + /********************************************************************** Converts an identifier from my_charset_filename to UTF-8 charset. @return result string length, as returned by strconvert() */ diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index 10ce5b4606be..4dc7fb421d89 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -726,15 +726,6 @@ trx_t* innobase_trx_allocate( MYSQL_THD thd); /*!< in: user thread handle */ -/** Gets the InnoDB transaction handle for a MySQL handler object, creates -an InnoDB transaction struct if the corresponding MySQL thread struct still -lacks one. -@param[in] thd MySQL thd (connection) object -@return InnoDB transaction handle */ -trx_t* -check_trx_exists( - THD* thd); - /** Match index columns between MySQL and InnoDB. This function checks whether the index column information is consistent between KEY info from mysql and that from innodb index. @@ -1232,12 +1223,6 @@ page_cur_mode_t convert_search_mode_to_innobase( enum ha_rkey_function find_flag); -/** Commits a transaction in an InnoDB database. -@param[in] trx Transaction handle. */ -void -innobase_commit_low( - trx_t* trx); - extern bool innobase_stats_on_metadata; /** Calculate Record Per Key value. diff --git a/storage/innobase/handler/ha_innopart.cc b/storage/innobase/handler/ha_innopart.cc index 14c3e0a02c94..06c5f0ee4393 100644 --- a/storage/innobase/handler/ha_innopart.cc +++ b/storage/innobase/handler/ha_innopart.cc @@ -69,13 +69,6 @@ Created Nov 22, 2013 Mattias Jonsson */ #include "ut0ut.h" /* To be backwards compatible we also fold partition separator on windows. */ -#ifdef _WIN32 -const char* part_sep = "#p#"; -const char* sub_sep = "#sp#"; -#else -const char* part_sep = "#P#"; -const char* sub_sep = "#SP#"; -#endif /* _WIN32 */ Ha_innopart_share::Ha_innopart_share( TABLE_SHARE* table_share) @@ -159,18 +152,19 @@ Ha_innopart_share::create_partition_postfix( part_name_len = append_sep_and_name( partition_name, part->name().c_str(), - part_sep, size); + PART_SEPARATOR, size); if (part_name_len < size && part != dd_part) { char* part_name_end = partition_name + part_name_len; subpart_name_len = append_sep_and_name( part_name_end, dd_part->name().c_str(), - sub_sep, size - part_name_len); + SUB_PART_SEPARATOR, size - part_name_len); + + if (subpart_name_len >= SUB_PART_SEPARATOR_LEN) { - if (subpart_name_len >= strlen(sub_sep)) { partition_name_casedn_str( - part_name_end + strlen(sub_sep)); + part_name_end + SUB_PART_SEPARATOR_LEN); } } @@ -2441,8 +2435,8 @@ ha_innopart::update_part_elem( if (part_elem->tablespace_name != NULL) { if (0 != strcmp(part_elem->tablespace_name, tablespace_name)) { - /* Update part_elem ablespace to NULL same as in - innodb data dictionary ib_table. */ + /* Update part_elem tablespace to NULL same + as in innodb data dictionary ib_table. */ part_elem->tablespace_name = NULL; } } else if (display_tablespace) { diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 9c36a051cc3e..bf051dbfe3e4 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -72,28 +72,28 @@ struct buf_page_desc_t{ /** We also define I_S_PAGE_TYPE_INDEX as the Index Page's position in i_s_page_type[] array */ -#define I_S_PAGE_TYPE_INDEX 1 +constexpr size_t I_S_PAGE_TYPE_INDEX = 1; /** Any unassigned FIL_PAGE_TYPE will be treated as unknown. */ -#define I_S_PAGE_TYPE_UNKNOWN FIL_PAGE_TYPE_UNKNOWN +constexpr auto I_S_PAGE_TYPE_UNKNOWN = FIL_PAGE_TYPE_UNKNOWN; /** R-tree index page */ -#define I_S_PAGE_TYPE_RTREE (FIL_PAGE_TYPE_LAST + 1) +constexpr auto I_S_PAGE_TYPE_RTREE = (FIL_PAGE_TYPE_LAST + 1); /** Change buffer B-tree page */ -#define I_S_PAGE_TYPE_IBUF (FIL_PAGE_TYPE_LAST + 2) +constexpr auto I_S_PAGE_TYPE_IBUF = (FIL_PAGE_TYPE_LAST + 2); /** SDI B-tree page */ -#define I_S_PAGE_TYPE_SDI (FIL_PAGE_TYPE_LAST + 3) +constexpr auto I_S_PAGE_TYPE_SDI = (FIL_PAGE_TYPE_LAST + 3); -#define I_S_PAGE_TYPE_LAST I_S_PAGE_TYPE_SDI +constexpr auto I_S_PAGE_TYPE_LAST = I_S_PAGE_TYPE_SDI; -#define I_S_PAGE_TYPE_BITS 6 +constexpr auto I_S_PAGE_TYPE_BITS = 6; /* Check if we can hold all page types */ -#if I_S_PAGE_TYPE_LAST >= 1 << I_S_PAGE_TYPE_BITS -# error i_s_page_type[] is too large -#endif +static_assert( + I_S_PAGE_TYPE_LAST < (1 << I_S_PAGE_TYPE_BITS), + "i_s_page_type[] is too large"); /** Name string for File Page Types */ static buf_page_desc_t i_s_page_type[] = { @@ -7512,7 +7512,7 @@ i_s_dict_fill_innodb_tablespaces( filepath = fil_space_get_first_path(space); mutex_exit(&dict_sys->mutex); } else { - filepath = fil_make_filepath(NULL, name, IBD, false); + filepath = Fil_path::make_ibd_from_table_name(name); } } @@ -7610,7 +7610,8 @@ i_s_innodb_tablespaces_fill_table( uint32 server_version; uint32 space_version; - /* Extract necessary information from a INNODB_TABLESPACES row */ + /* Extract necessary information from a INNODB_TABLESPACES + row */ ret = dd_process_dd_tablespaces_rec( heap, rec, &space, &name, &flags, &server_version, &space_version, dd_spaces); @@ -7715,7 +7716,6 @@ struct st_mysql_plugin i_s_innodb_tablespaces = STRUCT_FLD(flags, 0UL), }; - /** INFORMATION_SCHEMA.INNODB_CACHED_INDEXES */ /* Fields of the dynamic table INFORMATION_SCHEMA.INNODB_CACHED_INDEXES */ diff --git a/storage/innobase/ibuf/ibuf0ibuf.cc b/storage/innobase/ibuf/ibuf0ibuf.cc index 51207f30e18c..3c8238f5a5f1 100644 --- a/storage/innobase/ibuf/ibuf0ibuf.cc +++ b/storage/innobase/ibuf/ibuf0ibuf.cc @@ -2187,10 +2187,10 @@ void ibuf_free_excess_pages(void) /*========================*/ { - ut_ad(rw_lock_own(fil_space_get_latch(IBUF_SPACE_ID, NULL), RW_LOCK_X)); + ut_ad(rw_lock_own(fil_space_get_latch(IBUF_SPACE_ID), RW_LOCK_X)); ut_ad(rw_lock_get_x_lock_count( - fil_space_get_latch(IBUF_SPACE_ID, NULL)) == 1); + fil_space_get_latch(IBUF_SPACE_ID)) == 1); /* NOTE: We require that the thread did not own the latch before, because then we know that we can obey the correct latching order diff --git a/storage/innobase/include/arch0arch.h b/storage/innobase/include/arch0arch.h index d28b874c2dc7..50f36c6acae7 100644 --- a/storage/innobase/include/arch0arch.h +++ b/storage/innobase/include/arch0arch.h @@ -955,6 +955,13 @@ using Arch_Block_Vec = std::vector>; /** Page archiver in memory data */ struct ArchPageData { + /** Constructor */ + ArchPageData() : + m_blocks(), + m_block_size(), + m_num_blocks(), + m_buffer() { } + /** Allocate buffer and initialize blocks @return true, if successful */ bool init(); diff --git a/storage/innobase/include/dict0dd.h b/storage/innobase/include/dict0dd.h index 373b4c99b279..cf076f1bf1fa 100644 --- a/storage/innobase/include/dict0dd.h +++ b/storage/innobase/include/dict0dd.h @@ -29,7 +29,6 @@ Data dictionary interface */ #ifndef UNIV_HOTBACKUP # include
# include "sess0sess.h" - # include "dd/dd.h" # include "dd/dictionary.h" # include "dd/cache/dictionary_client.h" @@ -48,6 +47,8 @@ Data dictionary interface */ # include "dd_table_share.h" # include "dd/types/foreign_key.h" # include "dd/types/foreign_key_element.h" +#else +# include "mysql_com.h" #endif /* !UNIV_HOTBACKUP */ #include "mysql_version.h" @@ -58,7 +59,7 @@ class MDL_ticket; /** Handler name for InnoDB */ static constexpr char handler_name[] = "InnoDB"; -static const char innobase_hton_name[]= "InnoDB"; +static const char innobase_hton_name[] = "InnoDB"; #endif /* !UNIV_HOTBACKUP */ /** Postfix for a table name which is being altered. Since during @@ -66,10 +67,14 @@ ALTER TABLE ... PARTITION, new partitions have to be created before dropping existing partitions, so a postfix is appended to the name to prevent name conflicts. This is also used for EXCHANGE PARTITION */ static constexpr char TMP_POSTFIX[] = "#tmp"; +static constexpr size_t TMP_POSTFIX_LEN = sizeof(TMP_POSTFIX) - 1; /** Max space name length */ -#define MAX_SPACE_NAME_LEN ((4 * NAME_LEN) + strlen(part_sep) \ - + strlen(sub_sep) + strlen(TMP_POSTFIX)) +static constexpr size_t MAX_SPACE_NAME_LEN = + (4 * NAME_LEN) + + PART_SEPARATOR_LEN + + SUB_PART_SEPARATOR_LEN + + TMP_POSTFIX_LEN; #ifndef UNIV_HOTBACKUP /** Maximum hardcoded data dictionary tables. */ @@ -286,7 +291,6 @@ inline const dd::Index* dd_first_index(const dd::Table* table) { - ut_ad(table->partitions().empty()); return(dd_first(table)); } @@ -840,8 +844,8 @@ operation. @param[in] dd_space_id dd tablespace id @param[in] new_space_name dd_tablespace name @param[in] new_path new data file path -@retval false if fail. */ -bool +@retval DB_SUCCESS on success. */ +dberr_t dd_rename_tablespace( dd::Object_id dd_space_id, const char* new_space_name, diff --git a/storage/innobase/include/dict0dd.ic b/storage/innobase/include/dict0dd.ic index c2b0c3a96e15..c166309ff339 100644 --- a/storage/innobase/include/dict0dd.ic +++ b/storage/innobase/include/dict0dd.ic @@ -61,11 +61,11 @@ dd_get_space_id(const dd::Partition& partition) return(id); } -/** partition name separator */ -extern const char* part_sep; - -/** sub-partition name separator */ -extern const char* sub_sep; +#ifdef DBUG_OFF +#define path_to_tablename(t, n, l, f) filename_to_tablename((t), (n), (l)) +#else +#define path_to_tablename(t, n, l, f) filename_to_tablename((t), (n), (l), (f)) +#endif /* DBUG_OFF */ /** Parse a table file name into table name and database name. Note the table name may have trailing TMP_POSTFIX for temporary table name. @@ -91,10 +91,6 @@ dd_parse_tbl_name( char db_buf[MAX_DATABASE_NAME_LEN + 1]; char tbl_buf[MAX_TABLE_NAME_LEN + 1]; ulint db_len = dict_get_db_name_len(tbl_name); - ulint tbl_len = 0; - char* is_part; - char* is_sub; - char* trailing_sharp; if (db_len == 0) { return(false); @@ -103,45 +99,46 @@ dd_parse_tbl_name( ut_ad(db_len <= MAX_DATABASE_NAME_LEN); memcpy(db_buf, tbl_name, db_len); db_buf[db_len] = 0; - tbl_len = strlen(tbl_name) - db_len - 1; + + size_t tbl_len = strlen(tbl_name) - db_len - 1; + memcpy(tbl_buf, tbl_name + db_len + 1, tbl_len); tbl_buf[tbl_len] = 0; filename_to_tablename(db_buf, dd_db_name, MAX_DATABASE_NAME_LEN + 1, true); - is_part = strstr(tbl_buf, part_sep); + auto is_part = strstr(tbl_buf, PART_SEPARATOR); if (is_part != nullptr) { *is_part = '\0'; } else { - if (tbl_len > strlen(TMP_POSTFIX) - && strcmp(tbl_buf + tbl_len - strlen(TMP_POSTFIX), + if (tbl_len > TMP_POSTFIX_LEN + && strcmp(tbl_buf + tbl_len - TMP_POSTFIX_LEN, TMP_POSTFIX) == 0) { - tbl_buf[tbl_len - strlen(TMP_POSTFIX)] = '\0'; + tbl_buf[tbl_len - TMP_POSTFIX_LEN] = '\0'; if (is_temp != nullptr) { *is_temp = true; } } } - filename_to_tablename(tbl_buf, dd_tbl_name, MAX_TABLE_NAME_LEN + 1 -#ifndef DBUG_OFF - , true -#endif /* DBUG_OFF */ - ); + path_to_tablename(tbl_buf, dd_tbl_name, MAX_TABLE_NAME_LEN + 1, true); if (is_part == nullptr || dd_part_name == nullptr) { return(true); } - is_sub = strstr(is_part + strlen(part_sep), sub_sep); + auto is_sub = strstr( + is_part + PART_SEPARATOR_LEN, SUB_PART_SEPARATOR); + + char* trailing_sharp; if (is_sub == nullptr) { - trailing_sharp = strchr(is_part + strlen(part_sep), '#'); + trailing_sharp = strchr(is_part + PART_SEPARATOR_LEN, '#'); } else { *is_sub = '\0'; - trailing_sharp = strchr(is_sub + strlen(sub_sep), '#'); + trailing_sharp = strchr(is_sub + SUB_PART_SEPARATOR_LEN, '#'); } if (trailing_sharp != nullptr) { @@ -153,14 +150,17 @@ dd_parse_tbl_name( } } - if (is_part && dd_part_name) { - filename_to_tablename(is_part + strlen(part_sep), dd_part_name, - MAX_TABLE_NAME_LEN + 1); + if (is_part != nullptr && dd_part_name != nullptr ) { + + filename_to_tablename( + is_part + PART_SEPARATOR_LEN, + dd_part_name, MAX_TABLE_NAME_LEN + 1); + + if (is_sub != nullptr && dd_sub_name != nullptr ) { - if (is_sub && dd_sub_name) { filename_to_tablename( - is_sub + strlen(sub_sep), dd_sub_name, - MAX_TABLE_NAME_LEN + 1); + is_sub + SUB_PART_SEPARATOR_LEN, + dd_sub_name, MAX_TABLE_NAME_LEN + 1); } } @@ -476,6 +476,7 @@ dd_get_version( { uint64_t version = 0; const dd::Properties& p = dd_table->se_private_data(); + if (!p.exists(dd_table_key_strings[DD_TABLE_VERSION]) || p.get_uint64(dd_table_key_strings[DD_TABLE_VERSION], reinterpret_cast(&version))) { diff --git a/storage/innobase/include/dict0dict.h b/storage/innobase/include/dict0dict.h index 78fc0cfa1c20..304485cf73cf 100644 --- a/storage/innobase/include/dict0dict.h +++ b/storage/innobase/include/dict0dict.h @@ -1533,7 +1533,9 @@ struct dict_sys_t{ @param[in] space tablespace id to check @return true if a reserved tablespace id, otherwise false */ static bool is_reserved(space_id_t space) - { return(space >= dict_sys_t::s_reserved_space_id); } + { + return(space >= dict_sys_t::s_reserved_space_id); + } /** Check if a table is hardcoded. it only includes the dd tables @param[in] id table ID @@ -2252,6 +2254,12 @@ void dict_table_change_id_sys_tables(); +/** Get the tablespace data directory if set, otherwise empty string. +@return the data directory */ +std::string +dict_table_get_datadir(const dict_table_t* table) + MY_ATTRIBUTE((warn_unused_result)); + #include "dict0dict.ic" #endif diff --git a/storage/innobase/include/dict0load.h b/storage/innobase/include/dict0load.h index b710e4e6307e..f5276f08de65 100644 --- a/storage/innobase/include/dict0load.h +++ b/storage/innobase/include/dict0load.h @@ -207,6 +207,16 @@ dict_load_tablespace( mem_heap_t* heap, dict_err_ignore_t ignore_err); +/** Using the table->heap, copy the null-terminated filepath into +table->data_dir_path. The data directory patch is derived form the +filepath by stripping the the table->name.m_name component suffix. +@param[in,out] table table obj +@param[in] filepath filepath of tablespace */ +void +dict_save_data_dir_path( + dict_table_t* table, + char* filepath); + /** Load all tablespaces during upgrade */ void dict_load_tablespaces_for_upgrade(); diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index ca9a1124c506..3f5f7b83bf8f 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -35,27 +35,29 @@ Created 10/25/1995 Heikki Tuuri #ifndef UNIV_HOTBACKUP # include "ibuf0types.h" #endif /* !UNIV_HOTBACKUP */ +#include "ut0new.h" + +#include "sql/dd/object_id.h" #include #include extern const char general_space_name[]; +extern volatile bool recv_recovery_on; #ifdef UNIV_HOTBACKUP # include -using dir_set = std::unordered_set; -extern dir_set rem_gen_ts_dirs; -extern bool replay_in_datadir; +using Dir_set = std::unordered_set; +extern Dir_set rem_gen_ts_dirs; +extern bool replay_in_datadir; #endif /* UNIV_HOTBACKUP */ // Forward declaration struct trx_t; class page_id_t; -struct fil_node_t; -using Filenames = std::vector; - -typedef std::list > space_name_list_t; +using Filenames = std::vector>; +using Space_ids = std::vector>; /** File types */ enum fil_type_t : uint8_t { @@ -69,6 +71,25 @@ enum fil_type_t : uint8_t { FIL_TYPE_LOG = 8 }; +/** Result of comparing a path. */ +enum class Fil_state { + /** The path matches what was found during the scan. */ + MATCHES, + + /** No MLOG_FILE_DELETE record and the file could not be found. */ + MISSING, + + /** A MLOG_FILE_DELETE was found, file was deleted. */ + DELETED, + + /** Space ID matches but the paths don't match. */ + MOVED, + + /** Tablespace and/or filename was renamed. The DDL log will handle + this case. */ + RENAMED +}; + /** Check if fil_type is any of FIL_TYPE_TEMPORARY, FIL_TYPE_IMPORT or FIL_TYPE_TABLESPACE. @param[in] type variable of type fil_type_t @@ -83,102 +104,162 @@ fil_type_is_data(fil_type_t type) || type == FIL_TYPE_TABLESPACE); } -struct fil_node_t; +struct fil_space_t; + +/** File node of a tablespace or the log data space */ +struct fil_node_t { + + using List_node = UT_LIST_NODE_T(fil_node_t); + + /** tablespace containing this file */ + fil_space_t* space; + + /** file name; protected by Fil_shard::m_mutex and log_sys->mutex. */ + char* name; + + /** whether this file is open. Note: We set the is_open flag after + we increase the write the MLOG_FILE_OPEN record to redo log. Therefore + we increment the in_use reference count before setting the OPEN flag. */ + bool is_open; + + /** file handle (valid if is_open) */ + pfs_os_file_t handle; + + /** event that groups and serializes calls to fsync */ + os_event_t sync_event; + + /** whether the file actually is a raw device or disk partition */ + bool is_raw_disk; + + /** size of the file in database pages (0 if not known yet); + the possible last incomplete megabyte may be ignored + if space->id == 0 */ + page_no_t size; + + /** initial size of the file in database pages; + FIL_IBD_FILE_INITIAL_SIZE by default */ + page_no_t init_size; + + /** maximum size of the file in database pages */ + page_no_t max_size; + + /** count of pending i/o's; is_open must be true if nonzero */ + size_t n_pending; + + /** count of pending flushes; is_open must be true if nonzero */ + size_t n_pending_flushes; + + /** e.g., when a file is being extended or just opened. */ + size_t in_use; + + /** number of writes to the file since the system was started */ + int64_t modification_counter; + + /** the modification_counter of the latest flush to disk */ + int64_t flush_counter; + + /** link to the fil_system->LRU list (keeping track of open files) */ + List_node LRU; + + /** whether the file system of this file supports PUNCH HOLE */ + bool punch_hole; + + /** block size to use for punching holes */ + size_t block_size; + + /** whether atomic write is enabled for this file */ + bool atomic_write; + + /** FIL_NODE_MAGIC_N */ + size_t magic_n; +}; /** Tablespace or log data space */ struct fil_space_t { + using List_node = UT_LIST_NODE_T(fil_space_t); + using Files = std::vector>; + /** Tablespace name */ - char* name; + char* name; - /** space id */ - space_id_t id; + /** Tablespace ID */ + space_id_t id; /** LSN of the most recent fil_names_write_if_was_clean(). Reset to 0 by fil_names_clear(). Protected by log_sys->mutex. If and only if this is nonzero, the tablespace will be in named_spaces. */ - lsn_t max_lsn; + lsn_t max_lsn; - /** true if we want to rename the .ibd file of tablespace and + /** True if we want to rename the .ibd file of tablespace and want to stop temporarily posting of new i/o requests on the file */ - bool stop_ios; + bool stop_ios; - /** we set this true when we start deleting a single-table + /** We set this true when we start deleting a single-table tablespace. When this is set following new ops are not allowed: * read IO request * ibuf merge * file flush Note that we can still possibly have new write operations because we don't check this flag when doing flush batches. */ - bool stop_new_ops; + bool stop_new_ops; #ifdef UNIV_DEBUG - /** reference count for operations who want to skip redo log in + /** Reference count for operations who want to skip redo log in the file space in order to make fsp_space_modify_check pass. */ - ulint redo_skipped_count; -#endif + ulint redo_skipped_count; +#endif /* UNIV_DEBUG */ - /** purpose */ - fil_type_t purpose; + /** Purpose */ + fil_type_t purpose; - /** base node for the file chain */ - UT_LIST_BASE_NODE_T(fil_node_t) chain; + /** Files attached to this tablespace. Note: Only the system tablespace + can have multiple files, this is a legacy issue. */ + Files files; - /** tablespace file size in pages; 0 if not known yet */ - page_no_t size; + /** Tablespace file size in pages; 0 if not known yet */ + page_no_t size; /** FSP_SIZE in the tablespace header; 0 if not known yet */ - page_no_t size_in_header; + page_no_t size_in_header; - /** length of the FSP_FREE list */ - ulint free_len; + /** Length of the FSP_FREE list */ + uint32_t free_len; - /** contents of FSP_FREE_LIMIT */ - page_no_t free_limit; + /** Contents of FSP_FREE_LIMIT */ + page_no_t free_limit; - /** tablespace flags; - see fsp_flags_is_valid(), page_size_t(ulint) (constructor). + /** Tablespace flags; see fsp_flags_is_valid() and + page_size_t(ulint) (constructor). This is protected by space->latch and tablespace MDL */ - ulint flags; + uint32_t flags; - /** number of reserved free extents for ongoing operations like + /** Number of reserved free extents for ongoing operations like B-tree page split */ - ulint n_reserved_extents; + uint32_t n_reserved_extents; - /** this is positive when flushing the tablespace to disk; + /** This is positive when flushing the tablespace to disk; dropping of the tablespace is forbidden if this is positive */ - ulint n_pending_flushes; + uint32_t n_pending_flushes; - /** this is positive when we have pending operations against this + /** This is positive when we have pending operations against this tablespace. The pending operations can be ibuf merges or lock validation code trying to read a block. Dropping of the tablespace - is forbidden if this is positive. Protected by fil_system->mutex. */ - ulint n_pending_ops; - - /** hash chain node */ - hash_node_t hash; - - /** hash chain the name_hash table */ - hash_node_t name_hash; + is forbidden if this is positive. Protected by Fil_shard::m_mutex. */ + uint32_t n_pending_ops; #ifndef UNIV_HOTBACKUP - /** latch protecting the file space storage allocation */ - rw_lock_t latch; + /** Latch protecting the file space storage allocation */ + rw_lock_t latch; #endif /* !UNIV_HOTBACKUP */ - /** list of spaces with at least one unflushed file we have + /** List of spaces with at least one unflushed file we have written to */ - UT_LIST_NODE_T(fil_space_t) unflushed_spaces; - - /** list of spaces for which MLOG_FILE_OPEN records have been issued */ - UT_LIST_NODE_T(fil_space_t) named_spaces; + List_node unflushed_spaces; /** true if this space is currently in unflushed_spaces */ - bool is_in_unflushed_spaces; - - /** list of all spaces */ - UT_LIST_NODE_T(fil_space_t) space_list; + bool is_in_unflushed_spaces; /** Compression algorithm */ Compression::Type compression_type; @@ -205,6 +286,9 @@ struct fil_space_t { /** System tablespace */ static fil_space_t* s_sys_space; + /** Redo log tablespace */ + static fil_space_t* s_redo_space; + #ifdef UNIV_DEBUG /** Print the extent descriptor pages of this tablespace into the given output stream. @@ -220,72 +304,21 @@ struct fil_space_t { }; /** Value of fil_space_t::magic_n */ -#define FIL_SPACE_MAGIC_N 89472 - -/** File node of a tablespace or the log data space */ -struct fil_node_t { - /** tablespace containing this file */ - fil_space_t* space; - /** file name; protected by fil_system->mutex and log_sys->mutex. */ - char* name; - /** whether this file is open. Note: We set the is_open flag after - we increase the write the MLOG_FILE_OPEN record to redo log. Therefore - we increment the in_use reference count before setting the OPEN flag. */ - bool is_open; - /** file handle (valid if is_open) */ - pfs_os_file_t handle; - /** event that groups and serializes calls to fsync */ - os_event_t sync_event; - /** whether the file actually is a raw device or disk partition */ - bool is_raw_disk; - /** size of the file in database pages (0 if not known yet); - the possible last incomplete megabyte may be ignored - if space->id == 0 */ - page_no_t size; - /** initial size of the file in database pages; - FIL_IBD_FILE_INITIAL_SIZE by default */ - page_no_t init_size; - /** maximum size of the file in database pages */ - page_no_t max_size; - /** count of pending i/o's; is_open must be true if nonzero */ - ulint n_pending; - /** count of pending flushes; is_open must be true if nonzero */ - ulint n_pending_flushes; - /** e.g., when a file is being extended or just opened. */ - size_t in_use; - /** number of writes to the file since the system was started */ - int64_t modification_counter; - /** the modification_counter of the latest flush to disk */ - int64_t flush_counter; - /** link to other files in this tablespace */ - UT_LIST_NODE_T(fil_node_t) chain; - /** link to the fil_system->LRU list (keeping track of open files) */ - UT_LIST_NODE_T(fil_node_t) LRU; - - /** whether the file system of this file supports PUNCH HOLE */ - bool punch_hole; - - /** block size to use for punching holes */ - ulint block_size; - - /** whether atomic write is enabled for this file */ - bool atomic_write; - - /** FIL_NODE_MAGIC_N */ - ulint magic_n; -}; +constexpr size_t FIL_SPACE_MAGIC_N = 89472; /** Value of fil_node_t::magic_n */ -#define FIL_NODE_MAGIC_N 89389 +constexpr size_t FIL_NODE_MAGIC_N = 89389; /** Common InnoDB file extentions */ -enum ib_extention { +enum ib_file_suffix { NO_EXT = 0, IBD = 1, CFG = 2, CFP = 3 }; + extern const char* dot_ext[]; + #define DOT_IBD dot_ext[IBD] #define DOT_CFG dot_ext[CFG] #define DOT_CFP dot_ext[CFP] @@ -298,155 +331,517 @@ See https://msdn.microsoft.com/en-us/library/1ywe7hcy.aspx */ #endif /* _WIN32 */ /** Wrapper for a path to a directory that may or may not exist. */ -class Folder { +class Fil_path { public: - /** Default constructor */ - Folder() : m_folder(), m_folder_len(), m_abs_path(), m_abs_len() - {} + /** schema '/' table separator */ + static constexpr auto DB_SEPARATOR = '/'; + + /** OS specific path separator. */ + static constexpr auto OS_SEPARATOR = OS_PATH_SEPARATOR; + + /** Directory separators that are supported. */ +#if defined(__SUNPRO_CC) + static char* SEPARATOR; + static char* DOT_SLASH; + static char* DOT_DOT_SLASH; +#else + static constexpr auto SEPARATOR = "\\/"; +# ifdef _WIN32 + static constexpr auto DOT_SLASH = ".\\"; + static constexpr auto DOT_DOT_SLASH = "..\\"; +# else + static constexpr auto DOT_SLASH = "./"; + static constexpr auto DOT_DOT_SLASH = "../"; +# endif /* _WIN32 */ + +#endif /* __SUNPRO_CC */ + + /** Default constructor. Defaults to MySQL_datadir_path. */ + Fil_path(); /** Constructor - @param[in] path pathname (not necessarily NUL-terminated) - @param[in] len length of the path, in bytes */ - Folder(const char* path, size_t len); + @param[in] path Path, not necessarily NUL terminated + It's the callers responsibility to + ensure that the path is normalized. + @param[in] len Length of path */ + Fil_path(const char* path, size_t len); - /** Assignment operator - @param[in] path folder string provided */ - Folder& operator=(const char* path); + /** Constructor + @param[in] path pathname (may also include the file basename) + It's the callers responsibility to + ensure that the path is normalized. */ + explicit Fil_path(const std::string& path); /** Destructor */ - ~Folder() - { - ut_free(m_folder); - } + ~Fil_path(); /** Implicit type conversion - @return the wrapped object */ + @return pointer to m_path.c_str() */ operator const char*() const { - return(m_folder); + return(m_path.c_str()); } /** Explicit type conversion - @return the wrapped object */ + @return pointer to m_path.c_str() */ const char* operator()() const { - return(m_folder); + return(m_path.c_str()); + } + + /** @return the value of m_path */ + const std::string& path() const + MY_ATTRIBUTE((warn_unused_result)) + { + return(m_path); } - /** return the length of m_folder - @return the length of m_folder */ + /** @return the length of m_path */ size_t len() const + MY_ATTRIBUTE((warn_unused_result)) + { + return(m_path.length()); + } + + /** @return the length of m_abs_path */ + size_t abs_len() const + MY_ATTRIBUTE((warn_unused_result)) { - return(m_folder_len); + return(m_abs_path.length()); } - /** Determine if this folder is equal to the other folder. - @param[in] other folder to compare to - @return whether the folders are equal */ - bool operator==(const Folder& other) const + /** Determine if this path is equal to the other path. + @param[in] lhs Path to compare to + @return true if the paths are the same */ + bool operator==(const Fil_path& lhs) const { - return(m_abs_len == other.m_abs_len - && !memcmp(m_abs_path, other.m_abs_path, m_abs_len)); + return(m_path.compare(lhs.m_path)); } - /** Determine if this folder is an ancestor of (contains) - the other folder. - @param[in] other folder to compare to - @return whether this is an ancestor of the other folder */ - bool operator>(const Folder& other) const + /** Check if m_path is the same as path. + @param[in] path directory path to compare to + @return true if m_path is the same as path */ + bool is_same_as(const std::string& path) const + MY_ATTRIBUTE((warn_unused_result)) { - return(m_abs_len < other.m_abs_len - && (!memcmp(other.m_abs_path, m_abs_path, m_abs_len))); + if (m_path.empty() || path.empty()) { + return(false); + } + + return(m_abs_path == get_real_path(path)); } - /** Determine if the directory referenced by m_folder exists. - @return whether the directory exists */ - bool exists(); + /** Check if m_path is the parent of name. + @param[in] name Path to compare to + @return true if m_path is an ancestor of name */ + bool is_ancestor(const std::string& name) const + MY_ATTRIBUTE((warn_unused_result)) + { + if (m_path.empty() || name.empty()) { + return(false); + } + + return(is_ancestor(m_abs_path, name)); + } + + /** Check if m_path is the parent of other.m_path. + @param[in] other Path to compare to + @return true if m_path is an ancestor of name */ + bool is_ancestor(const Fil_path& other) const + MY_ATTRIBUTE((warn_unused_result)) + { + if (m_path.empty() || other.m_path.empty()) { + return(false); + } + + return(is_ancestor(m_abs_path, other.m_abs_path)); + } + + /** @return true if m_path exists and is a file. */ + bool is_file_and_exists() const + MY_ATTRIBUTE((warn_unused_result)); + + /** @return true if m_path exists and is a directory. */ + bool is_directory_and_exists() const + MY_ATTRIBUTE((warn_unused_result)); /** Return the absolute path */ - std::string abs_path() const + const std::string& abs_path() const + MY_ATTRIBUTE((warn_unused_result)) { - return(std::string(m_abs_path, m_abs_len)); + return(m_abs_path); } -private: - /** Build the basic folder name from the path and length provided - @param[in] path pathname (not necessarily NUL-terminated) - @param[in] len length of the path, in bytes */ - inline void make_path(const char* path, size_t len); - /** Resolve a relative path in m_folder to an absolute path - in m_abs_path setting m_abs_len. */ - void make_abs_path(); + /** @return true if the path is an absolute path. */ + bool is_absolute_path() const + MY_ATTRIBUTE((warn_unused_result)) + { + if (m_path.empty()) { + return(false); + } - /** The wrapped folder string */ - char* m_folder; + return(is_absolute_path(m_path)); + } + + /** This validation is only for ':'. + @return true if the path is valid. */ + bool is_valid() const + MY_ATTRIBUTE((warn_unused_result)); + + /** Remove quotes e.g., 'a;b' or "a;b" -> a;b. + Assumes matching quotes. + @return pathspec with the quotes stripped */ + static std::string parse(const char* pathspec) + { + std::string path(pathspec); + + ut_ad(!path.empty()); + + if (path.size() >= 2 + && (path.front() == '\'' || path.back() == '"')) { + + path.erase(0, 1); + + if (path.back() == '\'' || path.back() == '"') { + + path.erase(path.size() - 1); + } + } + + return(path); + } + + /** Convert the paths into absolute paths and compare them. The + paths to compare must be valid paths, otherwise the result is + undefined. + @param[in] lhs Filename to compare + @param[in] rhs Filename to compare + @return true if they are the same */ + static bool equal(const std::string& lhs, const std::string& rhs) + MY_ATTRIBUTE((warn_unused_result)) + { + Fil_path path1(lhs); + Fil_path path2(rhs); + + return(path1.abs_path().compare(path2.abs_path()) == 0); + } + + /** Determine if a path is an absolute path or not. + @param[in] path OS directory or file path to evaluate + @retval true if an absolute path + @retval false if a relative path */ + static bool is_absolute_path(const std::string& path) + MY_ATTRIBUTE((warn_unused_result)) + { + if (path.empty()) { + + return(false); + + } else if (path.at(0) == '\\' || path.at(0) == '/') { + + /* Any string that starts with an OS_SEPARATOR is + an absolute path. This includes any OS and even + paths like "\\Host\share" on Windows. */ + + return(true); + } +#ifdef _WIN32 + /* Windows may have an absolute path like 'A:\' */ + if (path.length() >= 3 + && isalpha(path.at(0)) + && path.at(1) == ':' + && (path.at(2) == '\\' || path.at(2) == '/')) { + + return(true); + } +#endif /* _WIN32 */ + + return(false); + } + + /* Check if the path is prefixed with pattern. + @return true if prefix matches */ + static bool has_prefix( + const std::string& path, + const std::string prefix) + MY_ATTRIBUTE((warn_unused_result)) + { + return(path.size() >= prefix.size() + && std::equal( + prefix.begin(), prefix.end(), path.begin())); + } + + /** Normalizes a directory path for the current OS: + On Windows, we convert '/' to '\', else we convert '\' to '/'. + @param[in,out] path Directory and file path */ + static void normalize(std::string& path) + { + for (auto& c : path) { + if (c == OS_PATH_SEPARATOR_ALT) { + c = OS_SEPARATOR; + } + } + } + + /** Normalizes a directory path for the current OS: + On Windows, we convert '/' to '\', else we convert '\' to '/'. + @param[in,out] path A NUL terminated path */ + static void normalize(char* path) + { + for (auto ptr = path; *ptr; ++ptr) { + + if (*ptr == OS_PATH_SEPARATOR_ALT) { + *ptr = OS_SEPARATOR; + } + } + } + + /** @return true if the path exists and is a file . */ + static os_file_type_t get_file_type(const std::string& path) + MY_ATTRIBUTE((warn_unused_result)); + + /** Get the real path for a directory or a file name, useful for + comparing symlinked files. + @param[in] path Directory or filename + @return the absolute path of dir + filename, or "" on error. */ + static std::string get_real_path(const std::string&path) + MY_ATTRIBUTE((warn_unused_result)); + + /** Check if lhs is the ancestor of rhs. If the two paths are the + same it will return false. + @param[in] lhs Parent path to check + @param[in] rhs Descendent path to check + @return true if lhs is an ancestor of rhs */ + static bool is_ancestor(const std::string& lhs, const std::string& rhs) + MY_ATTRIBUTE((warn_unused_result)) + { + if (lhs.empty() + || rhs.empty() + || rhs.length() <= lhs.length()) { + + return(false); + } + + return(std::equal(lhs.begin(), lhs.end(), rhs.begin())); + } + + /** Check if the name is an undo tablespace name. + @param[in] name Tablespace name + @return true if it is an undo tablespace name */ + static bool is_undo_tablespace_name(const std::string& name) + MY_ATTRIBUTE((warn_unused_result)); + + /** Check if the file has the .ibd suffix + @param[in] path Filename to check + @return true if it has the the ".ibd" suffix. */ + static bool has_ibd_suffix(const std::string& path) + { + static const char suffix[] = ".ibd"; + static constexpr auto len = sizeof(suffix) - 1; + + return(path.size() >= len + && path.compare(path.size() - len, len, suffix) == 0); + } + + /** Check if a character is a path separator ('\' or '/') + @param[in] c Character to check + @return true if it is a separator */ + static bool is_separator(char c) + { + return(c == '\\' || c == '/'); + } + + /** Allocate and build a file name from a path, a table or + tablespace name and a suffix. + @param[in] path_in nullptr or the direcory path or + the full path and filename + @param[in] name_in nullptr if path is full, or + Table/Tablespace name + @param[in] ext the file extension to use + @param[in] trim whether last name on the path should + be trimmed + @return own: file name; must be freed by ut_free() */ + static char* make( + const std::string& path_in, + const std::string& name_in, + ib_file_suffix ext, + bool trim = false) + MY_ATTRIBUTE((warn_unused_result)); + + /** Allocate and build a CFG file name from a path. + @param[in] path_in Full path to the filename + @return own: file name; must be freed by ut_free() */ + static char* make_cfg(const std::string& path_in) + MY_ATTRIBUTE((warn_unused_result)) + { + return(make(path_in, "", CFG)); + } + + /** Allocate and build a CFP file name from a path. + @param[in] path_in Full path to the filename + @return own: file name; must be freed by ut_free() */ + static char* make_cfp(const std::string& path_in) + MY_ATTRIBUTE((warn_unused_result)) + { + return(make(path_in, "", CFP)); + } + + /** Allocate and build a file name from a path, a table or + tablespace name and a suffix. + @param[in] path_in nullptr or the direcory path or + the full path and filename + @param[in] name_in nullptr if path is full, or + Table/Tablespace name + @return own: file name; must be freed by ut_free() */ + static char* make_ibd( + const std::string& path_in, + const std::string& name_in) + MY_ATTRIBUTE((warn_unused_result)) + { + return(make(path_in, name_in, IBD)); + } - /** Length of m_folder */ - size_t m_folder_len; + /** Allocate and build a file name from a path, a table or + tablespace name and a suffix. + @param[in] name_in Table/Tablespace name + @return own: file name; must be freed by ut_free() */ + static char* make_ibd_from_table_name(const std::string& name_in) + MY_ATTRIBUTE((warn_unused_result)) + { + return(make("", name_in, IBD)); + } + + /** Create an IBD path name after replacing the basename in an old path + with a new basename. The old_path is a full path name including the + extension. The tablename is in the normal form "schema/tablename". + @param[in] path_in Pathname + @param[in] name_in Contains new base name + @return new full pathname */ + static std::string make_new_ibd( + const std::string& path_in, + const std::string& name_in) + MY_ATTRIBUTE((warn_unused_result)); + + /** This function reduces a null-terminated full remote path name + into the path that is sent by MySQL for DATA DIRECTORY clause. + It replaces the 'databasename/tablename.ibd' found at the end of the + path with just 'tablename'. + + Since the result is always smaller than the path sent in, no new + memory is allocated. The caller should allocate memory for the path + sent in. This function manipulates that path in place. If the path + format is not as expected, set data_dir_path to "" and return. + + The result is used to inform a SHOW CREATE TABLE command. + @param[in,out] data_dir_path Full path/data_dir_path */ + static void make_data_dir_path(char* data_dir_path); + + /** @return the null path */ + static const Fil_path& null() + MY_ATTRIBUTE((warn_unused_result)) + { + return(s_null_path); + } + +#ifndef UNIV_HOTBACKUP + /** Check if the filepath provided is in a valid placement. + 1) File-per-table must be in a dir named for the schema. + 2) File-per-table must not be in the datadir. + 3) General tablespace must no be under the datadir. + @param[in] space_name tablespace name + @param[in] path filepath to validate + @retval true if the filepath is a valid datafile location */ + static bool is_valid_location( + const char* space_name, + const std::string& path); + + /** Convert filename to the file system charset format. + @param[in,out] name Filename to convert */ + static void convert_to_filename_charset(std::string& name); +#endif /* !UNIV_HOTBACKUP */ + +protected: + /** Path to a file or directory. */ + std::string m_path; /** A full absolute path to the same file. */ - char m_abs_path[FN_REFLEN + 2]; + std::string m_abs_path; - /** Length of m_abs_path to the deepest folder */ - size_t m_abs_len; + /** Empty (null) path. */ + static Fil_path s_null_path; }; -/** When mysqld is run, the default directory "." is the mysqld datadir, -but in the MySQL Embedded Server Library and mysqlbackup it is not the default -directory, and we must set the base file path explicitly */ -extern const char* fil_path_to_mysql_datadir; -extern Folder folder_mysql_datadir; +/** The MySQL server --datadir value */ +extern Fil_path MySQL_datadir_path; /** Initial size of a single-table tablespace in pages */ -#define FIL_IBD_FILE_INITIAL_SIZE 7 -#define FIL_IBD_FILE_INITIAL_SIZE_5_7 6 +constexpr size_t FIL_IBD_FILE_INITIAL_SIZE = 7; +constexpr size_t FIL_IBD_FILE_INITIAL_SIZE_5_7 = 6; /** 'null' (undefined) page offset in the context of file spaces */ -constexpr page_no_t FIL_NULL = std::numeric_limits::max(); +constexpr page_no_t FIL_NULL = std::numeric_limits::max(); /** Maximum Page Number, one less than FIL_NULL */ -constexpr page_no_t PAGE_NO_MAX = std::numeric_limits::max() - 1; +constexpr page_no_t PAGE_NO_MAX = std::numeric_limits::max() - 1; /** Unknown space id */ -constexpr space_id_t SPACE_UNKNOWN = std::numeric_limits::max(); +constexpr space_id_t SPACE_UNKNOWN = std::numeric_limits::max(); /* Space address data type; this is intended to be used when addresses accurate to a byte are stored in file pages. If the page part of the address is FIL_NULL, the address is considered undefined. */ -typedef byte fil_faddr_t; /*!< 'type' definition in C: an address - stored in a file page is a string of bytes */ -#define FIL_ADDR_PAGE 0 /* first in address is the page offset */ -#define FIL_ADDR_BYTE 4 /* then comes 2-byte byte offset within page*/ -#define FIL_ADDR_SIZE 6 /* address size is 6 bytes */ +/** 'type' definition in C: an address stored in a file page is a +string of bytes */ +using fil_faddr_t = byte; /** File space address */ struct fil_addr_t { + /* Default constructor */ fil_addr_t() : page(FIL_NULL), boffset(0) {} - fil_addr_t(page_no_t p, ulint off) : page(p), boffset(off) {} - - page_no_t page; /*!< page number within a space */ - ulint boffset; /*!< byte offset within the page */ - bool is_equal(const fil_addr_t& that) const { - return((page == that.page) && (boffset == that.boffset)); + /** Constructor + @param[in] p Logical page number + @param[in] boff Offset within the page */ + fil_addr_t(page_no_t p, uint32_t boff) : page(p), boffset(boff) {} + + /** Compare to instances + @param[in] rhs Instance to compare with + @return true if the page number and page offset are equal */ + bool is_equal(const fil_addr_t& rhs) const + { + return(page == rhs.page && boffset == rhs.boffset); } /** Check if the file address is null. - @return true if null, false otherwise. */ - bool is_null() const { + @return true if null */ + bool is_null() const + { return(page == FIL_NULL && boffset == 0); } + /** Print a string representation. + @param[in,out] out Stream to write to */ std::ostream& print(std::ostream& out) const { - out << "[fil_addr_t: page=" << page << ", boffset=" + out + << "[fil_addr_t: page=" << page << ", boffset=" << boffset << "]"; + return(out); } + + /** Page number within a space */ + page_no_t page; + + /** Byte offset within the page */ + uint32_t boffset; }; +/* For printing fil_addr_t to a stream. +@param[in,out] out Stream to write to +@param[in] obj fil_addr_t instance to write */ inline std::ostream& operator<<(std::ostream& out, const fil_addr_t& obj) @@ -456,67 +851,111 @@ operator<<(std::ostream& out, const fil_addr_t& obj) /** The null file address */ extern fil_addr_t fil_addr_null; -typedef uint16_t page_type_t; + +using page_type_t = uint16_t; /** File page types (values of FIL_PAGE_TYPE) @{ */ -#define FIL_PAGE_INDEX 17855 /*!< B-tree node */ -#define FIL_PAGE_RTREE 17854 /*!< R-tree node */ -#define FIL_PAGE_SDI 17853 /*!< Tablespace SDI Index page */ -#define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ -#define FIL_PAGE_INODE 3 /*!< Index node */ -#define FIL_PAGE_IBUF_FREE_LIST 4 /*!< Insert buffer free list */ +/** B-tree node */ +constexpr page_type_t FIL_PAGE_INDEX = 17855; + +/** R-tree node */ +constexpr page_type_t FIL_PAGE_RTREE = 17854; + +/** Tablespace SDI Index page */ +constexpr page_type_t FIL_PAGE_SDI = 17853; + +/** Undo log page */ +constexpr page_type_t FIL_PAGE_UNDO_LOG = 2; + +/** Index node */ +constexpr page_type_t FIL_PAGE_INODE = 3; + +/** Insert buffer free list */ +constexpr page_type_t FIL_PAGE_IBUF_FREE_LIST = 4; + /* File page types introduced in MySQL/InnoDB 5.1.7 */ -#define FIL_PAGE_TYPE_ALLOCATED 0 /*!< Freshly allocated page */ -#define FIL_PAGE_IBUF_BITMAP 5 /*!< Insert buffer bitmap */ -#define FIL_PAGE_TYPE_SYS 6 /*!< System page */ -#define FIL_PAGE_TYPE_TRX_SYS 7 /*!< Transaction system data */ -#define FIL_PAGE_TYPE_FSP_HDR 8 /*!< File space header */ -#define FIL_PAGE_TYPE_XDES 9 /*!< Extent descriptor page */ -#define FIL_PAGE_TYPE_BLOB 10 /*!< Uncompressed BLOB page */ -#define FIL_PAGE_TYPE_ZBLOB 11 /*!< First compressed BLOB page */ -#define FIL_PAGE_TYPE_ZBLOB2 12 /*!< Subsequent compressed BLOB page */ -#define FIL_PAGE_TYPE_UNKNOWN 13 /*!< In old tablespaces, garbage - in FIL_PAGE_TYPE is replaced with this - value when flushing pages. */ -#define FIL_PAGE_COMPRESSED 14 /*!< Compressed page */ -#define FIL_PAGE_ENCRYPTED 15 /*!< Encrypted page */ -#define FIL_PAGE_COMPRESSED_AND_ENCRYPTED 16 - /*!< Compressed and Encrypted page */ -#define FIL_PAGE_ENCRYPTED_RTREE 17 /*!< Encrypted R-tree page */ -#define FIL_PAGE_SDI_BLOB 18 /*!< Uncompressed SDI BLOB page */ -#define FIL_PAGE_SDI_ZBLOB 19 /*!< Commpressed SDI BLOB page */ -#define FIL_PAGE_TYPE_UNUSED 20 /*!< Available for future use. */ -#define FIL_PAGE_TYPE_RSEG_ARRAY 21 /*!< Rollback Segment Array page */ +/** Freshly allocated page */ +constexpr page_type_t FIL_PAGE_TYPE_ALLOCATED = 0; + +/** Insert buffer bitmap */ +constexpr page_type_t FIL_PAGE_IBUF_BITMAP = 5; + +/** System page */ +constexpr page_type_t FIL_PAGE_TYPE_SYS = 6; + +/** Transaction system data */ +constexpr page_type_t FIL_PAGE_TYPE_TRX_SYS = 7; + +/** File space header */ +constexpr page_type_t FIL_PAGE_TYPE_FSP_HDR = 8; + +/** Extent descriptor page */ +constexpr page_type_t FIL_PAGE_TYPE_XDES = 9; + +/** Uncompressed BLOB page */ +constexpr page_type_t FIL_PAGE_TYPE_BLOB = 10; + +/** First compressed BLOB page */ +constexpr page_type_t FIL_PAGE_TYPE_ZBLOB = 11; + +/** Subsequent compressed BLOB page */ +constexpr page_type_t FIL_PAGE_TYPE_ZBLOB2 = 12; + +/** In old tablespaces, garbage in FIL_PAGE_TYPE is replaced with +this value when flushing pages. */ +constexpr page_type_t FIL_PAGE_TYPE_UNKNOWN = 13; + +/** Compressed page */ +constexpr page_type_t FIL_PAGE_COMPRESSED = 14; + +/** Encrypted page */ +constexpr page_type_t FIL_PAGE_ENCRYPTED = 15; + +/** Compressed and Encrypted page */ +constexpr page_type_t FIL_PAGE_COMPRESSED_AND_ENCRYPTED = 16; + +/** Encrypted R-tree page */ +constexpr page_type_t FIL_PAGE_ENCRYPTED_RTREE = 17; + +/** Uncompressed SDI BLOB page */ +constexpr page_type_t FIL_PAGE_SDI_BLOB = 18; + +/** Commpressed SDI BLOB page */ +constexpr page_type_t FIL_PAGE_SDI_ZBLOB = 19; + +/** Available for future use */ +constexpr page_type_t FIL_PAGE_TYPE_UNUSED = 20; + +/** Rollback Segment Array page */ +constexpr page_type_t FIL_PAGE_TYPE_RSEG_ARRAY = 21; /** Index pages of uncompressed LOB */ -#define FIL_PAGE_TYPE_LOB_INDEX 22 +constexpr page_type_t FIL_PAGE_TYPE_LOB_INDEX = 22; /** Data pages of uncompressed LOB */ -#define FIL_PAGE_TYPE_LOB_DATA 23 +constexpr page_type_t FIL_PAGE_TYPE_LOB_DATA = 23; /** The first page of an uncompressed LOB */ -#define FIL_PAGE_TYPE_LOB_FIRST 24 +constexpr page_type_t FIL_PAGE_TYPE_LOB_FIRST = 24; /** The first page of a compressed LOB */ -#define FIL_PAGE_TYPE_ZLOB_FIRST 25 +constexpr page_type_t FIL_PAGE_TYPE_ZLOB_FIRST = 25; /** Data pages of compressed LOB */ -#define FIL_PAGE_TYPE_ZLOB_DATA 26 +constexpr page_type_t FIL_PAGE_TYPE_ZLOB_DATA = 26; /** Index pages of compressed LOB. This page contains an array of z_index_entry_t objects.*/ -#define FIL_PAGE_TYPE_ZLOB_INDEX 27 +constexpr page_type_t FIL_PAGE_TYPE_ZLOB_INDEX = 27; /** Fragment pages of compressed LOB. */ -#define FIL_PAGE_TYPE_ZLOB_FRAG 28 +constexpr page_type_t FIL_PAGE_TYPE_ZLOB_FRAG = 28; /** Index pages of fragment pages (compressed LOB). */ -#define FIL_PAGE_TYPE_ZLOB_FRAG_ENTRY 29 +constexpr page_type_t FIL_PAGE_TYPE_ZLOB_FRAG_ENTRY = 29; /** Used by i_s.cc to index into the text description. */ -#define FIL_PAGE_TYPE_LAST FIL_PAGE_TYPE_ZLOB_FRAG_ENTRY - /*!< Last page type */ -/* @} */ +constexpr page_type_t FIL_PAGE_TYPE_LAST = FIL_PAGE_TYPE_ZLOB_FRAG_ENTRY; /** Check whether the page type is index (Btree or Rtree or SDI) type */ #define fil_page_type_is_index(page_type) \ @@ -546,48 +985,37 @@ or the caller should be in single-threaded crash recovery mode (no user connections that could drop tablespaces). If this is not the case, fil_space_acquire() and fil_space_release() should be used instead. -@param[in] id tablespace ID -@return tablespace, or NULL if not found */ +@param[in] space_id Tablespace ID +@return tablespace, or nullptr if not found */ fil_space_t* -fil_space_get( - space_id_t id) +fil_space_get(space_id_t space_id) MY_ATTRIBUTE((warn_unused_result)); + #ifndef UNIV_HOTBACKUP /** Returns the latch of a file space. -@param[in] id space id -@param[out] flags tablespace flags +@param[in] space_id Tablespace ID @return latch protecting storage allocation */ rw_lock_t* -fil_space_get_latch( - space_id_t id, - ulint* flags); +fil_space_get_latch(space_id_t id) + MY_ATTRIBUTE((warn_unused_result)); -#ifdef UNIV_DEBUG +# ifdef UNIV_DEBUG /** Gets the type of a file space. -@param[in] id tablespace identifier +@param[in] space_id Tablespace ID @return file type */ fil_type_t -fil_space_get_type(space_id_t id); -#endif /* UNIV_DEBUG */ +fil_space_get_type(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); +# endif /* UNIV_DEBUG */ /** Note that a tablespace has been imported. It is initially marked as FIL_TYPE_IMPORT so that no logging is done during the import process when the space ID is stamped to each page. Now we change it to FIL_SPACE_TABLESPACE to start redo and undo logging. NOTE: temporary tablespaces are never imported. -@param[in] id tablespace identifier */ +@param[in] space_id Tablespace ID */ void -fil_space_set_imported( - space_id_t id); - -# ifdef UNIV_DEBUG -/** Determine if a tablespace is temporary. -@param[in] id tablespace identifier -@return whether it is a temporary tablespace */ -bool -fsp_is_temporary(ulint id) -MY_ATTRIBUTE((warn_unused_result, pure)); -# endif /* UNIV_DEBUG */ +fil_space_set_imported(space_id_t space_id); #endif /* !UNIV_HOTBACKUP */ /** Append a file to the chain of files of a space. @@ -598,7 +1026,7 @@ MY_ATTRIBUTE((warn_unused_result, pure)); @param[in] atomic_write true if atomic write enabled @param[in] max_pages maximum number of pages in file @return pointer to the file name -@retval NULL if error */ +@retval nullptr if error */ char* fil_node_create( const char* name, @@ -612,282 +1040,224 @@ fil_node_create( /** Create a space memory object and put it to the fil_system hash table. The tablespace name is independent from the tablespace file-name. Error messages are issued to the server log. -@param[in] name tablespace name -@param[in] id tablespace identifier -@param[in] flags tablespace flags -@param[in] purpose tablespace purpose +@param[in] name Tablespace name +@param[in] space_id Tablespace ID +@param[in] flags tablespace flags +@param[in] purpose tablespace purpose @return pointer to created tablespace, to be filled in with fil_node_create() -@retval NULL on failure (such as when the same tablespace exists) */ +@retval nullptr on failure (such as when the same tablespace exists) */ fil_space_t* fil_space_create( const char* name, - space_id_t id, + space_id_t space_id, ulint flags, fil_type_t purpose) MY_ATTRIBUTE((warn_unused_result)); -/*******************************************************************//** -Assigns a new space id for a new single-table tablespace. This works simply by -incrementing the global counter. If 4 billion id's is not enough, we may need -to recycle id's. +/** Assigns a new space id for a new single-table tablespace. +This works simply by incrementing the global counter. If 4 billion id's +is not enough, we may need to recycle id's. +@param[in,out] space_id New space ID @return true if assigned, false if not */ bool -fil_assign_new_space_id( -/*====================*/ - space_id_t* space_id); /*!< in/out: space id */ +fil_assign_new_space_id(space_id_t* space_id) + MY_ATTRIBUTE((warn_unused_result)); /** Returns the path from the first fil_node_t found with this space ID. The caller is responsible for freeing the memory allocated here for the value returned. -@param[in] id Tablespace ID -@return own: A copy of fil_node_t::path, NULL if space ID is zero -or not found. */ +@param[in] space_id Tablespace ID +@return own: A copy of fil_node_t::path, nullptr if space ID is zero + or not found. */ char* -fil_space_get_first_path(space_id_t id); +fil_space_get_first_path(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); -/*******************************************************************//** -Returns the size of the space in pages. The tablespace must be cached in the -memory cache. +/** Returns the size of the space in pages. The tablespace must be cached +in the memory cache. +@param[in] space_id Tablespace ID @return space size, 0 if space not found */ page_no_t -fil_space_get_size( -/*===============*/ - space_id_t id); /*!< in: space id */ +fil_space_get_size(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); /** Returns the flags of the space. The tablespace must be cached in the memory cache. @param[in] space_id Tablespace ID for which to get the flags @return flags, ULINT_UNDEFINED if space not found */ ulint -fil_space_get_flags(space_id_t space_id); +fil_space_get_flags(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); /** Sets the flags of the tablespace. The tablespace must be locked in MDL_EXCLUSIVE MODE. -@param[in] space tablespace in-memory struct -@param[in] flags tablespace flags */ +@param[in] space tablespace in-memory struct +@param[in] flags tablespace flags */ void fil_space_set_flags( fil_space_t* space, ulint flags); /** Open each file of a tablespace if not already open. -@param[in] space_id tablespace identifier +@param[in] space_id Tablespace ID @retval true if all file nodes were opened @retval false on failure */ bool -fil_space_open(space_id_t space_id); +fil_space_open(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); /** Close each file of a tablespace if open. -@param[in] space_id tablespace identifier */ +@param[in] space_id Tablespace ID */ void fil_space_close(space_id_t space_id); /** Returns the page size of the space and whether it is compressed or not. The tablespace must be cached in the memory cache. -@param[in] id space id -@param[out] found true if tablespace was found +@param[in] space_id Tablespace ID +@param[out] found true if tablespace was found @return page size */ const page_size_t fil_space_get_page_size( - space_id_t id, - bool* found); + space_id_t space_id, + bool* found) + MY_ATTRIBUTE((warn_unused_result)); -/****************************************************************//** -Initializes the tablespace memory cache. */ +/** Initializes the tablespace memory cache. +@param[in] max_n_open Max number of open files. */ void -fil_init( -/*=====*/ - ulint hash_size, /*!< in: hash table size */ - ulint max_n_open); /*!< in: max number of open files */ -/*******************************************************************//** -Initializes the tablespace memory cache. */ +fil_init(ulint max_n_open); + +/** Initializes the tablespace memory cache. */ void -fil_close(void); -/*===========*/ -/*******************************************************************//** -Opens all log files and system tablespace data files. They stay open until the -database server shutdown. This should be called at a server startup after the -space objects for the log and the system tablespace have been created. The -purpose of this operation is to make sure we never run out of file descriptors -if we need to read from the insert buffer or to write to the log. */ +fil_close(); + +/** Opens all log files and system tablespace data files. +They stay open until the database server shutdown. This should be called +at a server startup after the space objects for the log and the system +tablespace have been created. The purpose of this operation is to make +sure we never run out of file descriptors if we need to read from the +insert buffer or to write to the log. */ void -fil_open_log_and_system_tablespace_files(void); -/*==========================================*/ -/*******************************************************************//** -Closes all open files. There must not be any pending i/o's or not flushed +fil_open_log_and_system_tablespace_files(); + +/** Closes all open files. There must not be any pending i/o's or not flushed modifications in the files. */ void -fil_close_all_files(void); -/*=====================*/ -/*******************************************************************//** -Closes the redo log files. There must not be any pending i/o's or not -flushed modifications in the files. */ +fil_close_all_files(); + +/** Closes the redo log files. There must not be any pending i/o's or not +flushed modifications in the files. +@param[in] free_all Whether to free the instances. */ void -fil_close_log_files( -/*================*/ - bool free); /*!< in: whether to free the memory object */ - -/** File Node Iterator callback. */ -using fil_node_cbk_t = dberr_t (fil_node_t* node, void* context); - -/** Iterate through all persistent tablespace files (FIL_TYPE_TABLESPACE) -returning the nodes via callback function cbk. -@param[in] include_log include log files -@param[in] context callback function context -@param[in] callback callback function -@return any error returned by the callback function. */ -dberr_t -fil_iterate_tablespace_files( - bool include_log, - void* context, - fil_node_cbk_t* callback); - -/*******************************************************************//** -Sets the max tablespace id counter if the given number is bigger than the -previous value. */ +fil_close_log_files(bool free_all); + +/** Iterate over the files in all the tablespaces. */ +class Fil_iterator { +public: + using Function = std::function; + + /** For each data file, exclude redo log files. + @param[in] include_log include files, if true + @param[in] f Callback */ + template + static dberr_t for_each_file(bool include_log, F&& f) + { + return(iterate(include_log, [=](fil_node_t* file) + { + return(f(file)); + })); + } + + /** Iterate over the spaces and file lists. + @param[in] include_log if true then fetch log files too + @param[in,out] f Callback */ + static dberr_t iterate(bool include_log, Function&& f); +}; + +/** Sets the max tablespace id counter if the given number is bigger than the +previous value. +@param[in] max_id Maximum known tablespace ID */ void -fil_set_max_space_id_if_bigger( -/*===========================*/ - space_id_t max_id);/*!< in: maximum known id */ +fil_set_max_space_id_if_bigger(space_id_t max_id); + #ifndef UNIV_HOTBACKUP /** Write the flushed LSN to the page header of the first page in the system tablespace. -@param[in] lsn flushed LSN +@param[in] lsn Flushed LSN @return DB_SUCCESS or error number */ dberr_t -fil_write_flushed_lsn( - lsn_t lsn); +fil_write_flushed_lsn(lsn_t lsn) + MY_ATTRIBUTE((warn_unused_result)); + +#else /* !UNIV_HOTBACKUP */ +/** Extends all tablespaces to the size stored in the space header. During the +mysqlbackup --apply-log phase we extended the spaces on-demand so that log +records could be applied, but that may have left spaces still too small +compared to the size stored in the space header. */ +void +meb_extend_tablespaces_to_stored_len(); + +/** Process a file name passed as an input +@param[in] name absolute path of tablespace file +@param[in] space_id the tablespace ID */ +void +meb_fil_name_process( + const char* name, + space_id_t space_id); + #endif /* !UNIV_HOTBACKUP */ /** Acquire a tablespace when it could be dropped concurrently. Used by background threads that do not necessarily hold proper locks for concurrency control. -@param[in] id tablespace ID -@return the tablespace, or NULL if missing or being deleted */ +@param[in] space_id Tablespace ID +@return the tablespace, or nullptr if missing or being deleted */ fil_space_t* -fil_space_acquire( - space_id_t id) +fil_space_acquire(space_id_t space_id) MY_ATTRIBUTE((warn_unused_result)); /** Acquire a tablespace that may not exist. Used by background threads that do not necessarily hold proper locks for concurrency control. -@param[in] id tablespace ID -@return the tablespace, or NULL if missing or being deleted */ +@param[in] space_id Tablespace ID +@return the tablespace, or nullptr if missing or being deleted */ fil_space_t* -fil_space_acquire_silent( - space_id_t id) +fil_space_acquire_silent(space_id_t space_id) MY_ATTRIBUTE((warn_unused_result)); /** Release a tablespace acquired with fil_space_acquire(). -@param[in,out] space tablespace to release */ +@param[in,out] space Tablespace to release */ void -fil_space_release( - fil_space_t* space); - -#ifndef UNIV_HOTBACKUP -/** Wrapper with reference-counting for a fil_space_t. */ -class FilSpace -{ -public: - /** Default constructor: Use this when reference counting - is done outside this wrapper. */ - FilSpace() : m_space(NULL) {} - - /** Constructor: Look up the tablespace and increment the - referece count if found. - @param[in] space_id tablespace ID */ - explicit FilSpace(space_id_t space_id) - : m_space(fil_space_acquire(space_id)) {} - - /** Assignment operator: This assumes that fil_space_acquire() - has already been done for the fil_space_t. The caller must - assign NULL if it calls fil_space_release(). - @param[in] space tablespace to assign */ - class FilSpace& operator=( - fil_space_t* space) - { - /* fil_space_acquire() must have been invoked. */ - ut_ad(space == NULL || space->n_pending_ops > 0); - m_space = space; - return(*this); - } - - /** Destructor - Decrement the reference count if a fil_space_t - is still assigned. */ - ~FilSpace() - { - if (m_space != NULL) { - fil_space_release(m_space); - } - } - - /** Implicit type conversion - @return the wrapped object */ - operator const fil_space_t*() const - { - return(m_space); - } - - /** Explicit type conversion - @return the wrapped object */ - const fil_space_t* operator()() const - { - return(m_space); - } - -private: - /** The wrapped pointer */ - fil_space_t* m_space; -}; -#endif /* !UNIV_HOTBACKUP */ - -/** Deletes an IBD tablespace, either general or single-table. -The tablespace must be cached in the memory cache. This will delete the -datafile, fil_space_t & fil_node_t entries from the file_system_t cache. -@param[in] id Tablespace id -@param[in] buf_remove Specify the action to take on the pages -for this table in the buffer pool. -@return true if success */ -dberr_t -fil_delete_tablespace( - space_id_t id, - buf_remove_t buf_remove); - -/* Convert the paths into absolute paths and compare them. -@param[in] lhs Filename to compare -@param[in] rhs Filename to compare -@return true if they are the same */ -bool -fil_paths_equal(const char* lhs, const char* rhs); +fil_space_release(fil_space_t* space); /** Fetch the file name opened for a space_id during recovery from the file map. -@param[in] space_id undo tablespace id -@return file name that was opened */ +@param[in] space_id Undo tablespace ID +@return file name that was opened, empty string if space ID not found. */ std::string -fil_system_open_fetch(space_id_t space_id); +fil_system_open_fetch(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); /** Truncate the tablespace to needed size. -@param[in] space_id id of tablespace to truncate -@param[in] size_in_pages truncate size. +@param[in] space_id Id of tablespace to truncate +@param[in] size_in_pages Truncate size. @return true if truncate was successful. */ bool fil_truncate_tablespace( space_id_t space_id, - page_no_t size_in_pages); + page_no_t size_in_pages) + MY_ATTRIBUTE((warn_unused_result)); -/*******************************************************************//** -Closes a single-table tablespace. The tablespace must be cached in the +/** Closes a single-table tablespace. The tablespace must be cached in the memory cache. Free all pages used by the tablespace. +@param[in,out] trx Transaction covering the close +@param[in] space_id Tablespace ID @return DB_SUCCESS or error */ dberr_t -fil_close_tablespace( -/*=================*/ - trx_t* trx, /*!< in/out: Transaction covering the close */ - space_id_t id); /*!< in: space id */ -/*******************************************************************//** -Discards a single-table tablespace. The tablespace must be cached in the +fil_close_tablespace(trx_t* trx, space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Discards a single-table tablespace. The tablespace must be cached in the memory cache. Discarding is like deleting a tablespace, but 1. We do not drop the table from the data dictionary; @@ -899,65 +1269,52 @@ memory cache. Discarding is like deleting a tablespace, but same id as it originally had. 4. Free all the pages in use by the tablespace if rename=true. +@param[in] space_id Tablespace ID @return DB_SUCCESS or error */ dberr_t -fil_discard_tablespace( -/*===================*/ - space_id_t id) /*!< in: space id */ +fil_discard_tablespace(space_id_t space_id) MY_ATTRIBUTE((warn_unused_result)); /** Test if a tablespace file can be renamed to a new filepath by checking if that the old filepath exists and the new filepath does not exist. -@param[in] space_id tablespace id -@param[in] old_path old filepath -@param[in] new_path new filepath -@param[in] is_discarded whether the tablespace is discarded +@param[in] space_id Tablespace ID +@param[in] old_path Old filepath +@param[in] new_path New filepath +@param[in] is_discarded Whether the tablespace is discarded @return innodb error code */ dberr_t fil_rename_tablespace_check( space_id_t space_id, const char* old_path, const char* new_path, - bool is_discarded); + bool is_discarded) + MY_ATTRIBUTE((warn_unused_result)); /** Rename a single-table tablespace. The tablespace must exist in the memory cache. -@param[in] id tablespace identifier -@param[in] old_path old file name -@param[in] new_name new table name in the -databasename/tablename format -@param[in] new_path_in new file name, -or NULL if it is located in the normal data directory +@param[in] space_id Tablespace ID +@param[in] old_path Old file name +@param[in] new_name New tablespace name in the schema/name format +@param[in] new_path_in New file name, or nullptr if it is located in + The normal data directory @return true if success */ bool fil_rename_tablespace( - space_id_t id, + space_id_t space_id, const char* old_path, const char* new_name, - const char* new_path_in); - -/** Allocate and build a file name from a path, a table or tablespace name -and a suffix. -@param[in] path NULL or the direcory path or the full path and filename -@param[in] name NULL if path is full, or Table/Tablespace name -@param[in] ext the file extension to use -@param[in] trim whether last name on the path should be trimmed -@return own: file name; must be freed by ut_free() */ -char* -fil_make_filepath( - const char* path, - const char* name, - ib_extention ext, - bool trim); + const char* new_path_in) + MY_ATTRIBUTE((warn_unused_result)); /** Create a tablespace file. @param[in] space_id Tablespace ID @param[in] name Tablespace name in dbname/tablename format. -For general tablespaces, the 'dbname/' part may be missing. + For general tablespaces, the 'dbname/' part + may be missing. @param[in] path Path and filename of the datafile to create. @param[in] flags Tablespace flags @param[in] size Initial size of the tablespace file in pages, -must be >= FIL_IBD_FILE_INITIAL_SIZE + must be >= FIL_IBD_FILE_INITIAL_SIZE @return DB_SUCCESS or error code */ dberr_t fil_ibd_create( @@ -967,6 +1324,20 @@ fil_ibd_create( ulint flags, page_no_t size) MY_ATTRIBUTE((warn_unused_result)); + +/** Deletes an IBD tablespace, either general or single-table. +The tablespace must be cached in the memory cache. This will delete the +datafile, fil_space_t & fil_node_t entries from the file_system_t cache. +@param[in] space_id Tablespace ID +@param[in] buf_remove Specify the action to take on the pages +for this table in the buffer pool. +@return DB_SUCCESS, DB_TABLESPCE_NOT_FOUND or DB_IO_ERROR */ +dberr_t +fil_delete_tablespace( + space_id_t space_id, + buf_remove_t buf_remove) + MY_ATTRIBUTE((warn_unused_result)); + /** Open a single-table tablespace and optionally check the space id is right in it. If not successful, print an error message to the error log. This function is used to open a tablespace when we start up mysqld, and also in @@ -981,10 +1352,11 @@ The fil_node_t::handle will not be left open. (read the first page of the file and check that the space id in it matches id) @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_TEMPORARY -@param[in] id tablespace ID +@param[in] space_id Tablespace ID @param[in] flags tablespace flags @param[in] space_name tablespace name of the datafile -If file-per-table, it is the table name in the databasename/tablename format + If file-per-table, it is the table name in the + databasename/tablename format @param[in] table_name table name in case need to build filename from it @param[in] path_in expected filepath, usually read from dictionary @param[in] strict whether to report error when open ibd failed @@ -995,7 +1367,7 @@ dberr_t fil_ibd_open( bool validate, fil_type_t purpose, - space_id_t id, + space_id_t space_id, ulint flags, const char* space_name, const char* table_name, @@ -1004,80 +1376,105 @@ fil_ibd_open( bool old_space) MY_ATTRIBUTE((warn_unused_result)); -#ifndef UNIV_HOTBACKUP /** Returns true if a matching tablespace exists in the InnoDB tablespace -memory cache. Note that if we have not done a crash recovery at the database -startup, there may be many tablespaces which are not yet in the memory cache. -@param[in] id Tablespace ID -@param[in] name Tablespace name used in - fil_space_create(). -@param[in] print_err_if_not_exist Print detailed error information to the - error log if a matching tablespace is - not found from memory. -@param[in] adjust_space Whether to adjust spaceid on mismatch -@param[in] heap Heap memory -@param[in] table_id table id +memory cache. +@param[in] space_id Tablespace ID +@param[in] name Tablespace name used in + fil_space_create(). +@param[in] print_err detailed error information to the + error log if a matching tablespace is + not found from memory. +@param[in] adjust_space Whether to adjust spaceid on mismatch +@param[in] heap Heap memory +@param[in] table_id table id @return true if a matching tablespace exists in the memory cache */ bool -fil_space_for_table_exists_in_mem( - space_id_t id, +fil_space_exists_in_mem( + space_id_t space_id, const char* name, - bool print_err_if_not_exist, + bool print_err, bool adjust_space, mem_heap_t* heap, - table_id_t table_id); -#else /* !UNIV_HOTBACKUP */ -/** Extends all tablespaces to the size stored in the space header. -During the mysqlbackup --apply-log phase we extended the spaces -on-demand so that log records could be appllied, but that may have left -spaces still too small compared to the size stored in the space -header. */ + table_id_t table_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Extends all tablespaces to the size stored in the space header. During the +mysqlbackup --apply-log phase we extended the spaces on-demand so that log +records could be appllied, but that may have left spaces still too small +compared to the size stored in the space header. */ void -meb_extend_tablespaces_to_stored_len(void); -#endif /* !UNIV_HOTBACKUP */ +fil_extend_tablespaces_to_stored_len(); + /** Try to extend a tablespace if it is smaller than the specified size. -@param[in,out] space tablespace -@param[in] size desired size in pages +@param[in,out] space Tablespace ID +@param[in] size desired size in pages @return whether the tablespace is at least as big as requested */ bool fil_space_extend( fil_space_t* space, - page_no_t size); -/*******************************************************************//** -Tries to reserve free extents in a file space. + page_no_t size) + MY_ATTRIBUTE((warn_unused_result)); + +/** Tries to reserve free extents in a file space. +@param[in] space_id Tablespace ID +@param[in] n_free_now Number of free extents now +@param[in] n_to_reserve How many one wants to reserve @return true if succeed */ bool fil_space_reserve_free_extents( -/*===========================*/ - space_id_t id, /*!< in: space id */ - ulint n_free_now, /*!< in: number of free extents now */ - ulint n_to_reserve); /*!< in: how many one wants to reserve */ -/*******************************************************************//** -Releases free extents in a file space. */ + space_id_t space_id, + ulint n_free_now, + ulint n_to_reserve) + MY_ATTRIBUTE((warn_unused_result)); + +/** Releases free extents in a file space. +@param[in] space_id Tablespace ID +@param[in] n_reserved How many were reserved */ void fil_space_release_free_extents( -/*===========================*/ - space_id_t id, /*!< in: space id */ - ulint n_reserved); /*!< in: how many one reserved */ -/*******************************************************************//** -Gets the number of reserved extents. If the database is silent, this number -should be zero. */ + space_id_t space_id, + ulint n_reserved); + +/** Gets the number of reserved extents. If the database is silent, this +number should be zero. +@param[in] space_id Tablespace ID +@return the number of reserved extents */ ulint -fil_space_get_n_reserved_extents( -/*=============================*/ - space_id_t id); /*!< in: space id */ +fil_space_get_n_reserved_extents(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Read or write redo log data (synchronous buffered IO). +@param[in] type IO context +@param[in] page_id where to read or write +@param[in] page_size page size +@param[in] byte_offset remainder of offset in bytes +@param[in] len this must not cross a file boundary; +@param[in,out] buf buffer where to store read data or from where + to write +@retval DB_SUCCESS if all OK */ +dberr_t +fil_redo_io( + const IORequest& type, + const page_id_t& page_id, + const page_size_t& page_size, + ulint byte_offset, + ulint len, + void* buf) + MY_ATTRIBUTE((warn_unused_result)); -/** Read or write data. This operation could be asynchronous (aio). -@param[in,out] type IO context -@param[in] sync whether synchronous aio is desired +/** Read or write data. +@param[in] type IO context +@param[in] sync If true then do synchronous IO @param[in] page_id page id @param[in] page_size page size @param[in] byte_offset remainder of offset in bytes; in aio this -must be divisible by the OS block size + must be divisible by the OS block size @param[in] len how many bytes to read or write; this must -not cross a file boundary; in aio this must be a block size multiple + not cross a file boundary; in aio this must + be a block size multiple @param[in,out] buf buffer where to store read data or from where -to write; in aio this must be appropriately aligned + to write; in aio this must be appropriately + aligned @param[in] message message for aio handler if !sync, else ignored @return error code @retval DB_SUCCESS on success @@ -1091,83 +1488,88 @@ fil_io( ulint byte_offset, ulint len, void* buf, - void* message); + void* message) + MY_ATTRIBUTE((warn_unused_result)); -/**********************************************************************//** -Waits for an aio operation to complete. This function is used to write the +/** Waits for an aio operation to complete. This function is used to write the handler for completed requests. The aio array of pending requests is divided into segments (see os0file.cc for more info). The thread specifies which -segment it wants to wait for. */ +segment it wants to wait for. +@param[in] segment The number of the segment in the AIO array + to wait for */ void -fil_aio_wait( -/*=========*/ - ulint segment); /*!< in: the number of the segment in the aio - array to wait for */ -/**********************************************************************//** -Flushes to disk possible writes cached by the OS. If the space does not exist -or is being dropped, does not do anything. */ +fil_aio_wait(ulint segment); + +/** Flushes to disk possible writes cached by the OS. If the space does +not exist or is being dropped, does not do anything. +@param[in] space_id Tablespace ID (this can be a group of log files + or a tablespace of the database) */ void -fil_flush( -/*======*/ - space_id_t space_id); /*!< in: file space id (this can be - a group of log files or a tablespace - of the database) */ +fil_flush(space_id_t space_id); + +/** Flush to disk the writes in file spaces of the given type +possibly cached by the OS. */ +void +fil_flush_file_redo(); + /** Flush to disk the writes in file spaces of the given type possibly cached by the OS. @param[in] purpose FIL_TYPE_TABLESPACE or FIL_TYPE_LOG, can be ORred. */ void fil_flush_file_spaces(uint8_t purpose); -/******************************************************************//** -Checks the consistency of the tablespace cache. + +#ifdef UNIV_DEBUG +/** Checks the consistency of the tablespace cache. @return true if ok */ bool -fil_validate(void); -/*==============*/ -/********************************************************************//** -Returns true if file address is undefined. +fil_validate(); +#endif /* UNIV_DEBUG */ + +/** Returns true if file address is undefined. +@param[in] addr File address to check @return true if undefined */ bool -fil_addr_is_null( -/*=============*/ - fil_addr_t addr); /*!< in: address */ -/********************************************************************//** -Get the predecessor of a file page. +fil_addr_is_null(const fil_addr_t& addr) + MY_ATTRIBUTE((warn_unused_result)); + +/** Get the predecessor of a file page. +@param[in] page File page @return FIL_PAGE_PREV */ page_no_t -fil_page_get_prev( -/*==============*/ - const byte* page); /*!< in: file page */ -/********************************************************************//** -Get the successor of a file page. +fil_page_get_prev(const byte* page) + MY_ATTRIBUTE((warn_unused_result)); + +/** Get the successor of a file page. +@param[in] page File page @return FIL_PAGE_NEXT */ page_no_t -fil_page_get_next( -/*==============*/ - const byte* page); /*!< in: file page */ -/*********************************************************************//** -Sets the file page type. */ +fil_page_get_next(const byte* page) + MY_ATTRIBUTE((warn_unused_result)); + +/** Sets the file page type. +@param[in,out] page File page +@param[in] type File page type to set */ void -fil_page_set_type( -/*==============*/ - byte* page, /*!< in/out: file page */ - ulint type); /*!< in: type */ +fil_page_set_type(byte* page, ulint type); + /** Reset the page type. Data files created before MySQL 5.1 may contain garbage in FIL_PAGE_TYPE. In MySQL 3.23.53, only undo log pages and index pages were tagged. Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. -@param[in] page_id page number -@param[in,out] page page with invalid FIL_PAGE_TYPE -@param[in] type expected page type -@param[in,out] mtr mini-transaction */ +@param[in] page_id page number +@param[in,out] page page with invalid FIL_PAGE_TYPE +@param[in] type expected page type +@param[in,out] mtr mini-transaction */ void fil_page_reset_type( const page_id_t& page_id, byte* page, ulint type, mtr_t* mtr); + /** Get the file page type. -@param[in] page file page +@param[in] page File page @return page type */ inline page_type_t @@ -1182,10 +1584,10 @@ Data files created before MySQL 5.1 may contain garbage in the FIL_PAGE_TYPE field. In MySQL 3.23.53, only undo log pages and index pages were tagged. Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. -@param[in] page_id page number -@param[in,out] page page with possibly invalid FIL_PAGE_TYPE -@param[in] type expected page type -@param[in,out] mtr mini-transaction */ +@param[in] page_id page number +@param[in,out] page page with possibly invalid FIL_PAGE_TYPE +@param[in] type expected page type +@param[in,out] mtr mini-transaction */ inline void fil_page_check_type( @@ -1194,7 +1596,7 @@ fil_page_check_type( ulint type, mtr_t* mtr) { - ulint page_type = fil_page_get_type(page); + ulint page_type = fil_page_get_type(page); if (page_type != type) { fil_page_reset_type(page_id, page, type, mtr); @@ -1206,41 +1608,38 @@ Data files created before MySQL 5.1 may contain garbage in the FIL_PAGE_TYPE field. In MySQL 3.23.53, only undo log pages and index pages were tagged. Any other pages were written with uninitialized bytes in FIL_PAGE_TYPE. -@param[in,out] block block with possibly invalid FIL_PAGE_TYPE -@param[in] type expected page type -@param[in,out] mtr mini-transaction */ +@param[in,out] block block with possibly invalid FIL_PAGE_TYPE +@param[in] type expected page type +@param[in,out] mtr mini-transaction */ #define fil_block_check_type(block, type, mtr) \ fil_page_check_type(block->page.id, block->frame, type, mtr) #ifdef UNIV_DEBUG /** Increase redo skipped of a tablespace. -@param[in] id space id */ +@param[in] space_id Tablespace ID */ void -fil_space_inc_redo_skipped_count( - space_id_t id); +fil_space_inc_redo_skipped_count(space_id_t space_id); /** Decrease redo skipped of a tablespace. -@param[in] id space id */ +@param[in] space_id Tablespace ID */ void -fil_space_dec_redo_skipped_count( - space_id_t id); +fil_space_dec_redo_skipped_count(space_id_t space_id); -/*******************************************************************//** -Check whether a single-table tablespace is redo skipped. +/** Check whether a single-table tablespace is redo skipped. +@param[in] space_id Tablespace ID @return true if redo skipped */ bool -fil_space_is_redo_skipped( -/*======================*/ - space_id_t id); /*!< in: space id */ -#endif - -/********************************************************************//** -Delete the tablespace file and any related files like .cfg. -This should not be called for temporary tables. */ -void -fil_delete_file( -/*============*/ - const char* path); /*!< in: filepath of the ibd tablespace */ +fil_space_is_redo_skipped(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); +#endif /* UNIV_DEBUG */ + +/** Delete the tablespace file and any related files like .cfg. +This should not be called for temporary tables. +@param[in] path File path of the tablespace +@return true on success */ +bool +fil_delete_file(const char* path) + MY_ATTRIBUTE((warn_unused_result)); /** Callback functor. */ struct PageCallback { @@ -1259,7 +1658,9 @@ struct PageCallback { @retval DB_SUCCESS or error code. */ virtual dberr_t init( os_offset_t file_size, - const buf_block_t* block) UNIV_NOTHROW = 0; + const buf_block_t* block) + MY_ATTRIBUTE((warn_unused_result)) + UNIV_NOTHROW = 0; /** Called for every page in the tablespace. If the page was not updated then its state must be set to BUF_PAGE_NOT_USED. For @@ -1270,7 +1671,9 @@ struct PageCallback { @retval DB_SUCCESS or error code. */ virtual dberr_t operator()( os_offset_t offset, - buf_block_t* block) UNIV_NOTHROW = 0; + buf_block_t* block) + MY_ATTRIBUTE((warn_unused_result)) + UNIV_NOTHROW = 0; /** Set the name of the physical file and the file handle that is used to open it for the file that is being iterated over. @@ -1282,13 +1685,16 @@ struct PageCallback { m_filepath = filename; } - /** - @return the space id of the tablespace */ - virtual space_id_t get_space_id() const UNIV_NOTHROW = 0; + /** @return the space id of the tablespace */ + virtual space_id_t get_space_id() const + MY_ATTRIBUTE((warn_unused_result)) + UNIV_NOTHROW = 0; /** @retval the space flags of the tablespace being iterated over */ - virtual ulint get_space_flags() const UNIV_NOTHROW = 0; + virtual ulint get_space_flags() const + MY_ATTRIBUTE((warn_unused_result)) + UNIV_NOTHROW = 0; /** Set the tablespace table size. @param[in] page a page belonging to the tablespace */ @@ -1297,6 +1703,7 @@ struct PageCallback { /** The compressed page size @return the compressed page size */ const page_size_t& get_page_size() const + MY_ATTRIBUTE((warn_unused_result)) { return(m_page_size); } @@ -1310,29 +1717,27 @@ struct PageCallback { /** Physical file path. */ const char* m_filepath; -protected: // Disable copying - PageCallback(const PageCallback&); - PageCallback& operator=(const PageCallback&); + PageCallback(PageCallback&&) = delete; + PageCallback(const PageCallback&) = delete; + PageCallback& operator=(const PageCallback&) = delete; }; -/********************************************************************//** -Iterate over all the pages in the tablespace. +/** Iterate over all the pages in the tablespace. @param table the table definiton in the server @param n_io_buffers number of blocks to read and write together @param callback functor that will do the page updates @return DB_SUCCESS or error code */ dberr_t fil_tablespace_iterate( -/*===================*/ dict_table_t* table, ulint n_io_buffers, PageCallback& callback) MY_ATTRIBUTE((warn_unused_result)); -/********************************************************************//** -Looks for a pre-existing fil_space_t with the given tablespace ID -and, if found, returns the name and filepath in newly allocated buffers that the caller must free. +/** Looks for a pre-existing fil_space_t with the given tablespace ID +and, if found, returns the name and filepath in newly allocated buffers that +the caller must free. @param[in] space_id The tablespace ID to search for. @param[out] name Name of the tablespace found. @param[out] filepath The filepath of the first datafile for thtablespace found. @@ -1341,49 +1746,25 @@ bool fil_space_read_name_and_filepath( space_id_t space_id, char** name, - char** filepath); + char** filepath) + MY_ATTRIBUTE((warn_unused_result)); /** Convert a file name to a tablespace name. @param[in] filename directory/databasename/tablename.ibd @return database/tablename string, to be freed with ut_free() */ char* -fil_path_to_space_name( - const char* filename); +fil_path_to_space_name(const char* filename) + MY_ATTRIBUTE((warn_unused_result)); /** Returns the space ID based on the tablespace name. The tablespace must be found in the tablespace memory cache. This call is made from external to this module, so the mutex is not owned. -@param[in] tablespace Tablespace name +@param[in] name Tablespace name @return space ID if tablespace found, SPACE_UNKNOWN if space not. */ space_id_t -fil_space_get_id_by_name( - const char* tablespace); - -/** -Iterate over all the spaces in the space list and fetch the -tablespace names. It will return a copy of the name that must be -freed by the caller using: delete[]. -@return DB_SUCCESS if all OK. */ -dberr_t -fil_get_space_names( -/*================*/ - space_name_list_t& space_name_list) - /*!< in/out: Vector for collecting the names. */ +fil_space_get_id_by_name(const char* name) MY_ATTRIBUTE((warn_unused_result)); -/** Return the next fil_node_t in the current or next fil_space_t. -Once started, the caller must keep calling this until it returns NULL. -fil_space_acquire() and fil_space_release() are invoked here which -blocks a concurrent operation from dropping the tablespace. -@param[in] prev_node Pointer to the previous fil_node_t. -If NULL, use the first fil_space_t on fil_system->space_list. -@return pointer to the next fil_node_t. -@retval NULL if this was the last file node */ -const fil_node_t* -fil_node_next( - const fil_node_t* prev_node); - -#ifndef UNIV_HOTBACKUP /** Check if swapping two .ibd files can be done without failure @param[in] old_table old table @param[in] new_table new table @@ -1393,9 +1774,9 @@ dberr_t fil_rename_precheck( const dict_table_t* old_table, const dict_table_t* new_table, - const char* tmp_name); + const char* tmp_name) + MY_ATTRIBUTE((warn_unused_result)); -#endif /* !UNIV_HOTBACKUP */ /** Set the compression type for the tablespace of a table @param[in] table Table that should be compressesed @param[in] algorithm Text representation of the algorithm @@ -1413,6 +1794,10 @@ Compression::Type fil_get_compression(space_id_t space_id) MY_ATTRIBUTE((warn_unused_result)); +/** Set encryption. +@param[in,out] req_type IO request +@param[in] page_id Page address for IO +@param[in,out] space Tablespace instance */ void fil_io_set_encryption( IORequest& req_type, @@ -1433,31 +1818,26 @@ fil_set_encryption( byte* iv) MY_ATTRIBUTE((warn_unused_result)); -/** -@return true if the re-encrypt success */ +/** @return true if the re-encrypt success */ bool -fil_encryption_rotate(); - -extern volatile bool recv_recovery_on; +fil_encryption_rotate() + MY_ATTRIBUTE((warn_unused_result)); /** During crash recovery, open a tablespace if it had not been opened yet, to get valid size and flags. -@param[in,out] space tablespace */ +@param[in,out] space Tablespace instance */ inline void -fil_space_open_if_needed( - fil_space_t* space) +fil_space_open_if_needed(fil_space_t* space) { if (space->size == 0) { /* Initially, size and flags will be set to 0, until the files are opened for the first time. fil_space_get_size() will open the file and adjust the size and flags. */ -#ifdef UNIV_DEBUG - ulint size = -#endif /* UNIV_DEBUG */ - fil_space_get_size(space->id); - ut_ad(size == space->size); + page_no_t size = fil_space_get_size(space->id); + + ut_a(size == space->size); } } @@ -1472,8 +1852,8 @@ fil_fusionio_enable_atomic_write(pfs_os_file_t file) #endif /* !NO_FALLOCATE && UNIV_LINUX */ /** Note that the file system where the file resides doesn't support PUNCH HOLE -@param[in,out] node Node to set */ -void fil_no_punch_hole(fil_node_t* node); +@param[in,out] file File node to set */ +void fil_no_punch_hole(fil_node_t* file); #ifdef UNIV_ENABLE_UNIT_TEST_MAKE_FILEPATH void test_make_filepath(); @@ -1482,28 +1862,63 @@ void test_make_filepath(); /** @return the system tablespace instance */ #define fil_space_get_sys_space() (fil_space_t::s_sys_space) -/** Write the open table (space_id -> name) mapping to disk */ -void -fil_tablespace_open_sync_to_disk(); +/** Redo a tablespace create +@param[in] ptr redo log record +@param[in] end end of the redo log buffer +@param[in] page_id Tablespace Id and first page in file +@param[in] parsed_bytes Number of bytes parsed so far +@return pointer to next redo log record +@retval nullptr if this log record was truncated */ +byte* +fil_tablespace_redo_create( + byte* ptr, + const byte* end, + const page_id_t&page_id, + ulint parsed_bytes) + MY_ATTRIBUTE((warn_unused_result)); + +/** Redo a tablespace drop +@param[in] ptr redo log record +@param[in] end end of the redo log buffer +@param[in] page_id Tablespace Id and first page in file +@param[in] parsed_bytes Number of bytes parsed so far +@return pointer to next redo log record +@retval nullptr if this log record was truncated */ +byte* +fil_tablespace_redo_delete( + byte* ptr, + const byte* end, + const page_id_t&page_id, + ulint parsed_bytes) + MY_ATTRIBUTE((warn_unused_result)); -/** Parse or process a MLOG_FILE_* record. +/** Redo a tablespace rename @param[in] ptr redo log record @param[in] end end of the redo log buffer @param[in] page_id Tablespace Id and first page in file -@param[in] type MLOG_FILE_OPEN or MLOG_FILE_DELETE - or MLOG_FILE_CREATE2 or MLOG_FILE_RENAME2 @param[in] parsed_bytes Number of bytes parsed so far @return pointer to next redo log record -@retval NULL if this log record was truncated */ +@retval nullptr if this log record was truncated */ byte* -fil_tablespace_name_recover( +fil_tablespace_redo_rename( byte* ptr, const byte* end, const page_id_t&page_id, - mlog_id_t type, ulint parsed_bytes) MY_ATTRIBUTE((warn_unused_result)); +/** Parse and process an encryption redo record. +@param[in] ptr redo log record +@param[in] end end of the redo log buffer +@param[in] space_id the tablespace ID +@return log record end, nullptr if not a complete record */ +byte* +fil_tablespace_redo_encryption( + byte* ptr, + const byte* end, + space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + /** Read the tablespace id to path mapping from the file @param[in] recovery true if called from crash recovery */ void @@ -1516,6 +1931,24 @@ bool fil_tablespace_lookup_for_recovery(space_id_t space_id) MY_ATTRIBUTE((warn_unused_result)); +/** Lookup the tablespace ID and return the path to the file. The filename +is ignored when testing for equality. Only the path up to the file name is +considered for matching: e.g. ./test/a.ibd == ./test/b.ibd. +@param[in] dd_object_id Server DD tablespace ID +@param[in] space_id Tablespace ID to lookup +@param[in] space_name Tablespace name +@param[in] old_path Path in the data dictionary +@param[out] new_path New path if scanned path not equal to path +@return status of the match. */ +Fil_state +fil_tablespace_path_equals( + dd::Object_id dd_object_id, + space_id_t space_id, + const char* space_name, + std::string old_path, + std::string* new_path) + MY_ATTRIBUTE((warn_unused_result)); + /** This function should be called after recovery has completed. Check for tablespace files for which we did not see any MLOG_FILE_DELETE or MLOG_FILE_RENAME record. These could not be recovered @@ -1533,34 +1966,23 @@ fil_scan_for_tablespaces(const std::string& directories); /** Open the tabelspace and also get the tablespace filenames, space_id must already be known. -@param[in] space_id Tablespace ID to lookup */ -void -fil_tablespace_open_for_recovery(space_id_t space_id); - -/** Clear the tablspace ID to filename mapping. */ -void -fil_tablespace_open_clear(); - -/** Create tablespaces.open.* files */ -void -fil_tablespace_open_create(); - -/** Replay a file rename operation if possible. -@param[in] page_id Space ID and first page number in the file -@param[in] name old file name -@param[in] new_name new file name -@return whether the operation was successfully applied -(the name did not exist, or new_name did not exist and -name was successfully renamed to new_name) */ +@param[in] space_id Tablespace ID to lookup +@return true if open was successful */ bool -fil_op_replay_rename( - const page_id_t& page_id, - const char* name, - const char* new_name); +fil_tablespace_open_for_recovery(space_id_t space_id) + MY_ATTRIBUTE((warn_unused_result)); + +/** Callback to check tablespace size with space header size and extend +Caller must own the Fil_shard mutex that the file belongs to. +@param[in] file file node +@return error code */ +dberr_t +fil_check_extend_space(fil_node_t* file) + MY_ATTRIBUTE((warn_unused_result)); /** Replay a file rename operation for ddl replay. @param[in] page_id Space ID and first page number in the file -@param[in] name old file name +@param[in] old_name old file name @param[in] new_name new file name @return whether the operation was successfully applied (the name did not exist, or new_name did not exist and @@ -1568,9 +1990,29 @@ name was successfully renamed to new_name) */ bool fil_op_replay_rename_for_ddl( const page_id_t& page_id, - const char* name, + const char* old_name, const char* new_name); +/** Free the Tablespace_files instance. +@param[in] read_only_mode true if InnoDB is started in read only mode. +@return DB_SUCCESS if all OK */ +dberr_t +fil_open_for_business(bool read_only_mode) + MY_ATTRIBUTE((warn_unused_result)); + +/** Check if a path is known to InnoDB. +@param[in] path Path to check +@return true if path is known to InnoDB */ +bool +fil_check_path(const std::string& path) + MY_ATTRIBUTE((warn_unused_result)); + +/** Get the list of directories that InnoDB will search on startup. +@return the list of directories 'dir1;dir2;....;dirN' */ +std::string +fil_get_dirs() + MY_ATTRIBUTE((warn_unused_result)); + /** Rename a tablespace by its name only @param[in] old_name old tablespace name @param[in] new_name new tablespace name @@ -1578,19 +2020,19 @@ fil_op_replay_rename_for_ddl( dberr_t fil_rename_tablespace_by_name( const char* old_name, - const char* new_name); + const char* new_name) + MY_ATTRIBUTE((warn_unused_result)); + +/** Free the data structures required for recovery. */ +void +fil_free_scanned_files(); /** Update the tablespace name. Incase, the new name and old name are same, no update done. @param[in,out] space tablespace object on which name will be updated -@param[in] name new name for tablespace -@param[in] has_fil_sys true if fil_system mutex is - acquired */ +@param[in] name new name for tablespace */ void -fil_space_update_name( - fil_space_t* space, - const char* name, - bool has_fil_sys); +fil_space_update_name(fil_space_t* space, const char* name); #endif /* fil0fil_h */ diff --git a/storage/innobase/include/fil0types.h b/storage/innobase/include/fil0types.h index 889487f1be0c..8b2c52d30d02 100644 --- a/storage/innobase/include/fil0types.h +++ b/storage/innobase/include/fil0types.h @@ -74,38 +74,50 @@ pages, we store the compressed page control information in these 8 bytes. */ FIL_PAGE_FILE_FLUSH_LSN are broken down as follows: */ /** Control information version format (u8) */ -static const ulint FIL_PAGE_VERSION = FIL_PAGE_FILE_FLUSH_LSN; +constexpr ulint FIL_PAGE_VERSION = FIL_PAGE_FILE_FLUSH_LSN; /** Compression algorithm (u8) */ -static const ulint FIL_PAGE_ALGORITHM_V1 = FIL_PAGE_VERSION + 1; +constexpr ulint FIL_PAGE_ALGORITHM_V1 = FIL_PAGE_VERSION + 1; /** Original page type (u16) */ -static const ulint FIL_PAGE_ORIGINAL_TYPE_V1 = FIL_PAGE_ALGORITHM_V1 + 1; +constexpr ulint FIL_PAGE_ORIGINAL_TYPE_V1 = FIL_PAGE_ALGORITHM_V1 + 1; /** Original data size in bytes (u16)*/ -static const ulint FIL_PAGE_ORIGINAL_SIZE_V1 = FIL_PAGE_ORIGINAL_TYPE_V1 + 2; +constexpr ulint FIL_PAGE_ORIGINAL_SIZE_V1 = FIL_PAGE_ORIGINAL_TYPE_V1 + 2; /** Size after compression (u16) */ -static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; +constexpr ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; /** This overloads FIL_PAGE_FILE_FLUSH_LSN for RTREE Split Sequence Number */ -#define FIL_RTREE_SPLIT_SEQ_NUM FIL_PAGE_FILE_FLUSH_LSN +constexpr ulint FIL_RTREE_SPLIT_SEQ_NUM = FIL_PAGE_FILE_FLUSH_LSN; /** starting from 4.1.x this contains the space id of the page */ -#define FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID 34 +constexpr ulint FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID = 34; /** alias for space id */ #define FIL_PAGE_SPACE_ID FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID /** start of the data on the page */ -#define FIL_PAGE_DATA 38U +constexpr ulint FIL_PAGE_DATA = 38; /** File page trailer */ /** the low 4 bytes of this are used to store the page checksum, the last 4 bytes should be identical to the last 4 bytes of FIL_PAGE_LSN */ -#define FIL_PAGE_END_LSN_OLD_CHKSUM 8 +constexpr ulint FIL_PAGE_END_LSN_OLD_CHKSUM = 8; /** size of the page trailer */ -#define FIL_PAGE_DATA_END 8 +constexpr ulint FIL_PAGE_DATA_END = 8; + +/** First in address is the page offset. */ +constexpr size_t FIL_ADDR_PAGE = 0; + +/** Then comes 2-byte byte offset within page.*/ +constexpr size_t FIL_ADDR_BYTE = 4; + +/** Address size is 6 bytes. */ +constexpr size_t FIL_ADDR_SIZE = 6; + +/** Path separator e.g., 'dir;...;dirN' */ +constexpr char FIL_PATH_SEPARATOR = ';'; #endif /* fil0types_h */ diff --git a/storage/innobase/include/fsp0file.h b/storage/innobase/include/fsp0file.h index 43eb16f713ca..0d80ad398dc9 100644 --- a/storage/innobase/include/fsp0file.h +++ b/storage/innobase/include/fsp0file.h @@ -54,10 +54,15 @@ meb_get_encryption_key( /** Types of raw partitions in innodb_data_file_path */ enum device_t { - SRV_NOT_RAW = 0, /*!< Not a raw partition */ - SRV_NEW_RAW, /*!< A 'newraw' partition, only to be - initialized */ - SRV_OLD_RAW /*!< An initialized raw partition */ + + /** Not a raw partition */ + SRV_NOT_RAW = 0, + + /** A 'newraw' partition, only to be initialized */ + SRV_NEW_RAW, + + /** An initialized raw partition */ + SRV_OLD_RAW }; /** Data file control information. */ @@ -71,10 +76,6 @@ class Datafile { Datafile() : m_name(), - m_filepath(), -#ifdef UNIV_HOTBACKUP - m_dirpath(NULL), -#endif /* UNIV_HOTBACKUP */ m_filename(), m_open_flags(OS_FILE_OPEN), m_size(), @@ -87,10 +88,11 @@ class Datafile { m_first_page_buf(), m_first_page(), m_atomic_write(), + m_filepath(), m_last_os_error(), m_file_info(), - m_encryption_key(NULL), - m_encryption_iv(NULL) + m_encryption_key(), + m_encryption_iv() { m_handle.m_file = OS_FILE_CLOSED; } @@ -98,10 +100,6 @@ class Datafile { Datafile(const char* name, ulint flags, page_no_t size, ulint order) : m_name(mem_strdup(name)), - m_filepath(), -#ifdef UNIV_HOTBACKUP - m_dirpath(NULL), -#endif /* UNIV_HOTBACKUP */ m_filename(), m_open_flags(OS_FILE_OPEN), m_size(size), @@ -114,21 +112,19 @@ class Datafile { m_first_page_buf(), m_first_page(), m_atomic_write(), + m_filepath(), m_last_os_error(), m_file_info(), - m_encryption_key(NULL), - m_encryption_iv(NULL) + m_encryption_key(), + m_encryption_iv() { - ut_ad(m_name != NULL); + ut_ad(m_name != nullptr); m_handle.m_file = OS_FILE_CLOSED; /* No op */ } Datafile(const Datafile& file) : -#ifdef UNIV_HOTBACKUP - m_dirpath(NULL), -#endif /* UNIV_HOTBACKUP */ m_handle(file.m_handle), m_open_flags(file.m_open_flags), m_size(file.m_size), @@ -143,19 +139,19 @@ class Datafile { m_atomic_write(file.m_atomic_write), m_last_os_error(), m_file_info(), - m_encryption_key(NULL), - m_encryption_iv(NULL) + m_encryption_key(), + m_encryption_iv() { m_name = mem_strdup(file.m_name); - ut_ad(m_name != NULL); + ut_ad(m_name != nullptr); - if (file.m_filepath != NULL) { + if (file.m_filepath != nullptr) { m_filepath = mem_strdup(file.m_filepath); - ut_a(m_filepath != NULL); + ut_a(m_filepath != nullptr); set_filename(); } else { - m_filepath = NULL; - m_filename = NULL; + m_filepath = nullptr; + m_filename = nullptr; } } @@ -168,9 +164,9 @@ class Datafile { { ut_a(this != &file); - ut_ad(m_name == NULL); + ut_ad(m_name == nullptr); m_name = mem_strdup(file.m_name); - ut_a(m_name != NULL); + ut_a(m_name != nullptr); m_size = file.m_size; m_order = file.m_order; @@ -186,24 +182,24 @@ class Datafile { m_flags = file.m_flags; m_last_os_error = 0; - if (m_filepath != NULL) { + if (m_filepath != nullptr) { ut_free(m_filepath); - m_filepath = NULL; - m_filename = NULL; + m_filepath = nullptr; + m_filename = nullptr; } - if (file.m_filepath != NULL) { + if (file.m_filepath != nullptr) { m_filepath = mem_strdup(file.m_filepath); - ut_a(m_filepath != NULL); + ut_a(m_filepath != nullptr); set_filename(); } /* Do not make a copy of the first page, it should be reread if needed */ - m_first_page_buf = NULL; - m_first_page = NULL; - m_encryption_key = NULL; - m_encryption_iv = NULL; + m_first_page_buf = nullptr; + m_first_page = nullptr; + m_encryption_key = nullptr; + m_encryption_iv = nullptr; m_atomic_write = file.m_atomic_write; @@ -242,7 +238,7 @@ class Datafile { /** Make a full filepath from a directory path and a filename. Prepend the dirpath to filename using the extension given. - If dirpath is NULL, prepend the default datadir to filepath. + If dirpath is nullptr, prepend the default datadir to filepath. Store the result in m_filepath. @param[in] dirpath directory path @param[in] filename filename or filepath @@ -250,7 +246,7 @@ class Datafile { void make_filepath( const char* dirpath, const char* filename, - ib_extention ext); + ib_file_suffix ext); /** Set the filepath by duplicating the filepath sent in */ void set_filepath(const char* filepath); @@ -260,7 +256,7 @@ class Datafile { extract a file-per-table tablespace name from m_filepath; else it is a general tablespace, so just call it that for now. The value of m_name will be freed in the destructor. - @param[in] name Tablespace Name if known, NULL if not */ + @param[in] name Tablespace Name if known, nullptr if not */ void set_name(const char* name); /** Validates the datafile and checks that it conforms with @@ -297,7 +293,7 @@ class Datafile { @param[out] flush_lsn contents of FIL_PAGE_FILE_FLUSH_LSN @param[in] for_import if it is for importing (only valid for the first file of the system tablespace) - @retval DB_TABLESPACE_NOT_FOUND tablespace in file header doesn't match + @retval DB_WRONG_FILE_NAME tablespace in file header doesn't match expected value @retval DB_SUCCESS on if the datafile is valid @retval DB_CORRUPTION if the datafile is not readable @@ -398,20 +394,29 @@ class Datafile { @return true if it is the same file, else false */ bool same_as(const Datafile& other) const; - ulint size() const + /** Determine the space id of the given file descriptor by reading + a few pages from the beginning of the .ibd file. + @return DB_SUCCESS if space id was successfully identified, + else DB_ERROR. */ + dberr_t find_space_id(); + +#ifdef UNIV_HOTBACKUP + /** @return file size in number of pages */ + page_no_t size() const { return(m_size); } + /** Set the tablespace ID. + @param[in] space_id Tablespace ID to set */ void set_space_id(space_id_t space_id) { ut_ad(space_id <= 0xFFFFFFFFU); m_space_id = space_id; } -#ifdef UNIV_HOTBACKUP - /** Set the tablespace flags - @param[in] fsp_flags tablespace flags */ + /** Set th tablespace flags + @param[in] fsp_flags Tablespace flags */ void set_flags(ulint flags) { m_flags = flags; @@ -426,7 +431,7 @@ class Datafile { in the filepath. */ void set_filename() { - if (m_filepath == NULL) { + if (m_filepath == nullptr) { return; } @@ -458,31 +463,7 @@ class Datafile { void set_open_flags(os_file_create_t open_flags) { m_open_flags = open_flags; - }; - - /* DATA MEMBERS */ - - /** Datafile name at the tablespace location. - This is either the basename of the file if an absolute path - was entered, or it is the relative path to the datadir or - Tablespace::m_path. */ - char* m_name; - -protected: - /** Physical file path with base name and extension */ - char* m_filepath; - -#ifdef UNIV_HOTBACKUP - /** directory path where tablespace resides */ - char* m_dirpath; -#endif /* UNIV_HOTBACKUP */ - -private: - /** Determine the space id of the given file descriptor by reading - a few pages from the beginning of the .ibd file. - @return DB_SUCCESS if space id was successfully identified, - else DB_ERROR. */ - dberr_t find_space_id(); + } /** Finds a given page of the given space id from the double write buffer and copies it to the corresponding .ibd file. @@ -491,6 +472,13 @@ class Datafile { dberr_t restore_from_doublewrite( page_no_t restore_page_no); +private: + /** Datafile name at the tablespace location. + This is either the basename of the file if an absolute path + was entered, or it is the relative path to the datadir or + Tablespace::m_path. */ + char* m_name; + /** Points into m_filepath to the file name with extension */ char* m_filename; @@ -541,17 +529,22 @@ class Datafile { bool m_atomic_write; protected: + /** Physical file path with base name and extension */ + char* m_filepath; + /** Last OS error received so it can be reported if needed. */ ulint m_last_os_error; public: /** Use the following to determine the uniqueness of this datafile. */ #ifdef _WIN32 - /* Use fields dwVolumeSerialNumber, nFileIndexLow, nFileIndexHigh. */ - BY_HANDLE_FILE_INFORMATION m_file_info; + using WIN32_FILE_INFO = BY_HANDLE_FILE_INFORMATION; + + /** Use fields dwVolumeSerialNumber, nFileIndexLow, nFileIndexHigh. */ + WIN32_FILE_INFO m_file_info; #else - /* Use field st_ino. */ - struct stat m_file_info; + /** Use field st_ino. */ + struct stat m_file_info; #endif /* WIN32 */ /** Encryption key read from first page */ diff --git a/storage/innobase/include/fsp0fsp.ic b/storage/innobase/include/fsp0fsp.ic index 1f9558f14baa..cab23c88d8ee 100644 --- a/storage/innobase/include/fsp0fsp.ic +++ b/storage/innobase/include/fsp0fsp.ic @@ -279,18 +279,17 @@ xdes_calc_descriptor_page( const page_size_t& page_size, page_no_t offset) { -#ifndef DOXYGEN /* Doxygen gets confused by these */ -# if UNIV_PAGE_SIZE_MAX <= XDES_ARR_OFFSET \ - + (UNIV_PAGE_SIZE_MAX / FSP_EXTENT_SIZE_MAX) \ - * XDES_SIZE_MAX -# error -# endif -# if UNIV_ZIP_SIZE_MIN <= XDES_ARR_OFFSET \ - + (UNIV_ZIP_SIZE_MIN / FSP_EXTENT_SIZE_MIN) \ - * XDES_SIZE_MIN -# error -# endif -#endif /* !DOXYGEN */ + static_assert( + UNIV_PAGE_SIZE_MAX + > XDES_ARR_OFFSET + + (UNIV_PAGE_SIZE_MAX / FSP_EXTENT_SIZE_MAX) + * XDES_SIZE_MAX, "Extent descriptor won't fit on a page"); + + static_assert( + UNIV_ZIP_SIZE_MIN + > XDES_ARR_OFFSET + + (UNIV_ZIP_SIZE_MIN / FSP_EXTENT_SIZE_MIN) + * XDES_SIZE_MIN, "Extent descriptor won't fit on a page"); ut_ad(UNIV_PAGE_SIZE > XDES_ARR_OFFSET + (UNIV_PAGE_SIZE / FSP_EXTENT_SIZE) diff --git a/storage/innobase/include/fsp0space.h b/storage/innobase/include/fsp0space.h index e5bf3b0ae26f..0656e29d4b1c 100644 --- a/storage/innobase/include/fsp0space.h +++ b/storage/innobase/include/fsp0space.h @@ -1,6 +1,6 @@ /***************************************************************************** -Copyright (c) 2013, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2017, Oracle and/or its affiliates. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -100,7 +100,7 @@ class Tablespace { m_path = mem_strdupl(path, len); ut_ad(m_path != NULL); - os_normalize_path(m_path); + Fil_path::normalize(m_path); } /** Set tablespace path and filename members. diff --git a/storage/innobase/include/fut0lst.h b/storage/innobase/include/fut0lst.h index 4cd3ec4ea69a..ed8271ea6480 100644 --- a/storage/innobase/include/fut0lst.h +++ b/storage/innobase/include/fut0lst.h @@ -40,10 +40,10 @@ typedef byte flst_base_node_t; typedef byte flst_node_t; /* The physical size of a list base node in bytes */ -#define FLST_BASE_NODE_SIZE (4 + 2 * FIL_ADDR_SIZE) +constexpr ulint FLST_BASE_NODE_SIZE = 4 + 2 * FIL_ADDR_SIZE; /* The physical size of a list node in bytes */ -#define FLST_NODE_SIZE (2 * FIL_ADDR_SIZE) +constexpr ulint FLST_NODE_SIZE = 2 * FIL_ADDR_SIZE; /** Initializes a list base node. @param[in] base pointer to base node diff --git a/storage/innobase/include/ha_prototypes.h b/storage/innobase/include/ha_prototypes.h index 25a9003a148c..561656b6ed3c 100644 --- a/storage/innobase/include/ha_prototypes.h +++ b/storage/innobase/include/ha_prototypes.h @@ -511,4 +511,17 @@ buffer pool size. void innodb_set_buf_pool_size(long long buf_pool_size); +/** Gets the InnoDB transaction handle for a MySQL handler object, creates +an InnoDB transaction struct if the corresponding MySQL thread struct still +lacks one. +@param[in] thd MySQL thd (connection) object +@return InnoDB transaction handle */ +trx_t* +check_trx_exists(THD* thd); + +/** Commits a transaction in an InnoDB database. +@param[in] trx Transaction handle. */ +void +innobase_commit_low(trx_t* trx); + #endif /* HA_INNODB_PROTOTYPES_H */ diff --git a/storage/innobase/include/ibuf0ibuf.ic b/storage/innobase/include/ibuf0ibuf.ic index 0976b2f4e317..a00b814ed122 100644 --- a/storage/innobase/include/ibuf0ibuf.ic +++ b/storage/innobase/include/ibuf0ibuf.ic @@ -317,14 +317,16 @@ ibuf_update_free_bits_if_full( block->page.size.physical(), max_ins_size); if (max_ins_size >= increase) { -#if ULINT32_UNDEFINED <= UNIV_PAGE_SIZE_MAX -# error "ULINT32_UNDEFINED <= UNIV_PAGE_SIZE_MAX" -#endif + static_assert( + ULINT32_UNDEFINED > UNIV_PAGE_SIZE_MAX, + "ULINT32_UNDEFINED <= UNIV_PAGE_SIZE_MAX"); + after = ibuf_index_page_calc_free_bits( block->page.size.physical(), max_ins_size - increase); #ifdef UNIV_IBUF_DEBUG ut_a(after <= ibuf_index_page_calc_free(block)); -#endif +#endif /* UNIV_IBUF_DEBUG */ + } else { after = ibuf_index_page_calc_free(block); } diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 80d4ba3cfa2c..5764fcc209c3 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -553,7 +553,7 @@ because InnoDB never supported more than one copy of the redo log. */ /** 4 unused (zero-initialized) bytes. */ #define LOG_HEADER_PAD1 4 /** LSN of the start of data in this log file -(with format version 1 and 2). */ +(with format version 1, 2 and 3). */ #define LOG_HEADER_START_LSN 8 /** A null-terminated string which will contain either the string 'MEB' and the MySQL version if the log file was created by mysqlbackup, @@ -577,9 +577,13 @@ enum log_header_format_t redo log record. */ LOG_HEADER_FORMAT_8_0_1 = 2, + /** Remove MLOG_FILE_OPEN, MLOG_FILE_CREATE2 and MLOG_FILE_RENAME2 + Resurrect MLOG_FILE_CREATE and MLOG_FILE_RENAME. */ + LOG_HEADER_FORMAT_8_0_3 = 3, + /** The redo log format identifier corresponding to the current format version. */ - LOG_HEADER_FORMAT_CURRENT = LOG_HEADER_FORMAT_8_0_1 + LOG_HEADER_FORMAT_CURRENT = LOG_HEADER_FORMAT_8_0_3 }; /* @} */ diff --git a/storage/innobase/include/mtr0types.h b/storage/innobase/include/mtr0types.h index a9288fde8a55..b189f9bdbd00 100644 --- a/storage/innobase/include/mtr0types.h +++ b/storage/innobase/include/mtr0types.h @@ -141,6 +141,12 @@ enum mlog_id_t { /** dummy log record used to pad a log block full */ MLOG_DUMMY_RECORD = 32, + /** log record about creating an .ibd file, with format */ + MLOG_FILE_CREATE = 33, + + /** rename a tablespace file that starts with (space_id,page_no) */ + MLOG_FILE_RENAME = 34, + /** delete a tablespace file that starts with (space_id,page_no) */ MLOG_FILE_DELETE = 35, @@ -179,9 +185,6 @@ enum mlog_id_t { /** reorganize an index page */ MLOG_COMP_PAGE_REORGANIZE = 46, - /** log record about creating an .ibd file, with format */ - MLOG_FILE_CREATE2 = 47, - /** write the node pointer of a record on a compressed non-leaf B-tree page */ MLOG_ZIP_WRITE_NODE_PTR = 48, @@ -202,13 +205,6 @@ enum mlog_id_t { /** reorganize a compressed page */ MLOG_ZIP_PAGE_REORGANIZE = 53, - /** rename a tablespace file that starts with (space_id,page_no) */ - MLOG_FILE_RENAME2 = 54, - - /** Track the open files. For mapping space IDs to physical filenames - during recovery. */ - MLOG_FILE_OPEN = 55, - /** Create a R-Tree index page */ MLOG_PAGE_CREATE_RTREE = 57, diff --git a/storage/innobase/include/os0file.h b/storage/innobase/include/os0file.h index a77d6319223b..8b7cb6002baa 100644 --- a/storage/innobase/include/os0file.h +++ b/storage/innobase/include/os0file.h @@ -202,6 +202,7 @@ static const ulint OS_FILE_INSUFFICIENT_RESOURCE = 78; static const ulint OS_FILE_AIO_INTERRUPTED = 79; static const ulint OS_FILE_OPERATION_ABORTED = 80; static const ulint OS_FILE_ACCESS_VIOLATION = 81; +static const ulint OS_FILE_NAME_TOO_LONG = 82; static const ulint OS_FILE_ERROR_MAX = 100; /* @} */ @@ -234,16 +235,15 @@ static const ulint ENCRYPTION_SERVER_UUID_LEN = 36; /** Encryption information total size for 5.7.11: magic number + master_key_id + key + iv + checksum */ -static const ulint ENCRYPTION_INFO_SIZE_V1 = (ENCRYPTION_MAGIC_SIZE \ - + (ENCRYPTION_KEY_LEN * 2) \ - + 2 * sizeof(ulint)); +static const ulint ENCRYPTION_INFO_SIZE_V1 = + ENCRYPTION_MAGIC_SIZE + (ENCRYPTION_KEY_LEN * 2) + 2 * sizeof(ulint); /** Encryption information total size: magic number + master_key_id + key + iv + server_uuid + checksum */ -static const ulint ENCRYPTION_INFO_SIZE_V2 = (ENCRYPTION_MAGIC_SIZE \ - + (ENCRYPTION_KEY_LEN * 2) \ - + ENCRYPTION_SERVER_UUID_LEN \ - + 2 * sizeof(ulint)); +static const ulint ENCRYPTION_INFO_SIZE_V2 = + ENCRYPTION_MAGIC_SIZE + (ENCRYPTION_KEY_LEN * 2) + + ENCRYPTION_SERVER_UUID_LEN + 2 * sizeof(ulint); + /** Default master key for bootstrap */ static const char ENCRYPTION_DEFAULT_MASTER_KEY[] = "DefaultMasterKey"; @@ -428,8 +428,8 @@ struct Encryption { /** Decrypt the log block. @param[in] type IORequest - @param[in,out] src data read from disk, decrypted data will be - copied to this page + @param[in,out] src data read from disk, decrypted data + will be copied to this page @param[in,out] dst scratch area to use for decryption @return DB_SUCCESS or error code */ dberr_t decrypt_log_block( @@ -439,8 +439,8 @@ struct Encryption { /** Decrypt the log data contents. @param[in] type IORequest - @param[in,out] src data read from disk, decrypted data will be - copied to this page + @param[in,out] src data read from disk, decrypted data + will be copied to this page @param[in] src_len source data length @param[in,out] dst scratch area to use for decryption @param[in] dst_len size of the scratch area in bytes @@ -487,10 +487,10 @@ struct Encryption { byte* m_iv; /** Current master key id */ - static ulint master_key_id; + static ulint s_master_key_id; /** Current uuid of server instance */ - static char uuid[ENCRYPTION_SERVER_UUID_LEN + 1]; + static char s_uuid[ENCRYPTION_SERVER_UUID_LEN + 1]; }; /** Types for AIO operations @{ */ @@ -852,22 +852,24 @@ struct os_file_size_t { static const ulint OS_AIO_N_PENDING_IOS_PER_THREAD = 32; /** Modes for aio operations @{ */ -/** Normal asynchronous i/o not for ibuf pages or ibuf bitmap pages */ -static const ulint OS_AIO_NORMAL = 21; - -/** Asynchronous i/o for ibuf pages or ibuf bitmap pages */ -static const ulint OS_AIO_IBUF = 22; - -/** Asynchronous i/o for the log */ -static const ulint OS_AIO_LOG = 23; - -/** Asynchronous i/o where the calling thread will itself wait for -the i/o to complete, doing also the job of the i/o-handler thread; -can be used for any pages, ibuf or non-ibuf. This is used to save -CPU time, as we can do with fewer thread switches. Plain synchronous -I/O is not as good, because it must serialize the file seek and read -or write, causing a bottleneck for parallelism. */ -static const ulint OS_AIO_SYNC = 24; +enum class AIO_mode : size_t { + /** Normal asynchronous i/o not for ibuf pages or ibuf bitmap pages */ + NORMAL = 21, + + /** Asynchronous i/o for ibuf pages or ibuf bitmap pages */ + IBUF = 22, + + /** Asynchronous i/o for the log */ + LOG = 23, + + /** Asynchronous i/o where the calling thread will itself wait for + the i/o to complete, doing also the job of the i/o-handler thread; + can be used for any pages, ibuf or non-ibuf. This is used to save + CPU time, as we can do with fewer thread switches. Plain synchronous + I/O is not as good, because it must serialize the file seek and read + or write, causing a bottleneck for parallelism. */ + SYNC = 24 +}; /* @} */ extern ulint os_n_file_reads; @@ -877,12 +879,32 @@ extern ulint os_n_fsyncs; /* File types for directory entry data type */ enum os_file_type_t { - OS_FILE_TYPE_MISSING = 0, + /** Get status failed. */ + OS_FILE_TYPE_FAILED, + + /** stat() failed, with ENAMETOOLONG */ + OS_FILE_TYPE_NAME_TOO_LONG, + + /** stat() failed with EACCESS */ + OS_FILE_PERMISSION_ERROR, + + /** File doesn't exist. */ + OS_FILE_TYPE_MISSING, + + /** File exists but type is unknown. */ OS_FILE_TYPE_UNKNOWN, - OS_FILE_TYPE_FILE, /* regular file */ - OS_FILE_TYPE_DIR, /* directory */ - OS_FILE_TYPE_LINK, /* symbolic link */ - OS_FILE_TYPE_BLOCK /* block device */ + + /** Ordinary file. */ + OS_FILE_TYPE_FILE, + + /** Directory. */ + OS_FILE_TYPE_DIR, + + /** Symbolic link. */ + OS_FILE_TYPE_LINK, + + /** Block device. */ + OS_FILE_TYPE_BLOCK }; /* Maximum path string length in bytes when referring to tables with in the @@ -1497,7 +1519,7 @@ UNIV_INLINE dberr_t pfs_os_aio_func( IORequest& type, - ulint mode, + AIO_mode mode, const char* name, pfs_os_file_t file, void* buf, @@ -1959,39 +1981,6 @@ os_file_status( bool* exists, os_file_type_t* type); -/** This function returns a new path name after replacing the basename -in an old path with a new basename. The old_path is a full path -name including the extension. The tablename is in the normal -form "databasename/tablename". The new base name is found after -the forward slash. Both input strings are null terminated. - -This function allocates memory to be returned. It is the callers -responsibility to free the return value after it is no longer needed. - -@param[in] old_path pathname -@param[in] tablename new file name -@return own: new full pathname */ -char* -os_file_make_new_pathname( - const char* old_path, - const char* tablename); - -/** This function reduces a null-terminated full remote path name into -the path that is sent by MySQL for DATA DIRECTORY clause. It replaces -the 'databasename/tablename.ibd' found at the end of the path with just -'tablename'. - -Since the result is always smaller than the path sent in, no new memory -is allocated. The caller should allocate memory for the path sent in. -This function manipulates that path in place. - -If the path format is not as expected, just return. The result is used -to inform a SHOW CREATE TABLE command. -@param[in,out] data_dir_path full path/data_dir_path */ -void -os_file_make_data_dir_path( - char* data_dir_path); - /** Create all missing subdirectories along the given path. @return DB_SUCCESS if OK, otherwise error code. */ dberr_t @@ -2010,6 +1999,17 @@ void meb_free_block_cache(); #endif /* UNIV_HOTBACKUP */ +/** Creates and initializes block_cache. Creates array of MAX_BLOCKS +and allocates the memory in each block to hold BUFFER_BLOCK_SIZE +of data. + +This function is called by InnoDB during AIO init (os_aio_init()). +It is also by MEB while applying the redo logs on TDE tablespaces, the +"Blocks" allocated in this block_cache are used to hold the decrypted page +data. */ +void +os_create_block_cache(); + /** Initializes the asynchronous io system. Creates one array each for ibuf and log i/o. Also creates one array each for read and write where each array is divided logically into n_read_segs and n_write_segs @@ -2035,7 +2035,7 @@ os_aio_free(); NOTE! Use the corresponding macro os_aio(), not directly this function! Requests an asynchronous i/o operation. @param[in] type IO request context -@param[in] mode IO mode +@param[in] aio_mode IO mode @param[in] name Name of the file or path as NUL terminated string @param[in] file Open file handle @@ -2053,7 +2053,7 @@ Requests an asynchronous i/o operation. dberr_t os_aio_func( IORequest& type, - ulint mode, + AIO_mode aio_mode, const char* name, pfs_os_file_t file, void* buf, @@ -2217,7 +2217,6 @@ not then the source contents are left unchanged and DB_SUCCESS is returned. @param[in,out] dst Scratch area to use for decompression @param[in] dst_len Size of the scratch area in bytes @return DB_SUCCESS or error code */ - dberr_t os_file_decompress_page( bool dblwr_recover, @@ -2226,38 +2225,12 @@ os_file_decompress_page( ulint dst_len) MY_ATTRIBUTE((warn_unused_result)); -/** Normalizes a directory path for the current OS: -On Windows, we convert '/' to '\', else we convert '\' to '/'. -@param[in,out] str A null-terminated directory and file path */ -void os_normalize_path(char* str); - /** Determine if O_DIRECT is supported. @retval true if O_DIRECT is supported. @retval false if O_DIRECT is not supported. */ bool -os_is_o_direct_supported(); - -/* Determine if a path is an absolute path or not. -@param[in] OS directory or file path to evaluate -@retval true if an absolute path -@retval false if a relative path */ -UNIV_INLINE -bool -is_absolute_path( - const char* path) -{ - if (path[0] == OS_PATH_SEPARATOR) { - return(true); - } - -#ifdef _WIN32 - if (path[1] == ':' && path[2] == OS_PATH_SEPARATOR) { - return(true); - } -#endif /* _WIN32 */ - - return(false); -} +os_is_o_direct_supported() + MY_ATTRIBUTE((warn_unused_result)); /** Class to scan the directory heirarch using a depth first scan. */ class Dir_Walker { diff --git a/storage/innobase/include/os0file.ic b/storage/innobase/include/os0file.ic index d2a3d383a34b..3e69ce4ca1c7 100644 --- a/storage/innobase/include/os0file.ic +++ b/storage/innobase/include/os0file.ic @@ -215,7 +215,7 @@ function! Performance schema wrapper function of os_aio() which requests an asynchronous i/o operation. @param[in] type IO request context -@param[in] mode IO mode +@param[in] aio_mode IO mode @param[in] name Name of the file or path as NUL terminated string @param[in] file Open file handle @@ -236,7 +236,7 @@ UNIV_INLINE dberr_t pfs_os_aio_func( IORequest& type, - ulint mode, + AIO_mode aio_mode, const char* name, pfs_os_file_t file, void* buf, @@ -260,7 +260,7 @@ pfs_os_aio_func( src_file, src_line); dberr_t result = os_aio_func( - type, mode, name, file, buf, offset, n, read_only, m1, m2); + type, aio_mode, name, file, buf, offset, n, read_only, m1, m2); register_pfs_file_io_end(locker, n); diff --git a/storage/innobase/include/os0thread-create.h b/storage/innobase/include/os0thread-create.h index ca2c62147d53..e6aea473bae8 100644 --- a/storage/innobase/include/os0thread-create.h +++ b/storage/innobase/include/os0thread-create.h @@ -108,15 +108,17 @@ class Runnable { { my_thread_init(); -#ifdef UNIV_PFS_THREAD - PSI_thread* psi; +#if defined(UNIV_PFS_THREAD) && !defined(UNIV_HOTBACKUP) + if (m_pfs_key.m_value != PFS_NOT_INSTRUMENTED.m_value) { + PSI_thread* psi; - psi = PSI_THREAD_CALL(new_thread)(m_pfs_key.m_value, nullptr, - 0); + psi = PSI_THREAD_CALL(new_thread)( + m_pfs_key.m_value, nullptr, 0); - PSI_THREAD_CALL(set_thread_os_id)(psi); - PSI_THREAD_CALL(set_thread)(psi); -#endif /* UNIV_PFS_THREAD */ + PSI_THREAD_CALL(set_thread_os_id)(psi); + PSI_THREAD_CALL(set_thread)(psi); + } +#endif /* UNIV_PFS_THREAD && !UNIV_HOTBACKUP */ std::atomic_thread_fence(std::memory_order_release); @@ -139,9 +141,11 @@ class Runnable { my_thread_end(); -#ifdef UNIV_PFS_THREAD - PSI_THREAD_CALL(delete_current_thread)(); -#endif /* UNIV_PFS_THREAD */ +#if defined(UNIV_PFS_THREAD) && !defined(UNIV_HOTBACKUP) + if (m_pfs_key.m_value != PFS_NOT_INSTRUMENTED.m_value) { + PSI_THREAD_CALL(delete_current_thread)(); + } +#endif /* UNIV_PFS_THREAD && !UNIV_HOTBACKUP */ } private: #ifdef UNIV_PFS_THREAD @@ -169,5 +173,51 @@ create_detached_thread(mysql_pfs_key_t pfs_key, F&& f, Args&& ... args) #define os_thread_create(k, ...) create_detached_thread(0, __VA_ARGS__) #endif /* UNIV_PFS_THREAD */ +/** Parallel for loop over a container. +@param[in] pfs_key Performance schema thread key +@param[in] c Container to iterate over in parallel +@param[in] n Number of threads to create +@param[in] f Callable instance +@param[in] args zero or more args */ +template +void +par_for(mysql_pfs_key_t pfs_key, + const Container& c, + size_t n, + F&& f, + Args&& ... args) +{ + if (c.empty()) { + return; + } + + size_t slice = (n > 0) ? c.size() / n : 0; + + using Workers = std::vector; + + Workers workers; + + for (size_t i = 0; i < n; ++i) { + + auto b = c.begin() + (i * slice); + auto e = b + slice; + + workers.push_back( + std::thread{Runnable{pfs_key}, f, b, e, i, args ...}); + } + + f(c.begin() + (n * slice), c.end(), n, args ...); + + for (auto& worker : workers) { + worker.join(); + } +} + +#if defined(UNIV_PFS_THREAD) && !defined(UNIV_HOTBACKUP) +#define par_for(...) par_for(__VA_ARGS__) +#else +#define par_for(k, ...) par_for(0, __VA_ARGS__) +#endif /* UNIV_PFS_THREAD */ + #endif /* !os0thread_create_h */ diff --git a/storage/innobase/include/page0page.ic b/storage/innobase/include/page0page.ic index f4e6125fbb4c..f21dd4d4c2c4 100644 --- a/storage/innobase/include/page0page.ic +++ b/storage/innobase/include/page0page.ic @@ -409,24 +409,31 @@ page_rec_is_user_rec_low( ulint offset) /*!< in: record offset on page */ { ut_ad(offset >= PAGE_NEW_INFIMUM); -#if PAGE_OLD_INFIMUM < PAGE_NEW_INFIMUM -# error "PAGE_OLD_INFIMUM < PAGE_NEW_INFIMUM" -#endif -#if PAGE_OLD_SUPREMUM < PAGE_NEW_SUPREMUM -# error "PAGE_OLD_SUPREMUM < PAGE_NEW_SUPREMUM" -#endif -#if PAGE_NEW_INFIMUM > PAGE_OLD_SUPREMUM -# error "PAGE_NEW_INFIMUM > PAGE_OLD_SUPREMUM" -#endif -#if PAGE_OLD_INFIMUM > PAGE_NEW_SUPREMUM -# error "PAGE_OLD_INFIMUM > PAGE_NEW_SUPREMUM" -#endif -#if PAGE_NEW_SUPREMUM > PAGE_OLD_SUPREMUM_END -# error "PAGE_NEW_SUPREMUM > PAGE_OLD_SUPREMUM_END" -#endif -#if PAGE_OLD_SUPREMUM > PAGE_NEW_SUPREMUM_END -# error "PAGE_OLD_SUPREMUM > PAGE_NEW_SUPREMUM_END" -#endif + + static_assert( + PAGE_OLD_INFIMUM >= PAGE_NEW_INFIMUM, + "PAGE_OLD_INFIMUM < PAGE_NEW_INFIMUM"); + + static_assert( + PAGE_OLD_SUPREMUM >= PAGE_NEW_SUPREMUM, + "PAGE_OLD_SUPREMUM < PAGE_NEW_SUPREMUM"); + + static_assert( + PAGE_NEW_INFIMUM <= PAGE_OLD_SUPREMUM, + "PAGE_NEW_INFIMUM > PAGE_OLD_SUPREMUM"); + + static_assert( + PAGE_OLD_INFIMUM <= PAGE_NEW_SUPREMUM, + "PAGE_OLD_INFIMUM > PAGE_NEW_SUPREMUM"); + + static_assert( + PAGE_NEW_SUPREMUM <= PAGE_OLD_SUPREMUM_END, + "PAGE_NEW_SUPREMUM > PAGE_OLD_SUPREMUM_END"); + + static_assert( + PAGE_OLD_SUPREMUM <= PAGE_NEW_SUPREMUM_END, + "PAGE_OLD_SUPREMUM > PAGE_NEW_SUPREMUM_END"); + ut_ad(offset <= UNIV_PAGE_SIZE - PAGE_EMPTY_DIR_START); return(offset != PAGE_NEW_SUPREMUM diff --git a/storage/innobase/include/page0size.h b/storage/innobase/include/page0size.h index 69ce55bb2216..a83aa5eefc33 100644 --- a/storage/innobase/include/page0size.h +++ b/storage/innobase/include/page0size.h @@ -90,9 +90,10 @@ class page_size_t { ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); - /* If the fsp_flags have zero in the zip_ssize field, then it means - that the tablespace does not have compressed pages and the physical - page size is the same as the logical page size. */ + /* If the fsp_flags have zero in the zip_ssize field, then + it means that the tablespace does not have compressed pages + and the physical page size is the same as the logical page + size. */ if (ssize == 0) { m_is_compressed = false; m_physical = m_logical; @@ -205,8 +206,7 @@ class page_size_t { if (ssize == 0) { m_is_compressed = false; m_physical = m_logical; - } - else { + } else { m_is_compressed = true; /* Convert from a 'log2 minus 9' to a page size diff --git a/storage/innobase/include/page0zip.ic b/storage/innobase/include/page0zip.ic index d1ea29f02d89..ab046c637e77 100644 --- a/storage/innobase/include/page0zip.ic +++ b/storage/innobase/include/page0zip.ic @@ -145,11 +145,11 @@ page_zip_rec_needs_ext( ut_ad(rec_size > (comp ? REC_N_NEW_EXTRA_BYTES : REC_N_OLD_EXTRA_BYTES)); ut_ad(comp || !page_size.is_compressed()); -#if UNIV_PAGE_SIZE_MAX > REC_MAX_DATA_SIZE - if (rec_size >= REC_MAX_DATA_SIZE) { + if (UNIV_PAGE_SIZE_MAX > REC_MAX_DATA_SIZE + && rec_size >= REC_MAX_DATA_SIZE) { + return(TRUE); } -#endif if (page_size.is_compressed()) { ut_ad(comp); diff --git a/storage/innobase/include/rem0rec.h b/storage/innobase/include/rem0rec.h index 74e5856d78c1..4b6fd7995862 100644 --- a/storage/innobase/include/rem0rec.h +++ b/storage/innobase/include/rem0rec.h @@ -893,12 +893,12 @@ rec_get_trx_id( /* Maximum lengths for the data in a physical record if the offsets are given in one byte (resp. two byte) format. */ -#define REC_1BYTE_OFFS_LIMIT 0x7FUL -#define REC_2BYTE_OFFS_LIMIT 0x7FFFUL +constexpr ulint REC_1BYTE_OFFS_LIMIT = 0x7FUL; +constexpr ulint REC_2BYTE_OFFS_LIMIT = 0x7FFFUL; /* The data size of record must be smaller than this because we reserve two upmost bits in a two byte offset for special purposes */ -#define REC_MAX_DATA_SIZE 16384 +constexpr ulint REC_MAX_DATA_SIZE = 16384; #include "rem0rec.ic" diff --git a/storage/innobase/include/rem0rec.ic b/storage/innobase/include/rem0rec.ic index 3d762389dc09..fc5523fbc47f 100644 --- a/storage/innobase/include/rem0rec.ic +++ b/storage/innobase/include/rem0rec.ic @@ -165,22 +165,28 @@ rec_get_next_ptr_const( } if (comp) { -#if UNIV_PAGE_SIZE_MAX <= 32768 - /* Note that for 64 KiB pages, field_value can 'wrap around' - and the debug assertion is not valid */ - - /* In the following assertion, field_value is interpreted - as signed 16-bit integer in 2's complement arithmetics. - If all platforms defined int16_t in the standard headers, - the expression could be written simpler as - (int16_t) field_value + ut_align_offset(...) < UNIV_PAGE_SIZE - */ - ut_ad((field_value >= 32768 - ? field_value - 65536 - : field_value) - + ut_align_offset(rec, UNIV_PAGE_SIZE) - < UNIV_PAGE_SIZE); -#endif + + if (UNIV_PAGE_SIZE_MAX <= 32768) { + + /* Note that for 64 KiB pages, field_value can + 'wrap around' and the debug assertion is not valid */ + + /* In the following assertion, field_value is + interpreted as signed 16-bit integer in 2's complement + arithmetics. If all platforms defined int16_t in the + standard headers, the expression could be written + simpler as: + + (int16_t) field_value + ut_align_offset(...) + < UNIV_PAGE_SIZE */ + + ut_ad((field_value >= 32768 + ? field_value - 65536 + : field_value) + + ut_align_offset(rec, UNIV_PAGE_SIZE) + < UNIV_PAGE_SIZE); + } + /* There must be at least REC_N_NEW_EXTRA_BYTES + 1 between each record. */ ut_ad((field_value > REC_N_NEW_EXTRA_BYTES @@ -233,22 +239,29 @@ rec_get_next_offs( field_value = mach_read_from_2(rec - REC_NEXT); if (comp) { -#if UNIV_PAGE_SIZE_MAX <= 32768 - /* Note that for 64 KiB pages, field_value can 'wrap around' - and the debug assertion is not valid */ - - /* In the following assertion, field_value is interpreted - as signed 16-bit integer in 2's complement arithmetics. - If all platforms defined int16_t in the standard headers, - the expression could be written simpler as - (int16_t) field_value + ut_align_offset(...) < UNIV_PAGE_SIZE - */ - ut_ad((field_value >= 32768 - ? field_value - 65536 - : field_value) - + ut_align_offset(rec, UNIV_PAGE_SIZE) - < UNIV_PAGE_SIZE); -#endif + + if (UNIV_PAGE_SIZE_MAX <= 32768) { + + /* Note that for 64 KiB pages, field_value can + 'wrap around' and the debug assertion is not valid */ + + /* In the following assertion, field_value is + interpreted as signed 16-bit integer in 2's complement + arithmetics. + + If all platforms defined int16_t in the standard + headers, the expression could be written simpler as + + (int16_t) field_value + ut_align_offset(...) + < UNIV_PAGE_SIZE */ + + ut_ad((field_value >= 32768 + ? field_value - 65536 + : field_value) + + ut_align_offset(rec, UNIV_PAGE_SIZE) + < UNIV_PAGE_SIZE); + } + if (field_value == 0) { return(0); diff --git a/storage/innobase/include/srv0start.h b/storage/innobase/include/srv0start.h index 56870da4e8e2..d9665064e7d2 100644 --- a/storage/innobase/include/srv0start.h +++ b/storage/innobase/include/srv0start.h @@ -93,7 +93,7 @@ srv_undo_tablespaces_update(ulong target); recovery "dir1;dir2; ... dirN" @return DB_SUCCESS or error code */ dberr_t -srv_start(bool create_new_db, const char* scan_directories); +srv_start(bool create_new_db, const std::string& scan_directories); /** On a restart, initialize the remaining InnoDB subsystems so that any tables (including data dictionary tables) can be accessed. */ diff --git a/storage/innobase/include/sync0types.h b/storage/innobase/include/sync0types.h index a1da621dd78e..d76e3a41af9a 100644 --- a/storage/innobase/include/sync0types.h +++ b/storage/innobase/include/sync0types.h @@ -216,6 +216,8 @@ enum latch_level_t { SYNC_ANY_LATCH, + SYNC_FIL_SHARD, + SYNC_DOUBLEWRITE, SYNC_PAGE_ARCH_OPER, @@ -335,7 +337,7 @@ enum latch_id_t { LATCH_ID_CACHE_LAST_READ, LATCH_ID_DICT_FOREIGN_ERR, LATCH_ID_DICT_SYS, - LATCH_ID_FIL_SYSTEM, + LATCH_ID_FIL_SHARD, LATCH_ID_FLUSH_LIST, LATCH_ID_FTS_BG_THREADS, LATCH_ID_FTS_DELETE, diff --git a/storage/innobase/include/univ.i b/storage/innobase/include/univ.i index c1c92590857d..3df967c176c2 100644 --- a/storage/innobase/include/univ.i +++ b/storage/innobase/include/univ.i @@ -320,7 +320,7 @@ limit both with this same constant. */ /** Minimum page size InnoDB currently supports. */ #define UNIV_PAGE_SIZE_MIN (1 << UNIV_PAGE_SIZE_SHIFT_MIN) /** Maximum page size InnoDB currently supports. */ -#define UNIV_PAGE_SIZE_MAX (1 << UNIV_PAGE_SIZE_SHIFT_MAX) +constexpr size_t UNIV_PAGE_SIZE_MAX = (1 << UNIV_PAGE_SIZE_SHIFT_MAX); /** Default page size for InnoDB tablespaces. */ #define UNIV_PAGE_SIZE_DEF (1 << UNIV_PAGE_SIZE_SHIFT_DEF) /** Original 16k page size for InnoDB tablespaces. */ @@ -451,8 +451,9 @@ typedef long int lint; /** The bitmask of 32-bit unsigned integer */ #define ULINT32_MASK 0xFFFFFFFF + /** The undefined 32-bit unsigned integer */ -#define ULINT32_UNDEFINED ULINT32_MASK +constexpr uint32_t ULINT32_UNDEFINED = ULINT32_MASK; /** Maximum value for a ulint */ #define ULINT_MAX ((ulint)(-2)) @@ -673,4 +674,20 @@ constexpr auto to_int(T v) -> typename std::underlying_type::type { return(static_cast::type>(v)); } + +/** If we are doing something that takes longer than this many seconds then +print an informative message. Type should be return type of ut_time(). */ +static constexpr ib_time_t PRINT_INTERVAL_SECS = 10; + +constexpr size_t PART_SEPARATOR_LEN = 3; +constexpr size_t SUB_PART_SEPARATOR_LEN = 4; + +#ifdef _WIN32 +constexpr char PART_SEPARATOR[PART_SEPARATOR_LEN + 1] = "#p#"; +constexpr char SUB_PART_SEPARATOR[SUB_PART_SEPARATOR_LEN + 1] = "#sp#"; +#else +constexpr char PART_SEPARATOR[PART_SEPARATOR_LEN + 1] = "#P#"; +constexpr char SUB_PART_SEPARATOR[SUB_PART_SEPARATOR_LEN + 1] = "#SP#"; +#endif /* _WIN32 */ + #endif diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 66b6ee190b3a..f4cfe33db454 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -160,7 +160,7 @@ log_buf_pool_get_oldest_modification(void) } /* Note this will work between the two formats 5_7_9 & current because -the only change is the version number */ +the only change is the version number. Assumes an "empty" redo log. */ static void log_downgrade() @@ -177,11 +177,17 @@ log_downgrade() log_block_set_checksum(buf, log_block_calc_checksum_crc32(buf)); - fil_io(IORequestLogWrite, true, - page_id_t(group->space_id, page_no), - univ_page_size, - (ulint) (dest_offset % univ_page_size.physical()), - OS_FILE_LOG_BLOCK_SIZE, buf, group); + dberr_t err; + + err = fil_redo_io( + IORequestLogWrite, + page_id_t(group->space_id, page_no), + univ_page_size, + (ulint) (dest_offset % univ_page_size.physical()), + OS_FILE_LOG_BLOCK_SIZE, buf); + + ut_a(err == DB_SUCCESS); + } /** Extends the log buffer. @@ -915,34 +921,30 @@ log_group_init( void log_io_complete(log_group_t* group) { - if ((ulint) group & 0x1UL) { - /* It was a checkpoint write */ - group = (log_group_t*)((ulint) group - 1); + ut_a((ulint) group & 0x1UL); + + /* It was a checkpoint write */ + group = (log_group_t*)((ulint) group - 1); #ifdef _WIN32 - fil_flush(group->space_id); + fil_flush_file_redo(); #else - switch (srv_unix_file_flush_method) { - case SRV_UNIX_O_DSYNC: - case SRV_UNIX_NOSYNC: - break; - case SRV_UNIX_FSYNC: - case SRV_UNIX_LITTLESYNC: - case SRV_UNIX_O_DIRECT: - case SRV_UNIX_O_DIRECT_NO_FSYNC: - fil_flush(group->space_id); - } + switch (srv_unix_file_flush_method) { + case SRV_UNIX_O_DSYNC: + case SRV_UNIX_NOSYNC: + break; + case SRV_UNIX_FSYNC: + case SRV_UNIX_LITTLESYNC: + case SRV_UNIX_O_DIRECT: + case SRV_UNIX_O_DIRECT_NO_FSYNC: + fil_flush_file_redo(); + } #endif /* _WIN32 */ - DBUG_PRINT("ib_log", ("checkpoint info written to group %u", - unsigned(group->id))); - log_io_complete_checkpoint(); + log_io_complete_checkpoint(); - return; - } - - ut_error; /*!< We currently use synchronous writing of the - logs and cannot end up here! */ + DBUG_PRINT("ib_log", ("checkpoint info written to group %u", + unsigned(group->id))); } /** Fill redo log header @@ -1007,11 +1009,16 @@ log_group_file_header_flush( ut_ad(!log_sys->disable_redo_writes); - fil_io(IORequestLogWrite, true, - page_id_t(group->space_id, page_no), - univ_page_size, - (ulint) (dest_offset % univ_page_size.physical()), - OS_FILE_LOG_BLOCK_SIZE, buf, group); + dberr_t err; + + err = fil_redo_io( + IORequestLogWrite, + page_id_t(group->space_id, page_no), + univ_page_size, + (ulint) (dest_offset % univ_page_size.physical()), + OS_FILE_LOG_BLOCK_SIZE, buf); + + ut_a(err == DB_SUCCESS); srv_stats.os_log_pending_writes.dec(); } @@ -1037,9 +1044,12 @@ log_read_encryption() log_block_buf = static_cast( ut_align(log_block_buf_ptr, OS_FILE_LOG_BLOCK_SIZE)); - fil_io(IORequestLogRead, true, page_id, univ_page_size, - LOG_CHECKPOINT_1 + OS_FILE_LOG_BLOCK_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_block_buf, NULL); + err = fil_redo_io( + IORequestLogRead, page_id, univ_page_size, + LOG_CHECKPOINT_1 + OS_FILE_LOG_BLOCK_SIZE, + OS_FILE_LOG_BLOCK_SIZE, log_block_buf); + + ut_a(err == DB_SUCCESS); if (memcmp(log_block_buf + LOG_HEADER_CREATOR_END, ENCRYPTION_KEY_MAGIC_V2, ENCRYPTION_MAGIC_SIZE) == 0) { @@ -1169,11 +1179,15 @@ log_write_encryption( srv_stats.os_log_pending_writes.inc(); - fil_io(IORequestLogWrite, true, - page_id, - univ_page_size, - LOG_CHECKPOINT_1 + OS_FILE_LOG_BLOCK_SIZE, - OS_FILE_LOG_BLOCK_SIZE, log_block_buf, NULL); + dberr_t err; + + err = fil_redo_io( + IORequestLogWrite, + page_id, univ_page_size, + LOG_CHECKPOINT_1 + OS_FILE_LOG_BLOCK_SIZE, + OS_FILE_LOG_BLOCK_SIZE, log_block_buf); + + ut_a(err == DB_SUCCESS); srv_stats.os_log_pending_writes.dec(); log_write_mutex_exit(); @@ -1256,7 +1270,7 @@ log_enable_encryption_if_set() /* If the redo log space is using default key, rotate it. We also need the server_uuid initialized. */ if (space->encryption_type != Encryption::NONE - && Encryption::master_key_id == ENCRYPTION_DEFAULT_MASTER_KEY_ID + && Encryption::s_master_key_id == ENCRYPTION_DEFAULT_MASTER_KEY_ID && !srv_read_only_mode && strlen(server_uuid) > 0) { ut_ad(FSP_FLAGS_GET_ENCRYPTION(space->flags)); @@ -1380,11 +1394,15 @@ log_group_write_buf( ut_ad(!log_sys->disable_redo_writes); - fil_io(IORequestLogWrite, true, - page_id_t(group->space_id, page_no), - univ_page_size, - (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf, - group); + dberr_t err; + + err = fil_redo_io( + IORequestLogWrite, + page_id_t(group->space_id, page_no), + univ_page_size, + (ulint) (next_offset % UNIV_PAGE_SIZE), write_len, buf); + + ut_a(err == DB_SUCCESS); srv_stats.os_log_pending_writes.dec(); @@ -1415,8 +1433,7 @@ log_write_flush_to_disk_low() bool do_flush = true; #endif if (do_flush) { - log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); - fil_flush(group->space_id); + fil_flush_file_redo(); log_sys->flushed_to_disk_lsn = log_sys->current_flush_lsn; } @@ -1515,7 +1532,8 @@ log_write_up_to( } #ifdef _WIN32 - /* write requests during fil_flush() might not be good for Windows */ + /* write requests during fil_flush_file_redo() might not be good + for Windows */ if (log_sys->n_pending_flushes > 0 || !os_event_is_set(log_sys->flush_event)) { log_write_mutex_exit(); @@ -1931,13 +1949,17 @@ log_group_checkpoint( added with 1, as we want to distinguish between a normal log file write and a checkpoint field write */ - fil_io(IORequestLogWrite, false, - page_id_t(group->space_id, 0), - univ_page_size, - (log_sys->next_checkpoint_no & 1) - ? LOG_CHECKPOINT_2 : LOG_CHECKPOINT_1, - OS_FILE_LOG_BLOCK_SIZE, - buf, (byte*) group + 1); + dberr_t err; + + err = fil_io( + IORequestLogWrite, false, + page_id_t(group->space_id, 0), + univ_page_size, + (log_sys->next_checkpoint_no & 1) + ? LOG_CHECKPOINT_2 : LOG_CHECKPOINT_1, + OS_FILE_LOG_BLOCK_SIZE, buf, (byte*) group + 1); + + ut_a(err == DB_SUCCESS); ut_ad(((ulint) group & 0x1UL) == 0); } @@ -1958,12 +1980,17 @@ log_group_header_read( MONITOR_INC(MONITOR_LOG_IO); - fil_io(IORequestLogRead, true, - page_id_t(group->space_id, static_cast( - header / univ_page_size.physical())), - univ_page_size, - static_cast(header % univ_page_size.physical()), - OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf, NULL); + dberr_t err; + + err = fil_redo_io( + IORequestLogRead, + page_id_t(group->space_id, static_cast( + header / univ_page_size.physical())), + univ_page_size, + static_cast(header % univ_page_size.physical()), + OS_FILE_LOG_BLOCK_SIZE, log_sys->checkpoint_buf); + + ut_a(err == DB_SUCCESS); } /** Write checkpoint info to the log header and invoke log_mutex_exit(). @@ -2021,8 +2048,6 @@ log_checkpoint( if (recv_recovery_is_on()) { recv_apply_hashed_log_recs(true); - } else { - fil_tablespace_open_sync_to_disk(); } #ifdef UNIV_DEBUG @@ -2044,7 +2069,9 @@ log_checkpoint( log_mutex_exit(); } -#ifndef _WIN32 +#ifdef _WIN32 + fil_flush_file_spaces(to_int(FIL_TYPE_TABLESPACE)); +#else switch (srv_unix_file_flush_method) { case SRV_UNIX_NOSYNC: break; @@ -2510,18 +2537,18 @@ logs_empty_and_mark_files_at_shutdown(void) if (srv_downgrade_logs) { ut_ad(!srv_read_only_mode); log_downgrade(); - fil_flush_file_spaces(FIL_TYPE_LOG); + fil_flush_file_redo(); } if (!srv_read_only_mode) { - fil_write_flushed_lsn(lsn); + dberr_t err; + + err = fil_write_flushed_lsn(lsn); + ut_a(err == DB_SUCCESS); } fil_close_all_files(); - /* Safe to truncate the tablespace.open.* files now. */ - fil_tablespace_open_clear(); - /* Stop Archiver background thread. */ count = 0; while (archiver_is_active) { diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 51a7af2607e9..5acf5204b531 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -34,6 +34,7 @@ Created 9/20/1997 Heikki Tuuri #include #include #include +#include #include "log0recv.h" #include "btr0btr.h" @@ -260,9 +261,7 @@ recv_calc_lsn_on_data_add( /** Destructor */ MetadataRecover::~MetadataRecover() { - PersistentTables::iterator iter; - - for (auto table : m_tables) { + for (auto& table : m_tables) { UT_DELETE(table.second); } @@ -281,6 +280,7 @@ MetadataRecover::getMetadata( if (iter == m_tables.end()) { metadata = UT_NEW_NOKEY(PersistentTableMetadata(id, 0)); + m_tables.insert(std::make_pair(id, metadata)); } else { metadata = iter->second; @@ -352,7 +352,8 @@ MetadataRecover::apply() PersistentTableMetadata*metadata = iter->second; dict_table_t* table; - table = dd_table_open_on_id(table_id, nullptr, nullptr, false, true); + table = dd_table_open_on_id( + table_id, nullptr, nullptr, false, true); /* If the table is nullptr, it might be already dropped */ if (table == nullptr) { @@ -403,42 +404,6 @@ MetadataRecover::apply() } } -#ifndef UNIV_HOTBACKUP -/** Store the collected persistent dynamic metadata to -mysql.innodb_dynamic_metadata */ -void -MetadataRecover::store() -{ - ut_ad(dict_sys->dynamic_metadata != nullptr); - ut_ad(dict_persist->table_buffer != nullptr); - - DDTableBuffer* table_buffer = dict_persist->table_buffer; - - if (empty()) { - return; - } - - mutex_enter(&dict_persist->mutex); - - for (auto meta : m_tables) { - table_id_t table_id = meta.first; - PersistentTableMetadata*metadata = meta.second; - byte buffer[REC_MAX_DATA_SIZE]; - ulint size; - - size = dict_persist->persisters->write(*metadata, buffer); - - dberr_t error = table_buffer->replace( - table_id, metadata->get_version(), buffer, size); - if (error != DB_SUCCESS) { - ut_ad(0); - } - } - - mutex_exit(&dict_persist->mutex); -} -#endif /* !UNIV_HOTBACKUP */ - /** Creates the recovery system. */ void recv_sys_create() @@ -517,9 +482,7 @@ recv_sys_close() if (recv_sys->flush_end != nullptr) { os_event_destroy(recv_sys->flush_end); } -#endif /* !UNIV_HOTBACKUP */ -#ifndef UNIV_HOTBACKUP ut_ad(!recv_writer_thread_active); mutex_free(&recv_sys->writer_mutex); #endif /* !UNIV_HOTBACKUP */ @@ -639,62 +602,6 @@ recv_report_corrupt_log( return(true); } -#ifndef UNIV_HOTBACKUP -/** recv_writer thread tasked with flushing dirty pages from the buffer -pools. */ -static -void -recv_writer_thread() -{ - ut_ad(!srv_read_only_mode); - - /* The code flow is as follows: - Step 1: In recv_recovery_from_checkpoint_start(). - Step 2: This recv_writer thread is started. - Step 3: In recv_recovery_from_checkpoint_finish(). - Step 4: Wait for recv_writer thread to complete. This is based - on the flag recv_writer_thread_active. - Step 5: Assert that recv_writer thread is not active anymore. - - It is possible that the thread that is started in step 2, - becomes active only after step 4 and hence the assert in - step 5 fails. So mark this thread active only if necessary. */ - mutex_enter(&recv_sys->writer_mutex); - - if (recv_recovery_on) { - recv_writer_thread_active = true; - } else { - mutex_exit(&recv_sys->writer_mutex); - return; - } - mutex_exit(&recv_sys->writer_mutex); - - while (srv_shutdown_state == SRV_SHUTDOWN_NONE) { - - os_thread_sleep(100000); - - mutex_enter(&recv_sys->writer_mutex); - - if (!recv_recovery_on) { - mutex_exit(&recv_sys->writer_mutex); - break; - } - - /* Flush pages from end of LRU if required */ - os_event_reset(recv_sys->flush_end); - recv_sys->flush_type = BUF_FLUSH_LRU; - os_event_set(recv_sys->flush_start); - os_event_wait(recv_sys->flush_end); - - mutex_exit(&recv_sys->writer_mutex); - } - - recv_writer_thread_active = false; - - my_thread_end(); -} -#endif /* !UNIV_HOTBACKUP */ - /** Inits the recovery system for a recovery operation. @param[in] max_mem Available memory in bytes */ void @@ -794,141 +701,290 @@ recv_sys_empty_hash() recv_sys->spaces = UT_NEW(Spaces(), mem_log_recv_space_hash_key); } +/** Check the consistency of a log header block. +@param[in] buf header block +@return true if ok */ #ifndef UNIV_HOTBACKUP -/** Frees the recovery system. */ -void -recv_sys_free() +static +#endif /* !UNIV_HOTBACKUP */ +bool +recv_check_log_header_checksum(const byte* buf) { - mutex_enter(&recv_sys->mutex); + auto c1 = log_block_get_checksum(buf); + auto c2 = log_block_calc_checksum_crc32(buf); - recv_sys_finish(); + return(c1 == c2); +} - /* wake page cleaner up to progress */ - if (!srv_read_only_mode) { - ut_ad(!recv_recovery_on); - ut_ad(!recv_writer_thread_active); - os_event_reset(buf_flush_event); - os_event_set(recv_sys->flush_start); - } +/** Check the 4-byte checksum to the trailer checksum field of a log +block. +@param[in] block pointer to a log block +@return whether the checksum matches */ +static +bool +log_block_checksum_is_ok( + const byte* block) +{ + return(!innodb_log_checksums + || log_block_get_checksum(block) + == log_block_calc_checksum(block)); +} - /* Free encryption data structures. */ - if (recv_sys->keys != nullptr) { +/** Get the page map for a tablespace. It will create one if one isn't found. +@param[in] space_id Tablespace ID for which page map required. +@param[in] create false if lookup only +@return the space data or null if not found */ +static +recv_sys_t::Space* +recv_get_page_map(space_id_t space_id, bool create) +{ + auto it = recv_sys->spaces->find(space_id); - for (auto& key : *recv_sys->keys) { + if (it != recv_sys->spaces->end()) { - if (key.ptr != nullptr) { - ut_free(key.ptr); - key.ptr = nullptr; - } + return(&it->second); - if (key.iv != nullptr) { - ut_free(key.iv); - key.iv = nullptr; - } - } + } else if (create) { - recv_sys->keys->swap(*recv_sys->keys); + mem_heap_t* heap; - UT_DELETE(recv_sys->keys); - recv_sys->keys = nullptr; + heap = mem_heap_create_typed(256, MEM_HEAP_FOR_RECV_SYS); + + using Space = recv_sys_t::Space; + using value_type = recv_sys_t::Spaces::value_type; + + auto where = recv_sys->spaces->insert( + it, value_type(space_id, Space(heap))); + + return(&where->second); } - mutex_exit(&recv_sys->mutex); + return(nullptr); } -/** Copies a log segment from the most up-to-date log group to the other log -groups, so that they all contain the latest log data. Also writes the info -about the latest checkpoint to the groups, and inits the fields in the group -memory structs to up-to-date values. */ +/** Gets the list of log records for a . +@param[in] space_id Tablespace ID +@param[in] page_no Page number +@return the redo log entries or nullptr if not found */ static -void -recv_synchronize_groups() +recv_addr_t* +recv_get_rec( + space_id_t space_id, + page_no_t page_no) { - lsn_t recovered_lsn; - - recovered_lsn = recv_sys->recovered_lsn; - - /* Read the last recovered log block to the recovery system buffer: - the block is always incomplete */ - - lsn_t start_lsn = ut_uint64_align_down( - recovered_lsn, OS_FILE_LOG_BLOCK_SIZE); - - lsn_t end_lsn; + recv_sys_t::Space* space; - end_lsn = ut_uint64_align_up(recovered_lsn, OS_FILE_LOG_BLOCK_SIZE); + space = recv_get_page_map(space_id, false); - ut_a(start_lsn != end_lsn); + if (space != nullptr) { - recv_read_log_seg( - recv_sys->last_block, - UT_LIST_GET_FIRST(log_sys->log_groups), start_lsn, end_lsn); + auto it = space->m_pages.find(page_no); - for (log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); - group; - group = UT_LIST_GET_NEXT(log_groups, group)) { - /* Update the fields in the group struct to correspond to - recovered_lsn */ + if (it != space->m_pages.end()) { - log_group_set_fields(group, recovered_lsn); + return(it->second); + } } - /* Copy the checkpoint info to the log; remember that we have - incremented checkpoint_no by one, and the info will not be written - over the max checkpoint info, thus making the preservation of max - checkpoint info on disk certain */ - - log_write_checkpoint_info(true); - log_mutex_enter(); + return(nullptr); } -#endif /* !UNIV_HOTBACKUP */ -/** Check the consistency of a log header block. -@param[in] buf header block -@return true if ok */ #ifndef UNIV_HOTBACKUP -static -#endif /* !UNIV_HOTBACKUP */ -bool -recv_check_log_header_checksum(const byte* buf) +/** Store the collected persistent dynamic metadata to +mysql.innodb_dynamic_metadata */ +void +MetadataRecover::store() { - ulint block_checksum = log_block_get_checksum(buf); - ulint calc_checksum = log_block_calc_checksum_crc32(buf); -#ifdef UNIV_DEBUG - if (block_checksum != calc_checksum) { - ib::info() - << "log header checksum mismatch, found " - << block_checksum << " should be " << calc_checksum; - } -#endif /* UNIV_DEBUG */ - return(block_checksum == calc_checksum); -} + ut_ad(dict_sys->dynamic_metadata != nullptr); + ut_ad(dict_persist->table_buffer != nullptr); -#ifndef UNIV_HOTBACKUP -/** Copy of the LOG_HEADER_CREATOR field. */ -static char log_header_creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR + 1]; + DDTableBuffer* table_buffer = dict_persist->table_buffer; -/** Determine if a redo log from MySQL 5.7.9 is clean. -@param[in] lsn checkpoint LSN -@return error code -@retval DB_SUCCESS if the redo log is clean -@retval DB_ERROR if the redo log is corrupted or dirty */ -static -dberr_t -recv_log_recover_5_7(lsn_t lsn) -{ - log_mutex_enter(); + if (empty()) { + return; + } - log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); - lsn_t source_offset = log_group_calc_lsn_offset(lsn, group); + mutex_enter(&dict_persist->mutex); - log_mutex_exit(); + for (auto meta : m_tables) { + table_id_t table_id = meta.first; + PersistentTableMetadata*metadata = meta.second; + byte buffer[REC_MAX_DATA_SIZE]; + ulint size; - page_no_t page_no; + size = dict_persist->persisters->write(*metadata, buffer); + + dberr_t error = table_buffer->replace( + table_id, metadata->get_version(), buffer, size); + if (error != DB_SUCCESS) { + ut_ad(0); + } + } + + mutex_exit(&dict_persist->mutex); +} + +/** recv_writer thread tasked with flushing dirty pages from the buffer +pools. */ +static +void +recv_writer_thread() +{ + ut_ad(!srv_read_only_mode); + + /* The code flow is as follows: + Step 1: In recv_recovery_from_checkpoint_start(). + Step 2: This recv_writer thread is started. + Step 3: In recv_recovery_from_checkpoint_finish(). + Step 4: Wait for recv_writer thread to complete. This is based + on the flag recv_writer_thread_active. + Step 5: Assert that recv_writer thread is not active anymore. + + It is possible that the thread that is started in step 2, + becomes active only after step 4 and hence the assert in + step 5 fails. So mark this thread active only if necessary. */ + mutex_enter(&recv_sys->writer_mutex); + + if (recv_recovery_on) { + recv_writer_thread_active = true; + } else { + mutex_exit(&recv_sys->writer_mutex); + return; + } + mutex_exit(&recv_sys->writer_mutex); + + while (srv_shutdown_state == SRV_SHUTDOWN_NONE) { + + os_thread_sleep(100000); + + mutex_enter(&recv_sys->writer_mutex); + + if (!recv_recovery_on) { + mutex_exit(&recv_sys->writer_mutex); + break; + } + + /* Flush pages from end of LRU if required */ + os_event_reset(recv_sys->flush_end); + recv_sys->flush_type = BUF_FLUSH_LRU; + os_event_set(recv_sys->flush_start); + os_event_wait(recv_sys->flush_end); + + mutex_exit(&recv_sys->writer_mutex); + } + + recv_writer_thread_active = false; + + my_thread_end(); +} + +/** Frees the recovery system. */ +void +recv_sys_free() +{ + mutex_enter(&recv_sys->mutex); + + recv_sys_finish(); + + /* wake page cleaner up to progress */ + if (!srv_read_only_mode) { + ut_ad(!recv_recovery_on); + ut_ad(!recv_writer_thread_active); + os_event_reset(buf_flush_event); + os_event_set(recv_sys->flush_start); + } + + /* Free encryption data structures. */ + if (recv_sys->keys != nullptr) { + + for (auto& key : *recv_sys->keys) { + + if (key.ptr != nullptr) { + ut_free(key.ptr); + key.ptr = nullptr; + } + + if (key.iv != nullptr) { + ut_free(key.iv); + key.iv = nullptr; + } + } + + recv_sys->keys->swap(*recv_sys->keys); + + UT_DELETE(recv_sys->keys); + recv_sys->keys = nullptr; + } + + mutex_exit(&recv_sys->mutex); +} + +/** Copies a log segment from the most up-to-date log group to the other log +groups, so that they all contain the latest log data. Also writes the info +about the latest checkpoint to the groups, and inits the fields in the group +memory structs to up-to-date values. */ +static +void +recv_synchronize_groups() +{ + lsn_t recovered_lsn; + + recovered_lsn = recv_sys->recovered_lsn; + + /* Read the last recovered log block to the recovery system buffer: + the block is always incomplete */ + + lsn_t start_lsn = ut_uint64_align_down( + recovered_lsn, OS_FILE_LOG_BLOCK_SIZE); + + lsn_t end_lsn; + + end_lsn = ut_uint64_align_up(recovered_lsn, OS_FILE_LOG_BLOCK_SIZE); + + ut_a(start_lsn != end_lsn); + + recv_read_log_seg( + recv_sys->last_block, + UT_LIST_GET_FIRST(log_sys->log_groups), start_lsn, end_lsn); + + for (log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); + group; + group = UT_LIST_GET_NEXT(log_groups, group)) { + /* Update the fields in the group struct to correspond to + recovered_lsn */ + + log_group_set_fields(group, recovered_lsn); + } + + /* Copy the checkpoint info to the log; remember that we have + incremented checkpoint_no by one, and the info will not be written + over the max checkpoint info, thus making the preservation of max + checkpoint info on disk certain */ + + log_write_checkpoint_info(true); + log_mutex_enter(); +} + +/** Copy of the LOG_HEADER_CREATOR field. */ +static char log_header_creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR + 1]; + +/** Determine if a redo log from a version before MySQL 8.0.3 is clean. +@param[in] lsn checkpoint LSN +@return error code +@retval DB_SUCCESS if the redo log is clean +@retval DB_ERROR if the redo log is corrupted or dirty */ +static +dberr_t +recv_log_recover_pre_8_0_4(lsn_t lsn) +{ + log_group_t* group = UT_LIST_GET_FIRST(log_sys->log_groups); + lsn_t source_offset = log_group_calc_lsn_offset(lsn, group); + + page_no_t page_no; page_no = (page_no_t) (source_offset / univ_page_size.physical()); - byte* buf = log_sys->buf; + byte* buf = log_sys->buf; static const char* NO_UPGRADE_RECOVERY_MSG = "Upgrade after a crash is not supported." @@ -938,12 +994,16 @@ recv_log_recover_5_7(lsn_t lsn) ". Please follow the instructions at " REFMAN "upgrading.html"; - fil_io(IORequestLogRead, true, - page_id_t(group->space_id, page_no), - univ_page_size, - (ulint) ((source_offset & ~(OS_FILE_LOG_BLOCK_SIZE - 1)) - % univ_page_size.physical()), - OS_FILE_LOG_BLOCK_SIZE, buf, nullptr); + dberr_t err; + + err = fil_redo_io( + IORequestLogRead, + page_id_t(group->space_id, page_no), univ_page_size, + (ulint) ((source_offset & ~(OS_FILE_LOG_BLOCK_SIZE - 1)) + % univ_page_size.physical()), + OS_FILE_LOG_BLOCK_SIZE, buf); + + ut_a(err == DB_SUCCESS); if (log_block_calc_checksum(buf) != log_block_get_checksum(buf)) { @@ -957,7 +1017,7 @@ recv_log_recover_5_7(lsn_t lsn) } /* On a clean shutdown, the redo log will be logically empty - after the checkpoint lsn. */ + after the checkpoint LSN. */ if (log_block_get_data_len(buf) != (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) { @@ -993,108 +1053,115 @@ recv_log_recover_5_7(lsn_t lsn) static MY_ATTRIBUTE((warn_unused_result)) dberr_t recv_find_max_checkpoint( - log_group_t** max_group, + log_group_t*& max_group, ulint* max_field) { - uint64_t max_no = 0; *max_field = 0; - *max_group = nullptr; + max_group = nullptr; - byte* buf = log_sys->checkpoint_buf; + /* We've never supported more than one group. */ + ut_a(UT_LIST_GET_LEN(log_sys->log_groups) == 1); - for (auto group = UT_LIST_GET_FIRST(log_sys->log_groups); - group != nullptr; - /* No op */ ) { + auto group = UT_LIST_GET_FIRST(log_sys->log_groups); - group->state = LOG_GROUP_CORRUPTED; + group->state = LOG_GROUP_CORRUPTED; - /* Read block into log_sys->checkpoint_buf (== buf) */ - log_group_header_read(group, 0); + log_group_header_read(group, 0); - /* Check the header page checksum. There was no - checksum in the first redo log format (version 0). */ - group->format = mach_read_from_4(buf + LOG_HEADER_FORMAT); + byte* buf = log_sys->checkpoint_buf; - if (group->format != 0 - && !recv_check_log_header_checksum(buf)) { + /* Check the header page checksum. There was no + checksum in the first redo log format (version 0). */ + group->format = mach_read_from_4(buf + LOG_HEADER_FORMAT); - ib::error() << "Invalid redo log header checksum."; + if (group->format != 0 && !recv_check_log_header_checksum(buf)) { - return(DB_CORRUPTION); - } + ib::error() << "Invalid redo log header checksum."; - memcpy(log_header_creator, buf + LOG_HEADER_CREATOR, - sizeof log_header_creator); + return(DB_CORRUPTION); + } - log_header_creator[(sizeof log_header_creator) - 1] = 0; + auto ptr = buf + LOG_HEADER_CREATOR; - switch (group->format) { - case 0: - ib::error() << "Unsupported redo log format." - " The redo log was created" - " before MySQL 5.7.9."; - return(DB_ERROR); + memcpy(log_header_creator, ptr, sizeof(log_header_creator)); - case LOG_HEADER_FORMAT_5_7_9: - /* The checkpoint page format is identical. */ + log_header_creator[sizeof(log_header_creator) - 1] = 0; - case LOG_HEADER_FORMAT_CURRENT: - break; + switch (group->format) { + case 0: + ib::error() + << "Unsupported redo log format (" + << group->format << "). The redo log was created" + << " before MySQL 5.7.9"; - default: - ib::error() << "Unsupported redo log format." - " The redo log was created" - " with " << log_header_creator << - ". Please follow the instructions at " - REFMAN "upgrading-downgrading.html"; - return(DB_ERROR); - } + return(DB_ERROR); - for (auto field = LOG_CHECKPOINT_1; - field <= LOG_CHECKPOINT_2; - field += LOG_CHECKPOINT_2 - LOG_CHECKPOINT_1) { - - log_group_header_read(group, field); - - if (!recv_check_log_header_checksum(buf)) { - DBUG_PRINT("ib_log", - ("invalid checkpoint," - " group " ULINTPF " at %d" - ", checksum %x", - group->id, field, - (unsigned) log_block_get_checksum( - buf))); - continue; - } + case LOG_HEADER_FORMAT_5_7_9: + case LOG_HEADER_FORMAT_8_0_1: + + ib::info() + << "Redo log format is v" << group->format + << ". The redo log was created before" + << " MySQL 8.0.3."; - group->state = LOG_GROUP_OK; + case LOG_HEADER_FORMAT_CURRENT: + /* The checkpoint page format is identical upto v3. */ + break; + + default: + ib::error() + << "Unknown redo log format (" + << group->format << ")." + << " Please follow the instructions at " + REFMAN "upgrading-downgrading.html"; + + return(DB_ERROR); + } - group->lsn = mach_read_from_8( - buf + LOG_CHECKPOINT_LSN); + uint64_t max_no = 0; + constexpr ulint CKP1 = LOG_CHECKPOINT_1; + constexpr ulint CKP2 = LOG_CHECKPOINT_2; - group->lsn_offset = mach_read_from_8( - buf + LOG_CHECKPOINT_OFFSET); + for (auto i = CKP1; i <= CKP2; i += CKP2 - CKP1) { - uint64_t checkpoint_no = mach_read_from_8( - buf + LOG_CHECKPOINT_NO); + log_group_header_read(group, i); + if (!recv_check_log_header_checksum(buf)) { DBUG_PRINT("ib_log", - ("checkpoint " UINT64PF " at " LSN_PF - " found in group " ULINTPF, - checkpoint_no, group->lsn, group->id)); - - if (checkpoint_no >= max_no) { - *max_group = group; - *max_field = field; - max_no = checkpoint_no; - } + ("invalid checkpoint," + " group " ULINTPF " at %d" + ", checksum %x", + group->id, (int) i, + (unsigned) log_block_get_checksum(buf))); + continue; } - group = UT_LIST_GET_NEXT(log_groups, group); + group->state = LOG_GROUP_OK; + + group->lsn = mach_read_from_8(buf + LOG_CHECKPOINT_LSN); + + ptr = buf + LOG_CHECKPOINT_OFFSET; + + group->lsn_offset = mach_read_from_8(ptr); + + ptr = buf + LOG_CHECKPOINT_NO; + + uint64_t checkpoint_no = mach_read_from_8(ptr); + + DBUG_PRINT("ib_log", + ("checkpoint " UINT64PF " at " LSN_PF + " found in group " ULINTPF, + checkpoint_no, group->lsn, group->id)); + + if (checkpoint_no >= max_no) { + *max_field = i; + max_group = group; + max_no = checkpoint_no; + } } - if (*max_group == nullptr) { + if (max_group == nullptr) { /* Before 5.7.9, we could get here during database initialization if we created an ib_logfile0 file that @@ -1112,1948 +1179,1777 @@ recv_find_max_checkpoint( return(DB_SUCCESS); } -#else /* !UNIV_HOTBACKUP */ -/** Reads the checkpoint info needed in hot backup. -@param[in] hdr buffer containing the log group header -@param[out] lsn checkpoint lsn -@param[out] offset checkpoint offset in the log group -@param[out] cp_no checkpoint number -@param[out] first_header_lsn lsn of of the start of the first log file -@return true if success */ -bool -meb_read_checkpoint_info( - const byte* hdr, - lsn_t* lsn, - lsn_t* offset, - lsn_t* cp_no, - lsn_t* first_header_lsn) -{ - ulint max_cp = 0; - uint64_t max_cp_no = 0; - const byte* cp_buf = hdr + LOG_CHECKPOINT_1; - if (recv_check_log_header_checksum(cp_buf)) { - max_cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO); - max_cp = LOG_CHECKPOINT_1; - } +/** Reads in pages which have hashed log records, from an area around a given +page number. +@param[in] page_id Read the pages around this page number +@return number of pages found */ +static +ulint +recv_read_in_area(const page_id_t& page_id) +{ + page_no_t low_limit; - cp_buf = hdr + LOG_CHECKPOINT_2; + low_limit = page_id.page_no() + - (page_id.page_no() % RECV_READ_AHEAD_AREA); - if (recv_check_log_header_checksum(cp_buf)) { - if (mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO) > max_cp_no) { - max_cp = LOG_CHECKPOINT_2; - } - } + ulint n = 0; - if (max_cp == 0) { - return(false); - } + std::array page_nos; - cp_buf = hdr + max_cp; + for (page_no_t page_no = low_limit; + page_no < low_limit + RECV_READ_AHEAD_AREA; + ++page_no) { - *lsn = mach_read_from_8(cp_buf + LOG_CHECKPOINT_LSN); - *offset = mach_read_from_8(cp_buf + LOG_CHECKPOINT_OFFSET); + recv_addr_t* recv_addr; - *cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO); + recv_addr = recv_get_rec(page_id.space(), page_no); - *first_header_lsn = mach_read_from_8(hdr + LOG_HEADER_START_LSN); + const page_id_t cur_page_id(page_id.space(), page_no); - return(true); -} -#endif /* !UNIV_HOTBACKUP */ + if (recv_addr != nullptr && !buf_page_peek(cur_page_id)) { -/** Check the 4-byte checksum to the trailer checksum field of a log -block. -@param[in] block pointer to a log block -@return whether the checksum matches */ -static -bool -log_block_checksum_is_ok( - const byte* block) -{ - return(!innodb_log_checksums - || log_block_get_checksum(block) - == log_block_calc_checksum(block)); + mutex_enter(&recv_sys->mutex); + + if (recv_addr->state == RECV_NOT_PROCESSED) { + + recv_addr->state = RECV_BEING_READ; + + page_nos[n] = page_no; + + ++n; + } + + mutex_exit(&recv_sys->mutex); + } + } + + buf_read_recv_pages(false, page_id.space(), &page_nos[0], n); + + return(n); } -#ifdef UNIV_HOTBACKUP -/** Scans the log segment and n_bytes_scanned is set to the length of valid -log scanned. -@param[in] buf buffer containing log data -@param[in] buf_len data length in that buffer -@param[in,out] scanned_lsn LSN of buffer start, we return scanned -lsn -@param[in,out] scanned_checkpoint_no 4 lowest bytes of the highest scanned -checkpoint number so far -@param[out] n_bytes_scanned how much we were able to scan, smaller -than buf_len if log data ended here */ +/** Apply the log records to a page +@param[in,out] recv_addr Redo log records to apply */ +static void -meb_scan_log_seg( - byte* buf, - ulint buf_len, - lsn_t* scanned_lsn, - ulint* scanned_checkpoint_no, - ulint* n_bytes_scanned) +recv_apply_log_rec(recv_addr_t* recv_addr) { - *n_bytes_scanned = 0; + if (recv_addr->state == RECV_DISCARDED) { + ut_a(recv_sys->n_addrs > 0); + --recv_sys->n_addrs; + return; + } - for (auto log_block = buf; - log_block < buf + buf_len; - log_block += OS_FILE_LOG_BLOCK_SIZE) { + bool found; + const page_id_t page_id(recv_addr->space, recv_addr->page_no); - ulint no = log_block_get_hdr_no(log_block); + const page_size_t page_size = + fil_space_get_page_size(recv_addr->space, &found); - if (no != log_block_convert_lsn_to_no(*scanned_lsn) - || !log_block_checksum_is_ok(log_block)) { + if (!found + || recv_sys->missing_ids.find(recv_addr->space) + != recv_sys->missing_ids.end()) { - ib::trace_2() - << "Scanned lsn: " - << *scanned_lsn - << " header no: " - << no - << " converted no: " - << log_block_convert_lsn_to_no(*scanned_lsn) - << " checksum: " - << log_block_checksum_is_ok(log_block) - << " block cp no: " - << log_block_get_checkpoint_no(log_block); + /* Tablespace was discarded or dropped after changes were + made to it. Or, we have ignored redo log for this tablespace + earlier and somehow it has been found now. We can't apply + this redo log out of order. */ - /* Garbage or an incompletely written log block */ + recv_addr->state = RECV_PROCESSED; - log_block += OS_FILE_LOG_BLOCK_SIZE; - break; + ut_a(recv_sys->n_addrs > 0); + --recv_sys->n_addrs; + + /* If the tablespace has been explicitly deleted, we + can safely ignore it. */ + + if (recv_sys->deleted.find(recv_addr->space) + == recv_sys->deleted.end()) { + + recv_sys->missing_ids.insert(recv_addr->space); } - if (*scanned_checkpoint_no > 0 - && log_block_get_checkpoint_no(log_block) - < *scanned_checkpoint_no - && *scanned_checkpoint_no - - log_block_get_checkpoint_no(log_block) - > 0x80000000UL) { + } else if (recv_addr->state == RECV_NOT_PROCESSED) { - /* Garbage from a log buffer flush which was made - before the most recent database recovery */ + mutex_exit(&recv_sys->mutex); - ib::trace_2() - << "Scanned cp no: " - << *scanned_checkpoint_no - << " block cp no " - << log_block_get_checkpoint_no(log_block); + if (buf_page_peek(page_id)) { - break; - } + mtr_t mtr; - ulint data_len = log_block_get_data_len(log_block); + mtr_start(&mtr); - *scanned_checkpoint_no - = log_block_get_checkpoint_no(log_block); - *scanned_lsn += data_len; + buf_block_t* block; - *n_bytes_scanned += data_len; + block = buf_page_get( + page_id, page_size, RW_X_LATCH, &mtr); - if (data_len < OS_FILE_LOG_BLOCK_SIZE) { - /* Log data ends here */ + buf_block_dbg_add_level(block, SYNC_NO_ORDER_CHECK); - break; + recv_recover_page(false, block); + + mtr_commit(&mtr); + + } else { + recv_read_in_area(page_id); } + + mutex_enter(&recv_sys->mutex); } } -#endif /* UNIV_HOTBACKUP */ -/** Parse or process a write encryption info record. -@param[in] ptr redo log record -@param[in] end end of the redo log buffer -@param[in] space_id the tablespace ID -@return log record end, nullptr if not a complete record */ -static -byte* -fil_write_encryption_parse( - byte* ptr, - const byte* end, - space_id_t space_id) +/** Empties the hash table of stored log records, applying them to appropriate +pages. +@param[in] allow_ibuf if true, ibuf operations are allowed during + the application; if false, no ibuf operations + are allowed, and after the application all + file pages are flushed to disk and invalidated + in buffer pool: this alternative means that + no new log records can be generated during + the application; the caller must in this case + own the log mutex */ +void +recv_apply_hashed_log_recs(bool allow_ibuf) { - byte* iv = nullptr; - byte* key = nullptr; - bool is_new = false; - - fil_space_t* space = fil_space_get(space_id); + for (;;) { - if (space == nullptr) { + mutex_enter(&recv_sys->mutex); - if (recv_sys->keys == nullptr) { + if (!recv_sys->apply_batch_on) { - recv_sys->keys = UT_NEW_NOKEY( - recv_sys_t::Encryption_Keys()); + break; } - for (auto& recv_key : *recv_sys->keys) { + mutex_exit(&recv_sys->mutex); - if (recv_key.space_id == space_id) { - iv = recv_key.iv; - key = recv_key.ptr; - } - } + os_thread_sleep(500000); + } - if (key == nullptr) { + ut_ad(!allow_ibuf == log_mutex_own()); - key = static_cast( - ut_malloc_nokey(ENCRYPTION_KEY_LEN)); + if (!allow_ibuf) { + recv_no_ibuf_operations = true; + } - iv = static_cast( - ut_malloc_nokey(ENCRYPTION_KEY_LEN)); + recv_sys->apply_log_recs = true; + recv_sys->apply_batch_on = true; - is_new = true; - } + auto batch_size = recv_sys->n_addrs; - } else { - iv = space->encryption_iv; - key = space->encryption_key; + ib::info() + << "Applying a batch of " + << batch_size + << " redo log records ..."; + + static const size_t PCT = 10; + + size_t pct = PCT; + size_t applied = 0; + auto unit = batch_size / PCT; + + if (unit <= PCT) { + pct = 100; + unit = batch_size; } - ulint offset; + auto start_time = ut_time(); - offset = mach_read_from_2(ptr); - ptr += 2; + for (const auto& space : *recv_sys->spaces) { - ulint len; + bool dropped; - len = mach_read_from_2(ptr); - ptr += 2; + if (space.first != TRX_SYS_SPACE + && !fil_tablespace_open_for_recovery(space.first)) { - if (end < ptr + len) { - return(nullptr); - } + /* Tablespace was dropped. */ + ut_ad(!fil_tablespace_lookup_for_recovery(space.first)); + + dropped = true; + } else { + dropped = false; + } - if (offset >= UNIV_PAGE_SIZE - || len + offset > UNIV_PAGE_SIZE - || (len != ENCRYPTION_INFO_SIZE_V1 - && len != ENCRYPTION_INFO_SIZE_V2)) { + for (auto pages : space.second.m_pages) { - recv_sys->found_corrupt_log = true; - return(nullptr); - } + ut_ad(pages.second->space == space.first); -#ifdef UNIV_HOTBACKUP - if (fil_space_get(space_id) - && !meb_get_encryption_key(space_id, key, iv)) { - recv_sys->found_corrupt_log = TRUE; - ib::fatal() - << "Encryption information" - << " in the redo log of space " - << space_id << " is invalid" - << "MEB cannot proceed with the operation."; - } -#else /* UNIV_HOTBACKUP */ - if (!Encryption::decode_encryption_info(key, iv, ptr)) { + if (dropped) { + pages.second->state = RECV_DISCARDED; + } - recv_sys->found_corrupt_log = true; + recv_apply_log_rec(pages.second); - ib::warn() - << "Encryption information" - << " in the redo log of space " - << space_id << " is invalid"; - } -#endif /* UNIV_HOTBACKUP */ + ++applied; - ut_ad(len == ENCRYPTION_INFO_SIZE_V1 - || len == ENCRYPTION_INFO_SIZE_V2); + if (unit == 0 || (applied % unit) == 0) { - ptr += len; + ib::info() << pct << "%"; - if (space == nullptr) { + pct += PCT; - if (is_new) { + start_time = ut_time(); - recv_sys_t::Encryption_Key new_key; + } else if (ut_time() - start_time + >= PRINT_INTERVAL_SECS) { - new_key.iv = iv; - new_key.ptr = key; - new_key.space_id = space_id; + start_time = ut_time(); - recv_sys->keys->push_back(new_key); + ib::info() + << std::setprecision(2) + << ((double) applied * 100) + / (double) batch_size << "%"; + } } + } - } else { - ut_ad(FSP_FLAGS_GET_ENCRYPTION(space->flags)); + /* Wait until all the pages have been processed */ + + while (recv_sys->n_addrs != 0) { + + mutex_exit(&recv_sys->mutex); + + os_thread_sleep(500000); - space->encryption_type = Encryption::AES; - space->encryption_klen = ENCRYPTION_KEY_LEN; + mutex_enter(&recv_sys->mutex); } - return(ptr); -} + if (!allow_ibuf) { -/** Try to parse a single log record body and also applies it if -specified. -@param[in] type redo log entry type -@param[in] ptr redo log record body -@param[in] end_ptr end of buffer -@param[in] space_id tablespace identifier -@param[in] page_no page number -@param[in,out] block buffer block, or nullptr if - a page log record should not be applied - or if it is a MLOG_FILE_ operation -@param[in,out] mtr mini-transaction, or nullptr if - a page log record should not be applied -@param[in] parsed_bytes Number of bytes parsed so far -@return log record end, nullptr if not a complete record */ -static -byte* -recv_parse_or_apply_log_rec_body( - mlog_id_t type, - byte* ptr, - byte* end_ptr, - space_id_t space_id, - page_no_t page_no, - buf_block_t* block, - mtr_t* mtr, - ulint parsed_bytes) -{ - ut_ad(!block == !mtr); - - switch (type) { - case MLOG_FILE_OPEN: - ut_ad(parsed_bytes != ULINT_UNDEFINED); - // Fall through - case MLOG_FILE_DELETE: - case MLOG_FILE_CREATE2: - case MLOG_FILE_RENAME2: - - ut_ad(block == nullptr); - - return(fil_tablespace_name_recover( - ptr, end_ptr, page_id_t(space_id, page_no), type, - parsed_bytes)); - - case MLOG_INDEX_LOAD: -#ifdef UNIV_HOTBACKUP - /* While scaning redo logs during backup phase a - MLOG_INDEX_LOAD type redo log record indicates a DDL - (create index, alter table...)is performed with - 'algorithm=inplace'. This redo log indicates that - - 1. The DDL was started after MEB started backing up, in which - case MEB will not be able to take a consistent backup and should - fail. or - 2. There is a possibility of this record existing in the REDO - even after the completion of the index create operation. This is - because of InnoDB does not checkpointing after the flushing the - index pages. - - If MEB gets the last_redo_flush_lsn and that is less than the - lsn of the current record MEB fails the backup process. - Error out in case of online backup and emit a warning in case - of offline backup and continue. */ - if (!recv_recovery_on) { - if (is_online_redo_copy) { - if (backup_redo_log_flushed_lsn - < recv_sys->recovered_lsn) { - ib::trace() - << "Last flushed lsn: " - << backup_redo_log_flushed_lsn - << " load_index lsn " - << recv_sys->recovered_lsn; - - if (backup_redo_log_flushed_lsn == 0) { - ib::error() - << "MEB was not able" - << " to determine the" - << " InnoDB Engine" - << " Status"; - } - - ib::fatal() - << "An optimized(without" - << " redo logging) DDL" - << " operation has been" - << " performed. All modified" - << " pages may not have been" - << " flushed to the disk yet.\n" - << " MEB will not be able to" - << " take a consistent backup." - << " Retry the backup" - << " operation"; - } - /** else the index is flushed to disk before - backup started hence no error */ - } else { - /* offline backup */ - ib::trace() - << "Last flushed lsn: " - << backup_redo_log_flushed_lsn - << " load_index lsn " - << recv_sys->recovered_lsn; - - ib::warn() - << "An optimized(without redo" - << " logging) DDL operation has been" - << " performed. All modified pages may" - << " not have been flushed to the disk" - << " yet.\n This offline backup may" - << " not be consistent"; - } - } -#endif /* UNIV_HOTBACKUP */ - if (end_ptr < ptr + 8) { - - return(nullptr); - } - - return(ptr + 8); - - case MLOG_WRITE_STRING: - -#ifdef UNIV_HOTBACKUP - if (recv_recovery_on) { -#endif /* UNIV_HOTBACKUP */ - /* For encrypted tablespace, we need to get the - encryption key information before the page 0 is recovered. - Otherwise, redo will not find the key to decrypt - the data pages. */ - - if (page_no == 0 - && !fsp_is_system_or_temp_tablespace(space_id)) { - - return(fil_write_encryption_parse( - ptr, end_ptr, space_id)); - } -#ifdef UNIV_HOTBACKUP - } -#endif /* UNIV_HOTBACKUP */ - - break; - - default: - break; - } - - page_t* page; - page_zip_des_t* page_zip; - dict_index_t* index = nullptr; - -#ifdef UNIV_DEBUG - ulint page_type; -#endif /* UNIV_DEBUG */ - -#ifdef UNIV_HOTBACKUP -#ifdef UNIV_DEBUG - ib::trace_3() - << "recv_parse_or_apply_log_rec_body { type: " - << get_mlog_string(type) - << ", space_id: " << space_id - << ", page_no: " << page_no - << ", ptr : " << static_cast (ptr) - << ", end_ptr: " << static_cast(end_ptr) - << ", block: " << static_cast(block) - << ", mtr: " << static_cast(mtr) << " }"; -#endif /* UNIV_DEBUG */ -#endif /* UNIV_HOTBACKUP */ - - if (block != nullptr) { - - /* Applying a page log record. */ - - page = block->frame; - page_zip = buf_block_get_page_zip(block); - - ut_d(page_type = fil_page_get_type(page)); -#ifdef UNIV_HOTBACKUP -#ifdef UNIV_DEBUG - if (page_type == 0) { - meb_print_page_header(page); - } -#endif /* UNIV_DEBUG */ -#endif /* UNIV_HOTBACKUP */ - - } else { - - /* Parsing a page log record. */ - page = nullptr; - page_zip = nullptr; - - ut_d(page_type = FIL_PAGE_TYPE_ALLOCATED); - } - - const byte* old_ptr = ptr; - - switch (type) { -#ifdef UNIV_LOG_LSN_DEBUG - case MLOG_LSN: - /* The LSN is checked in recv_parse_log_rec(). */ - break; -#endif /* UNIV_LOG_LSN_DEBUG */ - case MLOG_4BYTES: - - ut_ad(page == nullptr || end_ptr > ptr + 2); - - /* Most FSP flags can only be changed by CREATE or ALTER with - ALGORITHM=COPY, so they do not change once the file - is created. The SDI flag is the only one that can be - changed by a recoverable transaction. So if there is - change in FSP flags, update the in-memory space structure - (fil_space_t) */ - - if (page != nullptr - && page_no == 0 - && mach_read_from_2(ptr) - == FSP_HEADER_OFFSET + FSP_SPACE_FLAGS) { - - ptr = mlog_parse_nbytes( - MLOG_4BYTES, ptr, end_ptr, page, page_zip); - - /* When applying log, we have complete records. - They can be incomplete (ptr=nullptr) only during - scanning (page==nullptr) */ - - ut_ad(ptr != nullptr); - - fil_space_t* space = fil_space_acquire(space_id); - - ut_ad(space != nullptr); - - fil_space_set_flags( - space, mach_read_from_4( - FSP_HEADER_OFFSET - + FSP_SPACE_FLAGS - + page)); - - fil_space_release(space); - - break; - } - - // fall through - - case MLOG_1BYTE: - case MLOG_2BYTES: - case MLOG_8BYTES: -#ifdef UNIV_DEBUG - if (page - && page_type == FIL_PAGE_TYPE_ALLOCATED - && end_ptr >= ptr + 2) { - - /* It is OK to set FIL_PAGE_TYPE and certain - list node fields on an empty page. Any other - write is not OK. */ - - /* NOTE: There may be bogus assertion failures for - dict_hdr_create(), trx_rseg_header_create(), - trx_sys_create_doublewrite_buf(), and - trx_sysf_create(). - These are only called during database creation. */ - - ulint offs = mach_read_from_2(ptr); - - switch (type) { - default: - ut_error; - case MLOG_2BYTES: - /* Note that this can fail when the - redo log been written with something - older than InnoDB Plugin 1.0.4. */ - ut_ad(offs == FIL_PAGE_TYPE - || offs == IBUF_TREE_SEG_HEADER - + IBUF_HEADER + FSEG_HDR_OFFSET - || offs == PAGE_BTR_IBUF_FREE_LIST - + PAGE_HEADER + FIL_ADDR_BYTE - || offs == PAGE_BTR_IBUF_FREE_LIST - + PAGE_HEADER + FIL_ADDR_BYTE - + FIL_ADDR_SIZE - || offs == PAGE_BTR_SEG_LEAF - + PAGE_HEADER + FSEG_HDR_OFFSET - || offs == PAGE_BTR_SEG_TOP - + PAGE_HEADER + FSEG_HDR_OFFSET - || offs == PAGE_BTR_IBUF_FREE_LIST_NODE - + PAGE_HEADER + FIL_ADDR_BYTE - + 0 /*FLST_PREV*/ - || offs == PAGE_BTR_IBUF_FREE_LIST_NODE - + PAGE_HEADER + FIL_ADDR_BYTE - + FIL_ADDR_SIZE /*FLST_NEXT*/); - break; - case MLOG_4BYTES: - /* Note that this can fail when the - redo log been written with something - older than InnoDB Plugin 1.0.4. */ - ut_ad(0 - || offs == IBUF_TREE_SEG_HEADER - + IBUF_HEADER + FSEG_HDR_SPACE - || offs == IBUF_TREE_SEG_HEADER - + IBUF_HEADER + FSEG_HDR_PAGE_NO - || offs == PAGE_BTR_IBUF_FREE_LIST - + PAGE_HEADER/* flst_init */ - || offs == PAGE_BTR_IBUF_FREE_LIST - + PAGE_HEADER + FIL_ADDR_PAGE - || offs == PAGE_BTR_IBUF_FREE_LIST - + PAGE_HEADER + FIL_ADDR_PAGE - + FIL_ADDR_SIZE - || offs == PAGE_BTR_SEG_LEAF - + PAGE_HEADER + FSEG_HDR_PAGE_NO - || offs == PAGE_BTR_SEG_LEAF - + PAGE_HEADER + FSEG_HDR_SPACE - || offs == PAGE_BTR_SEG_TOP - + PAGE_HEADER + FSEG_HDR_PAGE_NO - || offs == PAGE_BTR_SEG_TOP - + PAGE_HEADER + FSEG_HDR_SPACE - || offs == PAGE_BTR_IBUF_FREE_LIST_NODE - + PAGE_HEADER + FIL_ADDR_PAGE - + 0 /*FLST_PREV*/ - || offs == PAGE_BTR_IBUF_FREE_LIST_NODE - + PAGE_HEADER + FIL_ADDR_PAGE - + FIL_ADDR_SIZE /*FLST_NEXT*/); - break; - } - } -#endif /* UNIV_DEBUG */ - - ptr = mlog_parse_nbytes(type, ptr, end_ptr, page, page_zip); - - if (ptr != nullptr - && page != nullptr - && page_no == 0 && type == MLOG_4BYTES) { - - ulint offs = mach_read_from_2(old_ptr); - - switch (offs) { - fil_space_t* space; - uint32_t val; - default: - break; - - case FSP_HEADER_OFFSET + FSP_SPACE_FLAGS: - case FSP_HEADER_OFFSET + FSP_SIZE: - case FSP_HEADER_OFFSET + FSP_FREE_LIMIT: - case FSP_HEADER_OFFSET + FSP_FREE + FLST_LEN: - - space = fil_space_get(space_id); - - ut_a(space != nullptr); - - val = mach_read_from_4(page + offs); - - switch (offs) { - case FSP_HEADER_OFFSET + FSP_SPACE_FLAGS: - space->flags = val; - break; - - case FSP_HEADER_OFFSET + FSP_SIZE: - - space->size_in_header = val; - - if (space->size >= val) { - - break; - } - - ib::info() - << "Extending tablespace : " - << space->id - << " space name: " - << space->name - << " to new size: " << val - << " pages during recovery."; - - if (fil_space_extend(space, val)) { - - break; - } - - ib::error() - << "Could not extend tablespace" - << ": " << space->id << " space" - << " name: " << space->name - << " to new size: " << val - << " pages during recovery."; - - break; - - case FSP_HEADER_OFFSET + FSP_FREE_LIMIT: - space->free_limit = val; - break; - - case FSP_HEADER_OFFSET + FSP_FREE + FLST_LEN: - space->free_len = val; - ut_ad(val == flst_get_len(page + offs)); - break; - } - } - } - break; - - case MLOG_REC_INSERT: - case MLOG_COMP_REC_INSERT: - - ut_ad(!page || fil_page_type_is_index(page_type)); + /* Flush all the file pages to disk and invalidate them in + the buffer pool */ - if (nullptr != (ptr = mlog_parse_index( - ptr, end_ptr, - type == MLOG_COMP_REC_INSERT, - &index))) { - ut_a(!page - || (ibool)!!page_is_comp(page) - == dict_table_is_comp(index->table)); + ut_d(log_sys->disable_redo_writes = true); - ptr = page_cur_parse_insert_rec( - FALSE, ptr, end_ptr, block, index, mtr); - } + mutex_exit(&recv_sys->mutex); - break; + log_mutex_exit(); - case MLOG_REC_CLUST_DELETE_MARK: - case MLOG_COMP_REC_CLUST_DELETE_MARK: + /* Stop the recv_writer thread from issuing any LRU + flush batches. */ + mutex_enter(&recv_sys->writer_mutex); - ut_ad(!page || fil_page_type_is_index(page_type)); + /* Wait for any currently run batch to end. */ + buf_flush_wait_LRU_batch_end(); - if (nullptr != (ptr = mlog_parse_index( - ptr, end_ptr, - type == MLOG_COMP_REC_CLUST_DELETE_MARK, - &index))) { + os_event_reset(recv_sys->flush_end); - ut_a(!page - || (ibool)!!page_is_comp(page) - == dict_table_is_comp(index->table)); + recv_sys->flush_type = BUF_FLUSH_LIST; - ptr = btr_cur_parse_del_mark_set_clust_rec( - ptr, end_ptr, page, page_zip, index); - } + os_event_set(recv_sys->flush_start); - break; + os_event_wait(recv_sys->flush_end); - case MLOG_COMP_REC_SEC_DELETE_MARK: + buf_pool_invalidate(); - ut_ad(!page || fil_page_type_is_index(page_type)); + /* Allow batches from recv_writer thread. */ + mutex_exit(&recv_sys->writer_mutex); - /* This log record type is obsolete, but we process it for - backward compatibility with MySQL 5.0.3 and 5.0.4. */ + log_mutex_enter(); - ut_a(!page || page_is_comp(page)); - ut_a(!page_zip); + ut_d(log_sys->disable_redo_writes = false); - ptr = mlog_parse_index(ptr, end_ptr, true, &index); + mutex_enter(&recv_sys->mutex); - if (ptr == nullptr) { - break; - } + recv_no_ibuf_operations = false; + } - /* Fall through */ + recv_sys->apply_log_recs = false; + recv_sys->apply_batch_on = false; - case MLOG_REC_SEC_DELETE_MARK: + recv_sys_empty_hash(); - ut_ad(!page || fil_page_type_is_index(page_type)); + mutex_exit(&recv_sys->mutex); - ptr = btr_cur_parse_del_mark_set_sec_rec( - ptr, end_ptr, page, page_zip); - break; + ib::info() << "Apply batch completed!"; +} - case MLOG_REC_UPDATE_IN_PLACE: - case MLOG_COMP_REC_UPDATE_IN_PLACE: +#else /* !UNIV_HOTBACKUP */ +/** Reads the checkpoint info needed in hot backup. +@param[in] hdr buffer containing the log group header +@param[out] lsn checkpoint lsn +@param[out] offset checkpoint offset in the log group +@param[out] cp_no checkpoint number +@param[out] first_header_lsn lsn of of the start of the first log file +@return true if success */ +bool +meb_read_checkpoint_info( + const byte* hdr, + lsn_t* lsn, + lsn_t* offset, + lsn_t* cp_no, + lsn_t* first_header_lsn) +{ + ulint max_cp = 0; + uint64_t max_cp_no = 0; + const byte* cp_buf = hdr + LOG_CHECKPOINT_1; - ut_ad(!page || fil_page_type_is_index(page_type)); + if (recv_check_log_header_checksum(cp_buf)) { + max_cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO); + max_cp = LOG_CHECKPOINT_1; + } - if (nullptr != (ptr = mlog_parse_index( - ptr, end_ptr, - type == MLOG_COMP_REC_UPDATE_IN_PLACE, - &index))) { - ut_a(!page - || (ibool)!!page_is_comp(page) - == dict_table_is_comp(index->table)); + cp_buf = hdr + LOG_CHECKPOINT_2; - ptr = btr_cur_parse_update_in_place( - ptr, end_ptr, page, page_zip, index); + if (recv_check_log_header_checksum(cp_buf)) { + if (mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO) > max_cp_no) { + max_cp = LOG_CHECKPOINT_2; } + } - break; - - case MLOG_LIST_END_DELETE: - case MLOG_COMP_LIST_END_DELETE: - case MLOG_LIST_START_DELETE: - case MLOG_COMP_LIST_START_DELETE: - - ut_ad(!page || fil_page_type_is_index(page_type)); - - if (nullptr != (ptr = mlog_parse_index( - ptr, end_ptr, - type == MLOG_COMP_LIST_END_DELETE - || type == MLOG_COMP_LIST_START_DELETE, - &index))) { - ut_a(!page - || (ibool)!!page_is_comp(page) - == dict_table_is_comp(index->table)); - - ptr = page_parse_delete_rec_list( - type, ptr, end_ptr, block, index, mtr); - } + if (max_cp == 0) { + return(false); + } - break; + cp_buf = hdr + max_cp; - case MLOG_LIST_END_COPY_CREATED: case MLOG_COMP_LIST_END_COPY_CREATED: + *lsn = mach_read_from_8(cp_buf + LOG_CHECKPOINT_LSN); + *offset = mach_read_from_8(cp_buf + LOG_CHECKPOINT_OFFSET); - ut_ad(!page || fil_page_type_is_index(page_type)); + *cp_no = mach_read_from_8(cp_buf + LOG_CHECKPOINT_NO); - if (nullptr != (ptr = mlog_parse_index( - ptr, end_ptr, - type == MLOG_COMP_LIST_END_COPY_CREATED, - &index))) { + *first_header_lsn = mach_read_from_8(hdr + LOG_HEADER_START_LSN); - ut_a(!page - || (ibool)!!page_is_comp(page) - == dict_table_is_comp(index->table)); + return(true); +} - ptr = page_parse_copy_rec_list_to_created_page( - ptr, end_ptr, block, index, mtr); - } +/** Scans the log segment and n_bytes_scanned is set to the length of valid +log scanned. +@param[in] buf buffer containing log data +@param[in] buf_len data length in that buffer +@param[in,out] scanned_lsn LSN of buffer start, we return scanned +lsn +@param[in,out] scanned_checkpoint_no 4 lowest bytes of the highest scanned +checkpoint number so far +@param[out] n_bytes_scanned how much we were able to scan, smaller +than buf_len if log data ended here */ +void +meb_scan_log_seg( + byte* buf, + ulint buf_len, + lsn_t* scanned_lsn, + ulint* scanned_checkpoint_no, + ulint* n_bytes_scanned) +{ + *n_bytes_scanned = 0; - break; + for (auto log_block = buf; + log_block < buf + buf_len; + log_block += OS_FILE_LOG_BLOCK_SIZE) { - case MLOG_PAGE_REORGANIZE: - case MLOG_COMP_PAGE_REORGANIZE: - case MLOG_ZIP_PAGE_REORGANIZE: + ulint no = log_block_get_hdr_no(log_block); - ut_ad(!page || fil_page_type_is_index(page_type)); + if (no != log_block_convert_lsn_to_no(*scanned_lsn) + || !log_block_checksum_is_ok(log_block)) { - if (nullptr != (ptr = mlog_parse_index( - ptr, end_ptr, - type != MLOG_PAGE_REORGANIZE, - &index))) { - ut_a(!page - || (ibool)!!page_is_comp(page) - == dict_table_is_comp(index->table)); + ib::trace_2() + << "Scanned lsn: " + << *scanned_lsn + << " header no: " + << no + << " converted no: " + << log_block_convert_lsn_to_no(*scanned_lsn) + << " checksum: " + << log_block_checksum_is_ok(log_block) + << " block cp no: " + << log_block_get_checkpoint_no(log_block); - ptr = btr_parse_page_reorganize( - ptr, end_ptr, index, - type == MLOG_ZIP_PAGE_REORGANIZE, - block, mtr); + /* Garbage or an incompletely written log block */ + + log_block += OS_FILE_LOG_BLOCK_SIZE; + break; } - break; + if (*scanned_checkpoint_no > 0 + && log_block_get_checkpoint_no(log_block) + < *scanned_checkpoint_no + && *scanned_checkpoint_no + - log_block_get_checkpoint_no(log_block) + > 0x80000000UL) { - case MLOG_PAGE_CREATE: - case MLOG_COMP_PAGE_CREATE: + /* Garbage from a log buffer flush which was made + before the most recent database recovery */ - /* Allow anything in page_type when creating a page. */ - ut_a(!page_zip); + ib::trace_2() + << "Scanned cp no: " + << *scanned_checkpoint_no + << " block cp no " + << log_block_get_checkpoint_no(log_block); - page_parse_create( - block, type == MLOG_COMP_PAGE_CREATE, - FIL_PAGE_INDEX); + break; + } - break; + ulint data_len = log_block_get_data_len(log_block); - case MLOG_PAGE_CREATE_RTREE: - case MLOG_COMP_PAGE_CREATE_RTREE: + *scanned_checkpoint_no + = log_block_get_checkpoint_no(log_block); + *scanned_lsn += data_len; - page_parse_create( - block, type == MLOG_COMP_PAGE_CREATE_RTREE, - FIL_PAGE_RTREE); + *n_bytes_scanned += data_len; - break; + if (data_len < OS_FILE_LOG_BLOCK_SIZE) { + /* Log data ends here */ - case MLOG_PAGE_CREATE_SDI: - case MLOG_COMP_PAGE_CREATE_SDI: + break; + } + } +} - page_parse_create( - block, type == MLOG_COMP_PAGE_CREATE_SDI, - FIL_PAGE_SDI); +/** Apply a single log record stored in the hash table. +@param[in,out] recv_addr a parsed log record +@param[in,out] block a buffer pool frame for applying the record */ +void +meb_apply_log_record( + recv_addr_t* recv_addr, + buf_block_t* block) +{ + bool found; + const page_id_t page_id(recv_addr->space, recv_addr->page_no); - break; + const page_size_t& page_size = + fil_space_get_page_size(recv_addr->space, &found); - case MLOG_UNDO_INSERT: + ib::trace_3() + << "recv_addr {State: " << recv_addr->state + << ", Space id: " << recv_addr->space + << ", Page no: " << recv_addr->page_no + << ", Page size: " << page_size + << ", found: " << found << "\n"; - ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); + if (!found) { - ptr = trx_undo_parse_add_undo_rec(ptr, end_ptr, page); + recv_addr->state = RECV_DISCARDED; - break; + mutex_enter(&recv_sys->mutex); - case MLOG_UNDO_ERASE_END: + ut_a(recv_sys->n_addrs); + --recv_sys->n_addrs; - ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); + mutex_exit(&recv_sys->mutex); - ptr = trx_undo_parse_erase_page_end(ptr, end_ptr, page, mtr); + return; + } - break; + mutex_enter(&recv_sys->mutex); - case MLOG_UNDO_INIT: + /* We simulate a page read made by the buffer pool, to + make sure the recovery apparatus works ok. We must init + the block. */ - /* Allow anything in page_type when creating a page. */ + meb_page_init(page_id, page_size, block); - ptr = trx_undo_parse_page_init(ptr, end_ptr, page, mtr); + /* Extend the tablespace's last file if the page_no + does not fall inside its bounds; we assume the last + file is auto-extending, and mysqlbackup copied the file + when it still was smaller */ - break; - case MLOG_UNDO_HDR_CREATE: - case MLOG_UNDO_HDR_REUSE: + fil_space_t* space = fil_space_get(recv_addr->space); - ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); + bool success; - ptr = trx_undo_parse_page_header( - type, ptr, end_ptr, page, mtr); + success = fil_space_extend(space, recv_addr->page_no + 1); - break; + if (!success) { + ib::fatal() + << "Cannot extend tablespace " + << recv_addr->space << " to hold " + << recv_addr->page_no << " pages"; + } - case MLOG_REC_MIN_MARK: - case MLOG_COMP_REC_MIN_MARK: + mutex_exit(&recv_sys->mutex); - ut_ad(!page || fil_page_type_is_index(page_type)); + /* Read the page from the tablespace file. */ - /* On a compressed page, MLOG_COMP_REC_MIN_MARK - will be followed by MLOG_COMP_REC_DELETE - or MLOG_ZIP_WRITE_HEADER(FIL_PAGE_PREV, FIL_nullptr) - in the same mini-transaction. */ + dberr_t err; - ut_a(type == MLOG_COMP_REC_MIN_MARK || !page_zip); + if (page_size.is_compressed()) { - ptr = btr_parse_set_min_rec_mark( - ptr, end_ptr, type == MLOG_COMP_REC_MIN_MARK, - page, mtr); + err = fil_io( + IORequestRead, true, + page_id, + page_size, 0, page_size.physical(), + block->page.zip.data, nullptr); - break; + if (err == DB_SUCCESS && !buf_zip_decompress(block, TRUE)) { - case MLOG_REC_DELETE: - case MLOG_COMP_REC_DELETE: + ut_error; + } + } else { - ut_ad(!page || fil_page_type_is_index(page_type)); + err = fil_io( + IORequestRead, true, + page_id, page_size, 0, page_size.logical(), + block->frame, nullptr); + } - if (nullptr != (ptr = mlog_parse_index( - ptr, end_ptr, - type == MLOG_COMP_REC_DELETE, - &index))) { - ut_a(!page - || (ibool)!!page_is_comp(page) - == dict_table_is_comp(index->table)); + if (err != DB_SUCCESS) { - ptr = page_cur_parse_delete_rec( - ptr, end_ptr, block, index, mtr); - } + ib::fatal() + << "Cannot read from tablespace " + << recv_addr->space << " page number " + << recv_addr->page_no; + } - break; + apply_log_mutex.lock(); - case MLOG_IBUF_BITMAP_INIT: + /* Apply the log records to this page */ + recv_recover_page(false, block); - /* Allow anything in page_type when creating a page. */ + apply_log_mutex.unlock(); - ptr = ibuf_parse_bitmap_init(ptr, end_ptr, block, mtr); + mutex_enter(&recv_sys->mutex); - break; + /* Write the page back to the tablespace file using the + fil0fil.cc routines */ - case MLOG_INIT_FILE_PAGE: - case MLOG_INIT_FILE_PAGE2: + buf_flush_init_for_writing( + block, block->frame, buf_block_get_page_zip(block), + mach_read_from_8(block->frame + FIL_PAGE_LSN), + fsp_is_checksum_disabled(block->page.id.space())); - /* Allow anything in page_type when creating a page. */ + mutex_exit(&recv_sys->mutex); - ptr = fsp_parse_init_file_page(ptr, end_ptr, block); + if (page_size.is_compressed()) { - break; + err = fil_io( + IORequestWrite, true, page_id, + page_size, 0, page_size.physical(), + block->page.zip.data, nullptr); + } else { + err = fil_io( + IORequestWrite, true, page_id, + page_size, 0, page_size.logical(), + block->frame, nullptr); + } - case MLOG_WRITE_STRING: + if (err != DB_SUCCESS) { - ut_ad(!page || page_type != FIL_PAGE_TYPE_ALLOCATED - || page_no == 0); + ib::fatal() + << "Cannot write to tablespace " + << recv_addr->space << " page number " + << recv_addr->page_no; + } +} - ptr = mlog_parse_string(ptr, end_ptr, page, page_zip); +/** Apply a single log record stored in the hash table using default block. +@param[in,out] recv_addr a parsed log record */ +void +meb_apply_log_rec_func( + recv_addr_t* recv_addr) +{ + meb_apply_log_record(recv_addr, back_block1); +} - break; +/** Dummy wait function for meb_apply_log_recs_via_callback(). */ +void +meb_nowait_func() +{ + return; +} - case MLOG_ZIP_WRITE_NODE_PTR: +/** Applies log records in the hash table to a backup. */ +void +meb_apply_log_recs() +{ + meb_apply_log_recs_via_callback( + meb_apply_log_rec_func, meb_nowait_func); +} - ut_ad(!page || fil_page_type_is_index(page_type)); +/** Apply all log records in the hash table to a backup using callback +functions. This function employes two callback functions that allow redo +log records to be applied in parallel. The apply_log_record_function +assigns a parsed redo log record for application. The +apply_log_record_function is called repeatedly until all log records in +the hash table are assigned for application. After that the +wait_till_done_function is called once. The wait_till_done_function +function blocks until the application of all the redo log records +previously assigned with apply_log_record_function calls is complete. +Even though this function assigns the log records in the hash table +sequentially, the application of the log records may be done in parallel +if the apply_log_record_function delegates the actual application work +to multiple worker threads running in parallel. +@param[in] apply_log_record_function a function that assigns one redo log +record for application +@param[in] wait_till_done_function a function that blocks until all +assigned redo log records have been applied */ +void +meb_apply_log_recs_via_callback( + void (*apply_log_record_function)(recv_addr_t*), + void (*wait_till_done_function)()) +{ + ulint n_hash_cells = recv_sys->n_addrs; + ulint i = 0; - ptr = page_zip_parse_write_node_ptr( - ptr, end_ptr, page, page_zip); + recv_sys->apply_log_recs = true; + recv_sys->apply_batch_on = true; + + ib::info() + << "Starting to apply a batch of log records to the" + << " database..."; - break; + fputs("InnoDB: Progress in percent: ", stderr); - case MLOG_ZIP_WRITE_BLOB_PTR: + for (const auto& space : *recv_sys->spaces) { - ut_ad(!page || fil_page_type_is_index(page_type)); + for (auto pages : space.second.m_pages) { - ptr = page_zip_parse_write_blob_ptr( - ptr, end_ptr, page, page_zip); + ut_ad(pages.second->space == space.first); - break; + (*apply_log_record_function)(pages.second); + } - case MLOG_ZIP_WRITE_HEADER: + ++i; + if ((100 * i) / n_hash_cells + != (100 * (i + 1)) / n_hash_cells) { + fprintf(stderr, "%lu ", + (ulong) ((100 * i) / n_hash_cells)); + fflush(stderr); + } + } - ut_ad(!page || fil_page_type_is_index(page_type)); + /* wait till all the redo log records have been applied */ + (*wait_till_done_function)(); - ptr = page_zip_parse_write_header( - ptr, end_ptr, page, page_zip); + /* write logs in next line */ + fprintf(stderr, "\n"); + recv_sys->apply_log_recs = false; + recv_sys->apply_batch_on = false; + recv_sys_empty_hash(); +} - break; +#endif /* !UNIV_HOTBACKUP */ - case MLOG_ZIP_PAGE_COMPRESS: +/** Try to parse a single log record body and also applies it if +specified. +@param[in] type redo log entry type +@param[in] ptr redo log record body +@param[in] end_ptr end of buffer +@param[in] space_id tablespace identifier +@param[in] page_no page number +@param[in,out] block buffer block, or nullptr if + a page log record should not be applied + or if it is a MLOG_FILE_ operation +@param[in,out] mtr mini-transaction, or nullptr if + a page log record should not be applied +@param[in] parsed_bytes Number of bytes parsed so far +@return log record end, nullptr if not a complete record */ +static +byte* +recv_parse_or_apply_log_rec_body( + mlog_id_t type, + byte* ptr, + byte* end_ptr, + space_id_t space_id, + page_no_t page_no, + buf_block_t* block, + mtr_t* mtr, + ulint parsed_bytes) +{ + ut_ad(!block == !mtr); - /* Allow anything in page_type when creating a page. */ - ptr = page_zip_parse_compress(ptr, end_ptr, page, page_zip); - break; + switch (type) { + case MLOG_FILE_DELETE: - case MLOG_ZIP_PAGE_COMPRESS_NO_DATA: + return(fil_tablespace_redo_delete( + ptr,end_ptr, page_id_t(space_id, page_no), + parsed_bytes)); - if (nullptr != (ptr = mlog_parse_index( - ptr, end_ptr, true, &index))) { + case MLOG_FILE_CREATE: - ut_a(!page || ((ibool)!!page_is_comp(page) - == dict_table_is_comp(index->table))); + return(fil_tablespace_redo_create( + ptr,end_ptr, page_id_t(space_id, page_no), + parsed_bytes)); - ptr = page_zip_parse_compress_no_data( - ptr, end_ptr, page, page_zip, index); - } + case MLOG_FILE_RENAME: - break; + return(fil_tablespace_redo_rename( + ptr, end_ptr, page_id_t(space_id, page_no), + parsed_bytes)); - default: - ptr = nullptr; - recv_sys->found_corrupt_log = true; - } + case MLOG_INDEX_LOAD: +#ifdef UNIV_HOTBACKUP + /* While scaning redo logs during backup phase a + MLOG_INDEX_LOAD type redo log record indicates a DDL + (create index, alter table...)is performed with + 'algorithm=inplace'. This redo log indicates that - if (index != nullptr) { + 1. The DDL was started after MEB started backing up, in which + case MEB will not be able to take a consistent backup and should + fail. or + 2. There is a possibility of this record existing in the REDO + even after the completion of the index create operation. This is + because of InnoDB does not checkpointing after the flushing the + index pages. - dict_table_t* table = index->table; + If MEB gets the last_redo_flush_lsn and that is less than the + lsn of the current record MEB fails the backup process. + Error out in case of online backup and emit a warning in case + of offline backup and continue. */ + if (!recv_recovery_on) { + if (is_online_redo_copy) { + if (backup_redo_log_flushed_lsn + < recv_sys->recovered_lsn) { + ib::trace() + << "Last flushed lsn: " + << backup_redo_log_flushed_lsn + << " load_index lsn " + << recv_sys->recovered_lsn; - dict_mem_index_free(index); - dict_mem_table_free(table); - } + if (backup_redo_log_flushed_lsn == 0) { + ib::error() + << "MEB was not able" + << " to determine the" + << " InnoDB Engine" + << " Status"; + } - return(ptr); -} + ib::fatal() + << "An optimized(without" + << " redo logging) DDL" + << " operation has been" + << " performed. All modified" + << " pages may not have been" + << " flushed to the disk yet.\n" + << " MEB will not be able to" + << " take a consistent backup." + << " Retry the backup" + << " operation"; + } + /** else the index is flushed to disk before + backup started hence no error */ + } else { + /* offline backup */ + ib::trace() + << "Last flushed lsn: " + << backup_redo_log_flushed_lsn + << " load_index lsn " + << recv_sys->recovered_lsn; -/** Get the page map for a tablespace. It will create one if one isn't found. -@param[in] space_id Tablespace ID for which page map required. -@param[in] create false if lookup only -@return the space data or null if not found */ -static -recv_sys_t::Space* -recv_get_page_map(space_id_t space_id, bool create) -{ - auto it = recv_sys->spaces->find(space_id); + ib::warn() + << "An optimized(without redo" + << " logging) DDL operation has been" + << " performed. All modified pages may" + << " not have been flushed to the disk" + << " yet.\n This offline backup may" + << " not be consistent"; + } + } +#endif /* UNIV_HOTBACKUP */ + if (end_ptr < ptr + 8) { - if (it != recv_sys->spaces->end()) { + return(nullptr); + } - return(&it->second); + return(ptr + 8); - } else if (create) { + case MLOG_WRITE_STRING: - mem_heap_t* heap; +#ifdef UNIV_HOTBACKUP + if (recv_recovery_on) { +#endif /* UNIV_HOTBACKUP */ + /* For encrypted tablespace, we need to get the + encryption key information before the page 0 is + recovered. Otherwise, redo will not find the key + to decrypt the data pages. */ - heap = mem_heap_create_typed(256, MEM_HEAP_FOR_RECV_SYS); + if (page_no == 0 + && !fsp_is_system_or_temp_tablespace(space_id)) { - using Space = recv_sys_t::Space; - using value_type = recv_sys_t::Spaces::value_type; + return(fil_tablespace_redo_encryption( + ptr, end_ptr, space_id)); + } +#ifdef UNIV_HOTBACKUP + } +#endif /* UNIV_HOTBACKUP */ - auto where = recv_sys->spaces->insert( - it, value_type(space_id, Space(heap))); + break; - return(&where->second); + default: + break; } - return(nullptr); -} + page_t* page; + page_zip_des_t* page_zip; + dict_index_t* index = nullptr; -/** Gets the list of log records for a . -@param[in] space_id Tablespace ID -@param[in] page_no Page number -@return the redo log entries or nullptr if not found */ -static -recv_addr_t* -recv_get_rec( - space_id_t space_id, - page_no_t page_no) -{ - recv_sys_t::Space* space; +#ifdef UNIV_DEBUG + ulint page_type; +#endif /* UNIV_DEBUG */ - space = recv_get_page_map(space_id, false); +#if defined(UNIV_HOTBACKUP) && defined(UNIV_DEBUG) + ib::trace_3() + << "recv_parse_or_apply_log_rec_body { type: " + << get_mlog_string(type) + << ", space_id: " << space_id + << ", page_no: " << page_no + << ", ptr : " << static_cast (ptr) + << ", end_ptr: " << static_cast(end_ptr) + << ", block: " << static_cast(block) + << ", mtr: " << static_cast(mtr) << " }"; +#endif /* UNIV_HOTBACKUP && UNIV_DEBUG */ - if (space != nullptr) { + if (block != nullptr) { - auto it = space->m_pages.find(page_no); + /* Applying a page log record. */ - if (it != space->m_pages.end()) { + page = block->frame; + page_zip = buf_block_get_page_zip(block); - return(it->second); + ut_d(page_type = fil_page_get_type(page)); +#if defined(UNIV_HOTBACKUP) && defined(UNIV_DEBUG) + if (page_type == 0) { + meb_print_page_header(page); } - } +#endif /* UNIV_HOTBACKUP && UNIV_DEBUG */ - return(nullptr); -} + } else { -/** Adds a new log record to the hash table of log records. -@param[in] type log record type -@param[in] space_id Tablespace id -@param[in] page_no page number -@param[in] body log record body -@param[in] rec_end log record end -@param[in] start_lsn start lsn of the mtr -@param[in] end_lsn end lsn of the mtr */ -static -void -recv_add_to_hash_table( - mlog_id_t type, - space_id_t space_id, - page_no_t page_no, - byte* body, - byte* rec_end, - lsn_t start_lsn, - lsn_t end_lsn) -{ - ut_ad(type != MLOG_FILE_OPEN); - ut_ad(type != MLOG_FILE_DELETE); - ut_ad(type != MLOG_FILE_CREATE2); - ut_ad(type != MLOG_FILE_RENAME2); - ut_ad(type != MLOG_DUMMY_RECORD); - ut_ad(type != MLOG_INDEX_LOAD); + /* Parsing a page log record. */ + page = nullptr; + page_zip = nullptr; - recv_sys_t::Space* space; + ut_d(page_type = FIL_PAGE_TYPE_ALLOCATED); + } - space = recv_get_page_map(space_id, true); + const byte* old_ptr = ptr; - recv_t* recv; + switch (type) { +#ifdef UNIV_LOG_LSN_DEBUG + case MLOG_LSN: + /* The LSN is checked in recv_parse_log_rec(). */ + break; +#endif /* UNIV_LOG_LSN_DEBUG */ + case MLOG_4BYTES: - recv = static_cast( - mem_heap_alloc(space->m_heap, sizeof(*recv))); + ut_ad(page == nullptr || end_ptr > ptr + 2); - recv->type = type; - recv->end_lsn = end_lsn; - recv->len = rec_end - body; - recv->start_lsn = start_lsn; + /* Most FSP flags can only be changed by CREATE or ALTER with + ALGORITHM=COPY, so they do not change once the file + is created. The SDI flag is the only one that can be + changed by a recoverable transaction. So if there is + change in FSP flags, update the in-memory space structure + (fil_space_t) */ - auto it = space->m_pages.find(page_no); + if (page != nullptr + && page_no == 0 + && mach_read_from_2(ptr) + == FSP_HEADER_OFFSET + FSP_SPACE_FLAGS) { - recv_addr_t* recv_addr; + ptr = mlog_parse_nbytes( + MLOG_4BYTES, ptr, end_ptr, page, page_zip); - if (it != space->m_pages.end()) { + /* When applying log, we have complete records. + They can be incomplete (ptr=nullptr) only during + scanning (page==nullptr) */ - recv_addr = it->second; + ut_ad(ptr != nullptr); - } else { + fil_space_t* space = fil_space_acquire(space_id); - recv_addr = static_cast( - mem_heap_alloc(space->m_heap, sizeof(*recv_addr))); + ut_ad(space != nullptr); - recv_addr->space = space_id; - recv_addr->page_no = page_no; - recv_addr->state = RECV_NOT_PROCESSED; + fil_space_set_flags( + space, mach_read_from_4( + FSP_HEADER_OFFSET + + FSP_SPACE_FLAGS + + page)); - UT_LIST_INIT(recv_addr->rec_list, &recv_t::rec_list); + fil_space_release(space); - using value_type = recv_sys_t::Pages::value_type; + break; + } - space->m_pages.insert(it, value_type(page_no, recv_addr)); + // fall through - ++recv_sys->n_addrs; - } + case MLOG_1BYTE: + case MLOG_2BYTES: + case MLOG_8BYTES: +#ifdef UNIV_DEBUG + if (page + && page_type == FIL_PAGE_TYPE_ALLOCATED + && end_ptr >= ptr + 2) { - UT_LIST_ADD_LAST(recv_addr->rec_list, recv); + /* It is OK to set FIL_PAGE_TYPE and certain + list node fields on an empty page. Any other + write is not OK. */ - recv_data_t** prev_field; + /* NOTE: There may be bogus assertion failures for + dict_hdr_create(), trx_rseg_header_create(), + trx_sys_create_doublewrite_buf(), and + trx_sysf_create(). + These are only called during database creation. */ - prev_field = &recv->data; + ulint offs = mach_read_from_2(ptr); - /* Store the log record body in chunks of less than UNIV_PAGE_SIZE: - the heap grows into the buffer pool, and bigger chunks could not - be allocated */ + switch (type) { + default: + ut_error; + case MLOG_2BYTES: + /* Note that this can fail when the + redo log been written with something + older than InnoDB Plugin 1.0.4. */ + ut_ad(offs == FIL_PAGE_TYPE + || offs == IBUF_TREE_SEG_HEADER + + IBUF_HEADER + FSEG_HDR_OFFSET + || offs == PAGE_BTR_IBUF_FREE_LIST + + PAGE_HEADER + FIL_ADDR_BYTE + || offs == PAGE_BTR_IBUF_FREE_LIST + + PAGE_HEADER + FIL_ADDR_BYTE + + FIL_ADDR_SIZE + || offs == PAGE_BTR_SEG_LEAF + + PAGE_HEADER + FSEG_HDR_OFFSET + || offs == PAGE_BTR_SEG_TOP + + PAGE_HEADER + FSEG_HDR_OFFSET + || offs == PAGE_BTR_IBUF_FREE_LIST_NODE + + PAGE_HEADER + FIL_ADDR_BYTE + + 0 /*FLST_PREV*/ + || offs == PAGE_BTR_IBUF_FREE_LIST_NODE + + PAGE_HEADER + FIL_ADDR_BYTE + + FIL_ADDR_SIZE /*FLST_NEXT*/); + break; + case MLOG_4BYTES: + /* Note that this can fail when the + redo log been written with something + older than InnoDB Plugin 1.0.4. */ + ut_ad(0 + || offs == IBUF_TREE_SEG_HEADER + + IBUF_HEADER + FSEG_HDR_SPACE + || offs == IBUF_TREE_SEG_HEADER + + IBUF_HEADER + FSEG_HDR_PAGE_NO + || offs == PAGE_BTR_IBUF_FREE_LIST + + PAGE_HEADER/* flst_init */ + || offs == PAGE_BTR_IBUF_FREE_LIST + + PAGE_HEADER + FIL_ADDR_PAGE + || offs == PAGE_BTR_IBUF_FREE_LIST + + PAGE_HEADER + FIL_ADDR_PAGE + + FIL_ADDR_SIZE + || offs == PAGE_BTR_SEG_LEAF + + PAGE_HEADER + FSEG_HDR_PAGE_NO + || offs == PAGE_BTR_SEG_LEAF + + PAGE_HEADER + FSEG_HDR_SPACE + || offs == PAGE_BTR_SEG_TOP + + PAGE_HEADER + FSEG_HDR_PAGE_NO + || offs == PAGE_BTR_SEG_TOP + + PAGE_HEADER + FSEG_HDR_SPACE + || offs == PAGE_BTR_IBUF_FREE_LIST_NODE + + PAGE_HEADER + FIL_ADDR_PAGE + + 0 /*FLST_PREV*/ + || offs == PAGE_BTR_IBUF_FREE_LIST_NODE + + PAGE_HEADER + FIL_ADDR_PAGE + + FIL_ADDR_SIZE /*FLST_NEXT*/); + break; + } + } +#endif /* UNIV_DEBUG */ - while (rec_end > body) { + ptr = mlog_parse_nbytes(type, ptr, end_ptr, page, page_zip); - ulint len = rec_end - body; + if (ptr != nullptr + && page != nullptr + && page_no == 0 + && type == MLOG_4BYTES) { - if (len > RECV_DATA_BLOCK_SIZE) { - len = RECV_DATA_BLOCK_SIZE; - } + ulint offs = mach_read_from_2(old_ptr); - recv_data_t* recv_data; + switch (offs) { + fil_space_t* space; + uint32_t val; + default: + break; - recv_data = static_cast( - mem_heap_alloc( - space->m_heap, sizeof(*recv_data) + len)); + case FSP_HEADER_OFFSET + FSP_SPACE_FLAGS: + case FSP_HEADER_OFFSET + FSP_SIZE: + case FSP_HEADER_OFFSET + FSP_FREE_LIMIT: + case FSP_HEADER_OFFSET + FSP_FREE + FLST_LEN: - *prev_field = recv_data; + space = fil_space_get(space_id); - memcpy(recv_data + 1, body, len); + ut_a(space != nullptr); - prev_field = &recv_data->next; + val = mach_read_from_4(page + offs); - body += len; - } + switch (offs) { + case FSP_HEADER_OFFSET + FSP_SPACE_FLAGS: + space->flags = val; + break; - *prev_field = nullptr; -} + case FSP_HEADER_OFFSET + FSP_SIZE: -/** Copies the log record body from recv to buf. -@param[in] buf Buffer of length at least recv->len -@param[in] recv Log record */ -static -void -recv_data_copy_to_buf(byte* buf, recv_t* recv) -{ - ulint len = recv->len; - recv_data_t* recv_data = recv->data; + space->size_in_header = val; - while (len > 0) { + if (space->size >= val) { - ulint part_len; + break; + } - if (len > RECV_DATA_BLOCK_SIZE) { - part_len = RECV_DATA_BLOCK_SIZE; - } else { - part_len = len; - } + ib::info() + << "Extending tablespace : " + << space->id + << " space name: " + << space->name + << " to new size: " << val + << " pages during recovery."; - memcpy(buf, ((byte*) recv_data) + sizeof(*recv_data), part_len); + if (fil_space_extend(space, val)) { - buf += part_len; - len -= part_len; + break; + } - recv_data = recv_data->next; - } -} + ib::error() + << "Could not extend tablespace" + << ": " << space->id << " space" + << " name: " << space->name + << " to new size: " << val + << " pages during recovery."; -/** Applies the hashed log records to the page, if the page lsn is less than the -lsn of a log record. This can be called when a buffer page has just been -read in, or also for a page already in the buffer pool. -@param[in] just_read_in true if the IO handler calls this for a freshly - read page -@param[in,out] block Buffer block */ -void -recv_recover_page_func( -#ifndef UNIV_HOTBACKUP - bool just_read_in, -#endif /* !UNIV_HOTBACKUP */ - buf_block_t* block) -{ - mutex_enter(&recv_sys->mutex); + break; - if (recv_sys->apply_log_recs == false) { + case FSP_HEADER_OFFSET + FSP_FREE_LIMIT: + space->free_limit = val; + break; + + case FSP_HEADER_OFFSET + FSP_FREE + FLST_LEN: + space->free_len = val; + ut_ad(val == flst_get_len(page + offs)); + break; + } + } + } + break; + + case MLOG_REC_INSERT: + case MLOG_COMP_REC_INSERT: - /* Log records should not be applied now */ + ut_ad(!page || fil_page_type_is_index(page_type)); - mutex_exit(&recv_sys->mutex); + if (nullptr != (ptr = mlog_parse_index( + ptr, end_ptr, + type == MLOG_COMP_REC_INSERT, + &index))) { + ut_a(!page + || (ibool)!!page_is_comp(page) + == dict_table_is_comp(index->table)); - return; - } + ptr = page_cur_parse_insert_rec( + FALSE, ptr, end_ptr, block, index, mtr); + } - recv_addr_t* recv_addr; + break; - recv_addr = recv_get_rec( - block->page.id.space(), block->page.id.page_no()); + case MLOG_REC_CLUST_DELETE_MARK: + case MLOG_COMP_REC_CLUST_DELETE_MARK: - if (recv_addr == nullptr - || recv_addr->state == RECV_BEING_PROCESSED - || recv_addr->state == RECV_PROCESSED) { + ut_ad(!page || fil_page_type_is_index(page_type)); - ut_ad(recv_addr == nullptr || recv_needed_recovery); + if (nullptr != (ptr = mlog_parse_index( + ptr, end_ptr, + type == MLOG_COMP_REC_CLUST_DELETE_MARK, + &index))) { - mutex_exit(&recv_sys->mutex); + ut_a(!page + || (ibool)!!page_is_comp(page) + == dict_table_is_comp(index->table)); - return; - } + ptr = btr_cur_parse_del_mark_set_clust_rec( + ptr, end_ptr, page, page_zip, index); + } -#ifndef UNIV_HOTBACKUP - /* this is explicitly false in case of meb, skip the assert */ - ut_ad(recv_needed_recovery); + break; - DBUG_PRINT("ib_log", - ("Applying log to page %u:%u", - recv_addr->space, recv_addr->page_no)); + case MLOG_COMP_REC_SEC_DELETE_MARK: -# ifdef UNIV_DEBUG - lsn_t max_lsn; + ut_ad(!page || fil_page_type_is_index(page_type)); - ut_d(max_lsn = UT_LIST_GET_FIRST(log_sys->log_groups)->scanned_lsn); -# endif /* UNIV_DEBUG */ -#else /* !UNIV_HOTBACKUP */ - ib::trace_2() - << "Applying log to space " << recv_addr->space - << " page " << recv_addr->page_no; -#endif /* !UNIV_HOTBACKUP */ + /* This log record type is obsolete, but we process it for + backward compatibility with MySQL 5.0.3 and 5.0.4. */ - recv_addr->state = RECV_BEING_PROCESSED; + ut_a(!page || page_is_comp(page)); + ut_a(!page_zip); - mutex_exit(&recv_sys->mutex); + ptr = mlog_parse_index(ptr, end_ptr, true, &index); - mtr_t mtr; + if (ptr == nullptr) { + break; + } - mtr_start(&mtr); + /* Fall through */ - mtr_set_log_mode(&mtr, MTR_LOG_NONE); + case MLOG_REC_SEC_DELETE_MARK: - page_t* page = block->frame; + ut_ad(!page || fil_page_type_is_index(page_type)); - page_zip_des_t* page_zip = buf_block_get_page_zip(block); + ptr = btr_cur_parse_del_mark_set_sec_rec( + ptr, end_ptr, page, page_zip); + break; -#ifndef UNIV_HOTBACKUP - if (just_read_in) { - /* Move the ownership of the x-latch on the page to - this OS thread, so that we can acquire a second - x-latch on it. This is needed for the operations to - the page to pass the debug checks. */ + case MLOG_REC_UPDATE_IN_PLACE: + case MLOG_COMP_REC_UPDATE_IN_PLACE: - rw_lock_x_lock_move_ownership(&block->lock); - } + ut_ad(!page || fil_page_type_is_index(page_type)); - bool success = buf_page_get_known_nowait( - RW_X_LATCH, block, BUF_KEEP_OLD, - __FILE__, __LINE__, &mtr); - ut_a(success); + if (nullptr != (ptr = mlog_parse_index( + ptr, end_ptr, + type == MLOG_COMP_REC_UPDATE_IN_PLACE, + &index))) { + ut_a(!page + || (ibool)!!page_is_comp(page) + == dict_table_is_comp(index->table)); - buf_block_dbg_add_level(block, SYNC_NO_ORDER_CHECK); -#endif /* !UNIV_HOTBACKUP */ + ptr = btr_cur_parse_update_in_place( + ptr, end_ptr, page, page_zip, index); + } - /* Read the newest modification lsn from the page */ - lsn_t page_lsn = mach_read_from_8(page + FIL_PAGE_LSN); + break; -#ifndef UNIV_HOTBACKUP + case MLOG_LIST_END_DELETE: + case MLOG_COMP_LIST_END_DELETE: + case MLOG_LIST_START_DELETE: + case MLOG_COMP_LIST_START_DELETE: - /* It may be that the page has been modified in the buffer - pool: read the newest modification LSN there */ + ut_ad(!page || fil_page_type_is_index(page_type)); - lsn_t page_newest_lsn; + if (nullptr != (ptr = mlog_parse_index( + ptr, end_ptr, + type == MLOG_COMP_LIST_END_DELETE + || type == MLOG_COMP_LIST_START_DELETE, + &index))) { + ut_a(!page + || (ibool)!!page_is_comp(page) + == dict_table_is_comp(index->table)); - page_newest_lsn = buf_page_get_newest_modification(&block->page); + ptr = page_parse_delete_rec_list( + type, ptr, end_ptr, block, index, mtr); + } - if (page_newest_lsn) { + break; - page_lsn = page_newest_lsn; - } -#else /* !UNIV_HOTBACKUP */ - /* In recovery from a backup we do not really use the buffer pool */ - lsn_t page_newest_lsn = 0; - /* Count applied and skipped log records */ - size_t applied_recs = 0; - size_t skipped_recs = 0; -#endif /* !UNIV_HOTBACKUP */ + case MLOG_LIST_END_COPY_CREATED: case MLOG_COMP_LIST_END_COPY_CREATED: -#ifndef UNIV_HOTBACKUP - lsn_t end_lsn = 0; -#endif /* !UNIV_HOTBACKUP */ - lsn_t start_lsn = 0; - bool modification_to_page = false; + ut_ad(!page || fil_page_type_is_index(page_type)); - for (auto recv = UT_LIST_GET_FIRST(recv_addr->rec_list); - recv != nullptr; - recv = UT_LIST_GET_NEXT(rec_list, recv)) { + if (nullptr != (ptr = mlog_parse_index( + ptr, end_ptr, + type == MLOG_COMP_LIST_END_COPY_CREATED, + &index))) { -#ifndef UNIV_HOTBACKUP - end_lsn = recv->end_lsn; + ut_a(!page + || (ibool)!!page_is_comp(page) + == dict_table_is_comp(index->table)); - ut_ad(end_lsn <= max_lsn); -#endif /* !UNIV_HOTBACKUP */ + ptr = page_parse_copy_rec_list_to_created_page( + ptr, end_ptr, block, index, mtr); + } - byte* buf; + break; - if (recv->len > RECV_DATA_BLOCK_SIZE) { + case MLOG_PAGE_REORGANIZE: + case MLOG_COMP_PAGE_REORGANIZE: + case MLOG_ZIP_PAGE_REORGANIZE: - /* We have to copy the record body to a separate - buffer */ + ut_ad(!page || fil_page_type_is_index(page_type)); - buf = static_cast(ut_malloc_nokey(recv->len)); + if (nullptr != (ptr = mlog_parse_index( + ptr, end_ptr, + type != MLOG_PAGE_REORGANIZE, + &index))) { + ut_a(!page + || (ibool)!!page_is_comp(page) + == dict_table_is_comp(index->table)); - recv_data_copy_to_buf(buf, recv); - } else { - buf = ((byte*)(recv->data)) + sizeof(recv_data_t); + ptr = btr_parse_page_reorganize( + ptr, end_ptr, index, + type == MLOG_ZIP_PAGE_REORGANIZE, + block, mtr); } - if (recv->type == MLOG_INIT_FILE_PAGE) { + break; - page_lsn = page_newest_lsn; + case MLOG_PAGE_CREATE: + case MLOG_COMP_PAGE_CREATE: - memset(FIL_PAGE_LSN + page, 0, 8); - memset(UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM - + page, 0, 8); + /* Allow anything in page_type when creating a page. */ + ut_a(!page_zip); - if (page_zip) { - memset(FIL_PAGE_LSN + page_zip->data, 0, 8); - } - } + page_parse_create( + block, type == MLOG_COMP_PAGE_CREATE, + FIL_PAGE_INDEX); - /* Ignore applying the redo logs for tablespace that is - truncated. Truncated tablespaces are handled explicitly - post-recovery, where we will restore the tablespace back - to a normal state. + break; - Applying redo at this stage will cause problems because the - redo will have action recorded on page before tablespace - was re-inited and that would lead to a problem later. */ + case MLOG_PAGE_CREATE_RTREE: + case MLOG_COMP_PAGE_CREATE_RTREE: - if (recv->start_lsn >= page_lsn -#ifndef UNIV_HOTBACKUP - && undo::is_active(recv_addr->space) -#endif /* !UNIV_HOTBACKUP */ - ) { + page_parse_create( + block, type == MLOG_COMP_PAGE_CREATE_RTREE, + FIL_PAGE_RTREE); - lsn_t end_lsn; + break; - if (!modification_to_page) { + case MLOG_PAGE_CREATE_SDI: + case MLOG_COMP_PAGE_CREATE_SDI: - modification_to_page = true; - start_lsn = recv->start_lsn; - } + page_parse_create( + block, type == MLOG_COMP_PAGE_CREATE_SDI, + FIL_PAGE_SDI); - DBUG_PRINT("ib_log", - ("apply " LSN_PF ":" - " %s len " ULINTPF " page %u:%u", - recv->start_lsn, - get_mlog_string(recv->type), recv->len, - recv_addr->space, - recv_addr->page_no)); + break; - recv_parse_or_apply_log_rec_body( - recv->type, buf, buf + recv->len, - recv_addr->space, recv_addr->page_no, - block, &mtr, ULINT_UNDEFINED); + case MLOG_UNDO_INSERT: - end_lsn = recv->start_lsn + recv->len; + ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); - mach_write_to_8(FIL_PAGE_LSN + page, end_lsn); + ptr = trx_undo_parse_add_undo_rec(ptr, end_ptr, page); - mach_write_to_8(UNIV_PAGE_SIZE - - FIL_PAGE_END_LSN_OLD_CHKSUM - + page, end_lsn); + break; + + case MLOG_UNDO_ERASE_END: - if (page_zip) { + ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); - mach_write_to_8(FIL_PAGE_LSN - + page_zip->data, end_lsn); - } -#ifdef UNIV_HOTBACKUP - ++applied_recs; - } else { - ++skipped_recs; -#endif /* UNIV_HOTBACKUP */ - } + ptr = trx_undo_parse_erase_page_end(ptr, end_ptr, page, mtr); - if (recv->len > RECV_DATA_BLOCK_SIZE) { - ut_free(buf); - } - } + break; -#ifdef UNIV_ZIP_DEBUG - if (fil_page_index_page_check(page)) { - page_zip_des_t* page_zip = buf_block_get_page_zip(block); + case MLOG_UNDO_INIT: - ut_a(!page_zip - || page_zip_validate_low(page_zip, page, nullptr, FALSE)); - } -#endif /* UNIV_ZIP_DEBUG */ + /* Allow anything in page_type when creating a page. */ -#ifndef UNIV_HOTBACKUP - if (modification_to_page) { - ut_a(block); + ptr = trx_undo_parse_page_init(ptr, end_ptr, page, mtr); - log_flush_order_mutex_enter(); - buf_flush_recv_note_modification(block, start_lsn, end_lsn); - log_flush_order_mutex_exit(); - } -#else /* !UNIV_HOTBACKUP */ - UT_NOT_USED(start_lsn); -#endif /* !UNIV_HOTBACKUP */ + break; + case MLOG_UNDO_HDR_CREATE: + case MLOG_UNDO_HDR_REUSE: - /* Make sure that committing mtr does not change the modification - LSN values of page */ + ut_ad(!page || page_type == FIL_PAGE_UNDO_LOG); - mtr.discard_modifications(); + ptr = trx_undo_parse_page_header( + type, ptr, end_ptr, page, mtr); - mtr_commit(&mtr); + break; - mutex_enter(&recv_sys->mutex); + case MLOG_REC_MIN_MARK: + case MLOG_COMP_REC_MIN_MARK: - if (recv_max_page_lsn < page_lsn) { - recv_max_page_lsn = page_lsn; - } + ut_ad(!page || fil_page_type_is_index(page_type)); - recv_addr->state = RECV_PROCESSED; + /* On a compressed page, MLOG_COMP_REC_MIN_MARK + will be followed by MLOG_COMP_REC_DELETE + or MLOG_ZIP_WRITE_HEADER(FIL_PAGE_PREV, FIL_nullptr) + in the same mini-transaction. */ - ut_a(recv_sys->n_addrs > 0); - --recv_sys->n_addrs; + ut_a(type == MLOG_COMP_REC_MIN_MARK || !page_zip); - mutex_exit(&recv_sys->mutex); + ptr = btr_parse_set_min_rec_mark( + ptr, end_ptr, type == MLOG_COMP_REC_MIN_MARK, + page, mtr); -#ifdef UNIV_HOTBACKUP - ib::trace_2() - << "Applied " << applied_recs - << " Skipped " << skipped_recs; -#endif /* UNIV_HOTBACKUP */ -} + break; -#ifndef UNIV_HOTBACKUP -/** Reads in pages which have hashed log records, from an area around a given -page number. -@param[in] page_id Read the pages around this page number -@return number of pages found */ -static -ulint -recv_read_in_area(const page_id_t& page_id) -{ - page_no_t low_limit; + case MLOG_REC_DELETE: + case MLOG_COMP_REC_DELETE: - low_limit = page_id.page_no() - - (page_id.page_no() % RECV_READ_AHEAD_AREA); + ut_ad(!page || fil_page_type_is_index(page_type)); - ulint n = 0; + if (nullptr != (ptr = mlog_parse_index( + ptr, end_ptr, + type == MLOG_COMP_REC_DELETE, + &index))) { + ut_a(!page + || (ibool)!!page_is_comp(page) + == dict_table_is_comp(index->table)); - std::array page_nos; + ptr = page_cur_parse_delete_rec( + ptr, end_ptr, block, index, mtr); + } - for (page_no_t page_no = low_limit; - page_no < low_limit + RECV_READ_AHEAD_AREA; - ++page_no) { + break; - recv_addr_t* recv_addr; + case MLOG_IBUF_BITMAP_INIT: - recv_addr = recv_get_rec(page_id.space(), page_no); + /* Allow anything in page_type when creating a page. */ - const page_id_t cur_page_id(page_id.space(), page_no); + ptr = ibuf_parse_bitmap_init(ptr, end_ptr, block, mtr); - if (recv_addr != nullptr && !buf_page_peek(cur_page_id)) { + break; - mutex_enter(&recv_sys->mutex); + case MLOG_INIT_FILE_PAGE: + case MLOG_INIT_FILE_PAGE2: - if (recv_addr->state == RECV_NOT_PROCESSED) { + /* Allow anything in page_type when creating a page. */ - recv_addr->state = RECV_BEING_READ; + ptr = fsp_parse_init_file_page(ptr, end_ptr, block); - page_nos[n] = page_no; + break; - ++n; - } + case MLOG_WRITE_STRING: - mutex_exit(&recv_sys->mutex); - } - } + ut_ad(!page || page_type != FIL_PAGE_TYPE_ALLOCATED + || page_no == 0); - buf_read_recv_pages(false, page_id.space(), &page_nos[0], n); + ptr = mlog_parse_string(ptr, end_ptr, page, page_zip); - return(n); -} + break; -/** Apply the log records to a page -@param[in,out] recv_addr Redo log records to apply */ -static -void -recv_apply_log_rec(recv_addr_t* recv_addr) -{ - if (recv_addr->state == RECV_DISCARDED) { - ut_a(recv_sys->n_addrs); - --recv_sys->n_addrs; - return; - } + case MLOG_ZIP_WRITE_NODE_PTR: - bool found; - const page_id_t page_id(recv_addr->space, recv_addr->page_no); + ut_ad(!page || fil_page_type_is_index(page_type)); - const page_size_t page_size = - fil_space_get_page_size(recv_addr->space, &found); + ptr = page_zip_parse_write_node_ptr( + ptr, end_ptr, page, page_zip); - if (!found - || recv_sys->missing_ids.find(recv_addr->space) - != recv_sys->missing_ids.end()) { + break; - /* Tablespace was discarded or dropped after changes were - made to it. Or, we have ignored redo log for this tablespace - earlier and somehow it has been found now. We can't apply - this redo log out of order. */ + case MLOG_ZIP_WRITE_BLOB_PTR: - recv_addr->state = RECV_PROCESSED; + ut_ad(!page || fil_page_type_is_index(page_type)); - ut_a(recv_sys->n_addrs > 0); - --recv_sys->n_addrs; + ptr = page_zip_parse_write_blob_ptr( + ptr, end_ptr, page, page_zip); - /* If the tablespace has been explicitly deleted, we - can safely ignore it. */ + break; - if (recv_sys->deleted.find(recv_addr->space) - == recv_sys->deleted.end()) { + case MLOG_ZIP_WRITE_HEADER: - recv_sys->missing_ids.insert(recv_addr->space); - } + ut_ad(!page || fil_page_type_is_index(page_type)); - } else if (recv_addr->state == RECV_NOT_PROCESSED) { + ptr = page_zip_parse_write_header( + ptr, end_ptr, page, page_zip); - mutex_exit(&recv_sys->mutex); + break; - if (buf_page_peek(page_id)) { + case MLOG_ZIP_PAGE_COMPRESS: - mtr_t mtr; + /* Allow anything in page_type when creating a page. */ + ptr = page_zip_parse_compress(ptr, end_ptr, page, page_zip); + break; - mtr_start(&mtr); + case MLOG_ZIP_PAGE_COMPRESS_NO_DATA: - buf_block_t* block; + if (nullptr != (ptr = mlog_parse_index( + ptr, end_ptr, true, &index))) { - block = buf_page_get( - page_id, page_size, RW_X_LATCH, &mtr); + ut_a(!page || ((ibool)!!page_is_comp(page) + == dict_table_is_comp(index->table))); - buf_block_dbg_add_level(block, SYNC_NO_ORDER_CHECK); + ptr = page_zip_parse_compress_no_data( + ptr, end_ptr, page, page_zip, index); + } - recv_recover_page(false, block); + break; - mtr_commit(&mtr); + default: + ptr = nullptr; + recv_sys->found_corrupt_log = true; + } - } else { - recv_read_in_area(page_id); - } + if (index != nullptr) { - mutex_enter(&recv_sys->mutex); + dict_table_t* table = index->table; + + dict_mem_index_free(index); + dict_mem_table_free(table); } + + return(ptr); } -/** Empties the hash table of stored log records, applying them to appropriate -pages. -@param[in] allow_ibuf if true, ibuf operations are allowed during - the application; if false, no ibuf operations - are allowed, and after the application all - file pages are flushed to disk and invalidated - in buffer pool: this alternative means that - no new log records can be generated during - the application; the caller must in this case - own the log mutex */ +/** Adds a new log record to the hash table of log records. +@param[in] type log record type +@param[in] space_id Tablespace id +@param[in] page_no page number +@param[in] body log record body +@param[in] rec_end log record end +@param[in] start_lsn start lsn of the mtr +@param[in] end_lsn end lsn of the mtr */ +static void -recv_apply_hashed_log_recs(bool allow_ibuf) +recv_add_to_hash_table( + mlog_id_t type, + space_id_t space_id, + page_no_t page_no, + byte* body, + byte* rec_end, + lsn_t start_lsn, + lsn_t end_lsn) { - for (;;) { + ut_ad(type != MLOG_FILE_DELETE); + ut_ad(type != MLOG_FILE_CREATE); + ut_ad(type != MLOG_FILE_RENAME); + ut_ad(type != MLOG_DUMMY_RECORD); + ut_ad(type != MLOG_INDEX_LOAD); + + recv_sys_t::Space* space; + + space = recv_get_page_map(space_id, true); - mutex_enter(&recv_sys->mutex); + recv_t* recv; - if (!recv_sys->apply_batch_on) { + recv = static_cast( + mem_heap_alloc(space->m_heap, sizeof(*recv))); - break; - } + recv->type = type; + recv->end_lsn = end_lsn; + recv->len = rec_end - body; + recv->start_lsn = start_lsn; - mutex_exit(&recv_sys->mutex); + auto it = space->m_pages.find(page_no); - os_thread_sleep(500000); - } + recv_addr_t* recv_addr; - ut_ad(!allow_ibuf == log_mutex_own()); + if (it != space->m_pages.end()) { - if (!allow_ibuf) { - recv_no_ibuf_operations = true; - } + recv_addr = it->second; - recv_sys->apply_log_recs = true; - recv_sys->apply_batch_on = true; + } else { - auto batch_size = recv_sys->n_addrs; + recv_addr = static_cast( + mem_heap_alloc(space->m_heap, sizeof(*recv_addr))); - ib::info() - << "Applying a batch of " - << batch_size - << " redo log records ..."; + recv_addr->space = space_id; + recv_addr->page_no = page_no; + recv_addr->state = RECV_NOT_PROCESSED; - static const size_t PCT = 10; + UT_LIST_INIT(recv_addr->rec_list, &recv_t::rec_list); - size_t pct = PCT; - size_t applied = 0; - auto unit = batch_size / PCT; + using value_type = recv_sys_t::Pages::value_type; - if (unit <= PCT) { - pct = 100; - unit = batch_size; + space->m_pages.insert(it, value_type(page_no, recv_addr)); + + ++recv_sys->n_addrs; } - for (const auto& space : *recv_sys->spaces) { + UT_LIST_ADD_LAST(recv_addr->rec_list, recv); - fil_tablespace_open_for_recovery(space.first); + recv_data_t** prev_field; - for (auto pages : space.second.m_pages) { + prev_field = &recv->data; - ut_ad(pages.second->space == space.first); + /* Store the log record body in chunks of less than UNIV_PAGE_SIZE: + the heap grows into the buffer pool, and bigger chunks could not + be allocated */ - recv_apply_log_rec(pages.second); + while (rec_end > body) { - ++applied; + ulint len = rec_end - body; - if (unit == 0 || (applied % unit) == 0) { - ib::info() << pct << "%"; - pct += PCT; - } + if (len > RECV_DATA_BLOCK_SIZE) { + len = RECV_DATA_BLOCK_SIZE; } - } - /* Wait until all the pages have been processed */ + recv_data_t* recv_data; - while (recv_sys->n_addrs != 0) { + recv_data = static_cast( + mem_heap_alloc( + space->m_heap, sizeof(*recv_data) + len)); - mutex_exit(&recv_sys->mutex); + *prev_field = recv_data; - os_thread_sleep(500000); + memcpy(recv_data + 1, body, len); - mutex_enter(&recv_sys->mutex); + prev_field = &recv_data->next; + + body += len; } - if (!allow_ibuf) { + *prev_field = nullptr; +} - /* Flush all the file pages to disk and invalidate them in - the buffer pool */ +/** Copies the log record body from recv to buf. +@param[in] buf Buffer of length at least recv->len +@param[in] recv Log record */ +static +void +recv_data_copy_to_buf(byte* buf, recv_t* recv) +{ + ulint len = recv->len; + recv_data_t* recv_data = recv->data; - ut_d(log_sys->disable_redo_writes = true); + while (len > 0) { - mutex_exit(&recv_sys->mutex); + ulint part_len; - log_mutex_exit(); + if (len > RECV_DATA_BLOCK_SIZE) { + part_len = RECV_DATA_BLOCK_SIZE; + } else { + part_len = len; + } - /* Stop the recv_writer thread from issuing any LRU - flush batches. */ - mutex_enter(&recv_sys->writer_mutex); + memcpy(buf, ((byte*) recv_data) + sizeof(*recv_data), part_len); - /* Wait for any currently run batch to end. */ - buf_flush_wait_LRU_batch_end(); + buf += part_len; + len -= part_len; - os_event_reset(recv_sys->flush_end); + recv_data = recv_data->next; + } +} - recv_sys->flush_type = BUF_FLUSH_LIST; +/** Applies the hashed log records to the page, if the page lsn is less than the +lsn of a log record. This can be called when a buffer page has just been +read in, or also for a page already in the buffer pool. +@param[in] just_read_in true if the IO handler calls this for a freshly + read page +@param[in,out] block Buffer block */ +void +recv_recover_page_func( +#ifndef UNIV_HOTBACKUP + bool just_read_in, +#endif /* !UNIV_HOTBACKUP */ + buf_block_t* block) +{ + mutex_enter(&recv_sys->mutex); - os_event_set(recv_sys->flush_start); + if (recv_sys->apply_log_recs == false) { - os_event_wait(recv_sys->flush_end); + /* Log records should not be applied now */ - buf_pool_invalidate(); + mutex_exit(&recv_sys->mutex); - /* Allow batches from recv_writer thread. */ - mutex_exit(&recv_sys->writer_mutex); + return; + } - log_mutex_enter(); + recv_addr_t* recv_addr; - ut_d(log_sys->disable_redo_writes = false); + recv_addr = recv_get_rec( + block->page.id.space(), block->page.id.page_no()); - mutex_enter(&recv_sys->mutex); + if (recv_addr == nullptr + || recv_addr->state == RECV_BEING_PROCESSED + || recv_addr->state == RECV_PROCESSED) { - recv_no_ibuf_operations = false; - } + ut_ad(recv_addr == nullptr || recv_needed_recovery); - recv_sys->apply_log_recs = false; - recv_sys->apply_batch_on = false; + mutex_exit(&recv_sys->mutex); - recv_sys_empty_hash(); + return; + } - mutex_exit(&recv_sys->mutex); +#ifndef UNIV_HOTBACKUP + /* this is explicitly false in case of meb, skip the assert */ + ut_ad(recv_needed_recovery); - ib::info() << "Apply batch completed!"; -} + DBUG_PRINT("ib_log", + ("Applying log to page %u:%u", + recv_addr->space, recv_addr->page_no)); + +# ifdef UNIV_DEBUG + lsn_t max_lsn; + ut_d(max_lsn = UT_LIST_GET_FIRST(log_sys->log_groups)->scanned_lsn); +# endif /* UNIV_DEBUG */ #else /* !UNIV_HOTBACKUP */ + ib::trace_2() + << "Applying log to space " << recv_addr->space + << " page " << recv_addr->page_no; +#endif /* !UNIV_HOTBACKUP */ -/** Apply a single log record stored in the hash table. -@param[in,out] recv_addr a parsed log record -@param[in,out] block a buffer pool frame for applying the record */ -void -meb_apply_log_record( - recv_addr_t* recv_addr, - buf_block_t* block) -{ - bool found; - const page_id_t page_id(recv_addr->space, recv_addr->page_no); + recv_addr->state = RECV_BEING_PROCESSED; - const page_size_t& page_size = - fil_space_get_page_size(recv_addr->space, &found); + mutex_exit(&recv_sys->mutex); - ib::trace_3() - << "recv_addr {State: " << recv_addr->state - << ", Space id: " << recv_addr->space - << ", Page no: " << recv_addr->page_no - << ", Page size: " << page_size - << ", found: " << found << "\n"; + mtr_t mtr; - if (!found) { + mtr_start(&mtr); - recv_addr->state = RECV_DISCARDED; + mtr_set_log_mode(&mtr, MTR_LOG_NONE); - mutex_enter(&(recv_sys->mutex)); + page_t* page = block->frame; - ut_a(recv_sys->n_addrs); - --recv_sys->n_addrs; + page_zip_des_t* page_zip = buf_block_get_page_zip(block); - mutex_exit(&(recv_sys->mutex)); +#ifndef UNIV_HOTBACKUP + if (just_read_in) { + /* Move the ownership of the x-latch on the page to + this OS thread, so that we can acquire a second + x-latch on it. This is needed for the operations to + the page to pass the debug checks. */ - return; + rw_lock_x_lock_move_ownership(&block->lock); } - mutex_enter(&(recv_sys->mutex)); + bool success = buf_page_get_known_nowait( + RW_X_LATCH, block, BUF_KEEP_OLD, + __FILE__, __LINE__, &mtr); + ut_a(success); - /* We simulate a page read made by the buffer pool, to - make sure the recovery apparatus works ok. We must init - the block. */ + buf_block_dbg_add_level(block, SYNC_NO_ORDER_CHECK); +#endif /* !UNIV_HOTBACKUP */ - meb_page_init(page_id, page_size, block); + /* Read the newest modification lsn from the page */ + lsn_t page_lsn = mach_read_from_8(page + FIL_PAGE_LSN); - /* Extend the tablespace's last file if the page_no - does not fall inside its bounds; we assume the last - file is auto-extending, and mysqlbackup copied the file - when it still was smaller */ +#ifndef UNIV_HOTBACKUP + + /* It may be that the page has been modified in the buffer + pool: read the newest modification LSN there */ + + lsn_t page_newest_lsn; + + page_newest_lsn = buf_page_get_newest_modification(&block->page); + + if (page_newest_lsn) { + + page_lsn = page_newest_lsn; + } +#else /* !UNIV_HOTBACKUP */ + /* In recovery from a backup we do not really use the buffer pool */ + lsn_t page_newest_lsn = 0; + /* Count applied and skipped log records */ + size_t applied_recs = 0; + size_t skipped_recs = 0; +#endif /* !UNIV_HOTBACKUP */ + +#ifndef UNIV_HOTBACKUP + lsn_t end_lsn = 0; +#endif /* !UNIV_HOTBACKUP */ + lsn_t start_lsn = 0; + bool modification_to_page = false; - fil_space_t* space = fil_space_get(recv_addr->space); + for (auto recv = UT_LIST_GET_FIRST(recv_addr->rec_list); + recv != nullptr; + recv = UT_LIST_GET_NEXT(rec_list, recv)) { - bool success; +#ifndef UNIV_HOTBACKUP + end_lsn = recv->end_lsn; - success = fil_space_extend(space, recv_addr->page_no + 1); + ut_ad(end_lsn <= max_lsn); +#endif /* !UNIV_HOTBACKUP */ - if (!success) { - ib::fatal() - << "Cannot extend tablespace " - << recv_addr->space << " to hold " - << recv_addr->page_no << " pages"; - } + byte* buf; - mutex_exit(&(recv_sys->mutex)); + if (recv->len > RECV_DATA_BLOCK_SIZE) { - /* Read the page from the tablespace file using the - fil0fil.cc routines */ + /* We have to copy the record body to a separate + buffer */ - dberr_t err; + buf = static_cast(ut_malloc_nokey(recv->len)); - if (page_size.is_compressed()) { + recv_data_copy_to_buf(buf, recv); + } else { + buf = ((byte*)(recv->data)) + sizeof(recv_data_t); + } - err = fil_io( - IORequestRead, true, - page_id, - page_size, 0, page_size.physical(), - block->page.zip.data, NULL); + if (recv->type == MLOG_INIT_FILE_PAGE) { - if (err == DB_SUCCESS && !buf_zip_decompress(block, TRUE)) { + page_lsn = page_newest_lsn; - ut_error; + memset(FIL_PAGE_LSN + page, 0, 8); + memset(UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + + page, 0, 8); + + if (page_zip) { + memset(FIL_PAGE_LSN + page_zip->data, 0, 8); + } } - } else { - err = fil_io( - IORequestRead, true, - page_id, page_size, 0, - page_size.logical(), - block->frame, NULL); - } + /* Ignore applying the redo logs for tablespace that is + truncated. Truncated tablespaces are handled explicitly + post-recovery, where we will restore the tablespace back + to a normal state. - if (err != DB_SUCCESS) { + Applying redo at this stage will cause problems because the + redo will have action recorded on page before tablespace + was re-inited and that would lead to a problem later. */ - ib::fatal() - << "Cannot read from tablespace " - << recv_addr->space << " page number " - << recv_addr->page_no; - } + if (recv->start_lsn >= page_lsn +#ifndef UNIV_HOTBACKUP + && undo::is_active(recv_addr->space) +#endif /* !UNIV_HOTBACKUP */ + ) { - apply_log_mutex.lock(); + lsn_t end_lsn; - /* Apply the log records to this page */ - recv_recover_page(false, block); + if (!modification_to_page) { - apply_log_mutex.unlock(); + modification_to_page = true; + start_lsn = recv->start_lsn; + } - mutex_enter(&(recv_sys->mutex)); + DBUG_PRINT("ib_log", + ("apply " LSN_PF ":" + " %s len " ULINTPF " page %u:%u", + recv->start_lsn, + get_mlog_string(recv->type), recv->len, + recv_addr->space, + recv_addr->page_no)); - /* Write the page back to the tablespace file using the - fil0fil.cc routines */ + recv_parse_or_apply_log_rec_body( + recv->type, buf, buf + recv->len, + recv_addr->space, recv_addr->page_no, + block, &mtr, ULINT_UNDEFINED); - buf_flush_init_for_writing( - block, block->frame, buf_block_get_page_zip(block), - mach_read_from_8(block->frame + FIL_PAGE_LSN), - fsp_is_checksum_disabled(block->page.id.space())); + end_lsn = recv->start_lsn + recv->len; - mutex_exit(&(recv_sys->mutex)); + mach_write_to_8(FIL_PAGE_LSN + page, end_lsn); - if (page_size.is_compressed()) { + mach_write_to_8(UNIV_PAGE_SIZE + - FIL_PAGE_END_LSN_OLD_CHKSUM + + page, end_lsn); - err = fil_io( - IORequestWrite, true, page_id, - page_size, 0, page_size.physical(), - block->page.zip.data, NULL); - } else { - err = fil_io( - IORequestWrite, true, page_id, - page_size, 0, page_size.logical(), - block->frame, NULL); - } + if (page_zip) { - if (err != DB_SUCCESS) { + mach_write_to_8(FIL_PAGE_LSN + + page_zip->data, end_lsn); + } +#ifdef UNIV_HOTBACKUP + ++applied_recs; + } else { + ++skipped_recs; +#endif /* UNIV_HOTBACKUP */ + } - ib::fatal() - << "Cannot write to tablespace " - << recv_addr->space << " page number " - << recv_addr->page_no; + if (recv->len > RECV_DATA_BLOCK_SIZE) { + ut_free(buf); + } } -} - -/** Apply a single log record stored in the hash table using default block. -@param[in,out] recv_addr a parsed log record */ -void -meb_apply_log_rec_func( - recv_addr_t* recv_addr) -{ - meb_apply_log_record(recv_addr, back_block1); -} -/** Dummy wait function for meb_apply_log_recs_via_callback(). */ -void -meb_nowait_func() -{ - return; -} +#ifdef UNIV_ZIP_DEBUG + if (fil_page_index_page_check(page)) { + page_zip_des_t* page_zip = buf_block_get_page_zip(block); -/** Applies log records in the hash table to a backup. */ -void -meb_apply_log_recs() -{ - meb_apply_log_recs_via_callback( - meb_apply_log_rec_func, meb_nowait_func); -} + ut_a(!page_zip + || page_zip_validate_low(page_zip, page, nullptr, FALSE)); + } +#endif /* UNIV_ZIP_DEBUG */ -/** Apply all log records in the hash table to a backup using callback -functions. This function employes two callback functions that allow redo -log records to be applied in parallel. The apply_log_record_function -assigns a parsed redo log record for application. The -apply_log_record_function is called repeatedly until all log records in -the hash table are assigned for application. After that the -wait_till_done_function is called once. The wait_till_done_function -function blocks until the application of all the redo log records -previously assigned with apply_log_record_function calls is complete. -Even though this function assigns the log records in the hash table -sequentially, the application of the log records may be done in parallel -if the apply_log_record_function delegates the actual application work -to multiple worker threads running in parallel. -@param[in] apply_log_record_function a function that assigns one redo log -record for application -@param[in] wait_till_done_function a function that blocks until all -assigned redo log records have been applied */ -void -meb_apply_log_recs_via_callback( - void (*apply_log_record_function)(recv_addr_t*), - void (*wait_till_done_function)()) -{ - ulint n_hash_cells = recv_sys->n_addrs; - ulint i = 0; +#ifndef UNIV_HOTBACKUP + if (modification_to_page) { + ut_a(block); - recv_sys->apply_log_recs = true; - recv_sys->apply_batch_on = true; + log_flush_order_mutex_enter(); + buf_flush_recv_note_modification(block, start_lsn, end_lsn); + log_flush_order_mutex_exit(); + } +#else /* !UNIV_HOTBACKUP */ + UT_NOT_USED(start_lsn); +#endif /* !UNIV_HOTBACKUP */ - ib::info() - << "Starting to apply a batch of log records to the" - << " database..."; + /* Make sure that committing mtr does not change the modification + LSN values of page */ - fputs("InnoDB: Progress in percent: ", stderr); + mtr.discard_modifications(); - for (const auto& space : *recv_sys->spaces) { + mtr_commit(&mtr); - for (auto pages : space.second.m_pages) { + mutex_enter(&recv_sys->mutex); - ut_ad(pages.second->space == space.first); + if (recv_max_page_lsn < page_lsn) { + recv_max_page_lsn = page_lsn; + } - (*apply_log_record_function)(pages.second); - } + recv_addr->state = RECV_PROCESSED; - ++i; - if ((100 * i) / n_hash_cells - != (100 * (i + 1)) / n_hash_cells) { - fprintf(stderr, "%lu ", - (ulong) ((100 * i) / n_hash_cells)); - fflush(stderr); - } - } + ut_a(recv_sys->n_addrs > 0); + --recv_sys->n_addrs; - /* wait till all the redo log records have been applied */ - (*wait_till_done_function)(); + mutex_exit(&recv_sys->mutex); - /* write logs in next line */ - fprintf(stderr, "\n"); - recv_sys->apply_log_recs = false; - recv_sys->apply_batch_on = false; - recv_sys_empty_hash(); +#ifdef UNIV_HOTBACKUP + ib::trace_2() + << "Applied " << applied_recs + << " Skipped " << skipped_recs; +#endif /* UNIV_HOTBACKUP */ } -#endif /* !UNIV_HOTBACKUP */ - /** Tries to parse a single log record. @param[out] type log record type @param[in] ptr pointer to a buffer @@ -3259,10 +3155,9 @@ recv_single_rec( /* fall through */ case MLOG_INDEX_LOAD: - case MLOG_FILE_OPEN: case MLOG_FILE_DELETE: - case MLOG_FILE_RENAME2: - case MLOG_FILE_CREATE2: + case MLOG_FILE_RENAME: + case MLOG_FILE_CREATE: case MLOG_TABLE_DYNAMIC_META: /* These were already handled by @@ -3423,10 +3318,9 @@ recv_multi_rec(byte* ptr, byte* end_ptr) break; #endif /* UNIV_LOG_LSN_DEBUG */ - case MLOG_FILE_OPEN: case MLOG_FILE_DELETE: - case MLOG_FILE_CREATE2: - case MLOG_FILE_RENAME2: + case MLOG_FILE_CREATE: + case MLOG_FILE_RENAME: case MLOG_TABLE_DYNAMIC_META: /* case MLOG_TRUNCATE: Disabled for WL6378 */ /* These were already handled by @@ -3883,11 +3777,16 @@ recv_read_log_seg( const page_no_t page_no = static_cast( source_offset / univ_page_size.physical()); - fil_io(IORequestLogRead, true, - page_id_t(group->space_id, page_no), - univ_page_size, - (ulint) (source_offset % univ_page_size.physical()), - len, buf, NULL); + dberr_t + + err = fil_redo_io( + IORequestLogRead, + page_id_t(group->space_id, page_no), + univ_page_size, + (ulint) (source_offset % univ_page_size.physical()), + len, buf); + + ut_a(err == DB_SUCCESS); start_lsn += len; buf += len; @@ -3917,6 +3816,7 @@ recv_recovery_begin( recv_sys->parse_start_lsn = *contiguous_lsn; recv_sys->scanned_lsn = *contiguous_lsn; recv_sys->recovered_lsn = *contiguous_lsn; + recv_sys->scanned_checkpoint_no = 0; recv_previous_parsed_rec_type = MLOG_SINGLE_REC_FLAG; recv_previous_parsed_rec_offset = 0; @@ -3975,11 +3875,6 @@ recv_init_crash_recovery() ib::info() << "Database was not shutdown normally!"; ib::info() << "Starting crash recovery."; - /* Open the tablespace ID to file name mapping file. Required for - redo log apply and dblwr buffer page restore. */ - - fil_tablespace_open_init_for_recovery(true); - buf_dblwr_process(); if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { @@ -4026,7 +3921,7 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) log_group_t* max_cp_group; ulint max_cp_field; - err = recv_find_max_checkpoint(&max_cp_group, &max_cp_field); + err = recv_find_max_checkpoint(max_cp_group, &max_cp_field); if (err != DB_SUCCESS) { @@ -4055,8 +3950,11 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) byte log_hdr_buf[LOG_FILE_HDR_SIZE]; const page_id_t page_id(max_cp_group->space_id, 0); - fil_io(IORequestLogRead, true, page_id, univ_page_size, 0, - LOG_FILE_HDR_SIZE, log_hdr_buf, max_cp_group); + err = fil_redo_io( + IORequestLogRead, page_id, univ_page_size, 0, + LOG_FILE_HDR_SIZE, log_hdr_buf); + + ut_a(err == DB_SUCCESS); if (0 == ut_memcmp( log_hdr_buf + LOG_HEADER_CREATOR, (byte*) "MEB", @@ -4098,9 +3996,12 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) log_block_calc_checksum_crc32(log_hdr_buf)); /* Write to the log file to wipe over the label */ - fil_io(IORequestLogWrite, true, page_id, - univ_page_size, 0, OS_FILE_LOG_BLOCK_SIZE, log_hdr_buf, - max_cp_group); + err = fil_redo_io( + IORequestLogWrite, page_id, + univ_page_size, 0, OS_FILE_LOG_BLOCK_SIZE, + log_hdr_buf); + + ut_a(err == DB_SUCCESS); } else if (0 == ut_memcmp(log_hdr_buf + LOG_HEADER_CREATOR, (byte*)LOG_HEADER_CREATOR_CLONE, @@ -4129,14 +4030,30 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) contiguous_lsn = checkpoint_lsn; switch (group->format) { + case LOG_HEADER_FORMAT_CURRENT: + break; + case LOG_HEADER_FORMAT_5_7_9: + case LOG_HEADER_FORMAT_8_0_1: + + ib::info() + << "Redo log is from an earlier version," + << " v" << group->format << "."; + + /* Check if the redo log from an older known redo log + version is from a clean shutdown. */ + err = recv_log_recover_pre_8_0_4(checkpoint_lsn); + log_mutex_exit(); - return(recv_log_recover_5_7(checkpoint_lsn)); - case LOG_HEADER_FORMAT_CURRENT: - break; + return(err); default: + ib::error() + << "Redo log format (" << group->format << ")" + << " not supported. Current supported format is" + << " v" << LOG_HEADER_FORMAT_CURRENT; + ut_ad(0); recv_sys->found_corrupt_log = true; log_mutex_exit(); @@ -4184,9 +4101,6 @@ recv_recovery_from_checkpoint_start(lsn_t flush_lsn) recv_init_crash_recovery(); } - } else if (!srv_read_only_mode) { - /* Read the UNDO tablespace locations only. */ - fil_tablespace_open_init_for_recovery(false); } contiguous_lsn = checkpoint_lsn; @@ -4314,7 +4228,7 @@ recv_recovery_from_checkpoint_finish(bool aborting) } MetadataRecover* metadata; - + if (!aborting) { metadata = recv_sys->metadata_recover; @@ -4366,8 +4280,6 @@ recv_recovery_from_checkpoint_finish(bool aborting) fil_block_check_type(block, FIL_PAGE_TYPE_SYS, &mtr); mtr.commit(); - - fil_tablespace_open_create(); } /* Free up the flush_rbt. */ @@ -4600,8 +4512,8 @@ get_mlog_string(mlog_id_t type) case MLOG_COMP_PAGE_REORGANIZE: return("MLOG_COMP_PAGE_REORGANIZE"); - case MLOG_FILE_CREATE2: - return("MLOG_FILE_CREATE2"); + case MLOG_FILE_CREATE: + return("MLOG_FILE_CREATE"); case MLOG_ZIP_WRITE_NODE_PTR: return("MLOG_ZIP_WRITE_NODE_PTR"); @@ -4621,11 +4533,8 @@ get_mlog_string(mlog_id_t type) case MLOG_ZIP_PAGE_REORGANIZE: return("MLOG_ZIP_PAGE_REORGANIZE"); - case MLOG_FILE_RENAME2: - return("MLOG_FILE_RENAME2"); - - case MLOG_FILE_OPEN: - return("MLOG_FILE_OPEN"); + case MLOG_FILE_RENAME: + return("MLOG_FILE_RENAME"); case MLOG_PAGE_CREATE_RTREE: return("MLOG_PAGE_CREATE_RTREE"); diff --git a/storage/innobase/os/os0file.cc b/storage/innobase/os/os0file.cc index 8d8bae628b27..4d1e16e3bf4f 100644 --- a/storage/innobase/os/os0file.cc +++ b/storage/innobase/os/os0file.cc @@ -46,23 +46,19 @@ Created 10/21/1995 Heikki Tuuri # include "os0event.h" # include "os0thread.h" #endif /* !UNIV_HOTBACKUP */ + #ifdef _WIN32 -# include # include # include # include # include # include -# endif /* _WIN32 */ +#endif /* _WIN32 */ #ifdef __linux__ #include #endif /* __linux__ */ -#include -#include -#include - #ifdef LINUX_NATIVE_AIO # ifndef UNIV_HOTBACKUP # include @@ -78,15 +74,17 @@ Created 10/21/1995 Heikki Tuuri #include #include -#include -#include -#include -#ifndef UNIV_HOTBACKUP -# include -#endif /* !UNIV_HOTBACKUP */ -#include -#include +#include "my_aes.h" +#include "my_rnd.h" +#include "mysql/service_mysql_keyring.h" +#include "mysqld.h" + +#include +#include #include +#include +#include +#include #ifdef UNIV_HOTBACKUP # include @@ -662,14 +660,14 @@ class AIO { MY_ATTRIBUTE((warn_unused_result)); /** Select the IO slot array - @param[in] type Type of IO, READ or WRITE + @param[in,out] type Type of IO, READ or WRITE @param[in] read_only true if running in read-only mode - @param[in] mode IO mode + @param[in] aio_mode IO mode @return slot array or NULL if invalid mode specified */ static AIO* select_slot_array( IORequest& type, bool read_only, - ulint mode) + AIO_mode aio_mode) MY_ATTRIBUTE((warn_unused_result)); /** Calculates segment number for a slot. @@ -979,6 +977,66 @@ os_aio_windows_handler( IORequest* type); #endif /* WIN_ASYNC_IO */ +/** Check the file type and determine if it can be deleted. +@param[in] name Filename/Path to check +@return true if it's a file or a symlink and can be deleted */ +static +bool +os_file_can_delete(const char* name) +{ + switch (Fil_path::get_file_type(name)) { + case OS_FILE_TYPE_FILE: + case OS_FILE_TYPE_LINK: + return(true); + + case OS_FILE_TYPE_DIR: + + ib::warn() + << "'" << name << "'" + << " is a directory, can't delete!"; + break; + + case OS_FILE_TYPE_BLOCK: + + ib::warn() + << "'" << name << "'" + << " is a block device, can't delete!"; + break; + + case OS_FILE_TYPE_FAILED: + + ib::warn() + << "'" << name << "'" + << " get file type failed, won't delete!"; + break; + + case OS_FILE_TYPE_UNKNOWN: + + ib::warn() + << "'" << name << "'" + << " unknown file type, won't delete!"; + break; + + case OS_FILE_TYPE_NAME_TOO_LONG: + + ib::warn() + << "'" << name << "'" + << " name too long, can't delete!"; + break; + + case OS_FILE_PERMISSION_ERROR: + ib::warn() + << "'" << name << "'" + << " permission error, can't delete!"; + break; + + case OS_FILE_TYPE_MISSING: + break; + } + + return(false); +} + /** Allocate a page for sync IO @return pointer to page */ static @@ -1862,103 +1920,6 @@ os_file_io_complete( return(DB_SUCCESS); } -/** This function returns a new path name after replacing the basename -in an old path with a new basename. The old_path is a full path -name including the extension. The tablename is in the normal -form "databasename/tablename". The new base name is found after -the forward slash. Both input strings are null terminated. - -This function allocates memory to be returned. It is the callers -responsibility to free the return value after it is no longer needed. - -@param[in] old_path Pathname -@param[in] tablename Contains new base name -@return own: new full pathname */ -char* -os_file_make_new_pathname( - const char* old_path, - const char* tablename) -{ - ulint dir_len; - char* last_slash; - char* base_name; - char* new_path; - ulint new_path_len; - - /* Split the tablename into its database and table name components. - They are separated by a '/'. */ - last_slash = strrchr((char*) tablename, '/'); - base_name = last_slash ? last_slash + 1 : (char*) tablename; - - /* Find the offset of the last slash. We will strip off the - old basename.ibd which starts after that slash. */ - last_slash = strrchr((char*) old_path, OS_PATH_SEPARATOR); - dir_len = last_slash ? last_slash - old_path : strlen(old_path); - - /* allocate a new path and move the old directory path to it. */ - new_path_len = dir_len + strlen(base_name) + sizeof "/.ibd"; - new_path = static_cast(ut_malloc_nokey(new_path_len)); - memcpy(new_path, old_path, dir_len); - - snprintf(new_path + dir_len, - new_path_len - dir_len, - "%c%s.ibd", - OS_PATH_SEPARATOR, - base_name); - - return(new_path); -} - -/** This function reduces a null-terminated full remote path name into -the path that is sent by MySQL for DATA DIRECTORY clause. It replaces -the 'databasename/tablename.ibd' found at the end of the path with just -'tablename'. - -Since the result is always smaller than the path sent in, no new memory -is allocated. The caller should allocate memory for the path sent in. -This function manipulates that path in place. - -If the path format is not as expected, just return. The result is used -to inform a SHOW CREATE TABLE command. -@param[in,out] data_dir_path Full path/data_dir_path */ -void -os_file_make_data_dir_path( - char* data_dir_path) -{ - /* Replace the period before the extension with a null byte. */ - char* ptr = strrchr((char*) data_dir_path, '.'); - - if (ptr == NULL) { - return; - } - - ptr[0] = '\0'; - - /* The tablename starts after the last slash. */ - ptr = strrchr((char*) data_dir_path, OS_PATH_SEPARATOR); - - if (ptr == NULL) { - return; - } - - ptr[0] = '\0'; - - char* tablename = ptr + 1; - - /* The databasename starts after the next to last slash. */ - ptr = strrchr((char*) data_dir_path, OS_PATH_SEPARATOR); - - if (ptr == NULL) { - return; - } - - ulint tablename_len = ut_strlen(tablename); - - ut_memmove(++ptr, tablename, tablename_len); - - ptr[tablename_len] = '\0'; -} - /** Check if the path refers to the root of a drive using a pointer to the last directory separator that the caller has fixed. @param[in] path path name @@ -2059,8 +2020,8 @@ test_os_file_get_parent_dir( /* os_file_get_parent_dir() assumes that separators are converted to OS_PATH_SEPARATOR. */ - os_normalize_path(child); - os_normalize_path(expected); + Fil_path::normalize(child); + Fil_path::normalize(expected); char* parent = os_file_get_parent_dir(child); @@ -3015,8 +2976,6 @@ AIO::is_linux_native_aio_supported() } } else { - os_normalize_path(srv_log_group_home_dir); - ulint dirnamelen = strlen(srv_log_group_home_dir); ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile"); @@ -3191,6 +3150,8 @@ os_file_get_last_error_low( break; case EACCES: return(OS_FILE_ACCESS_VIOLATION); + case ENAMETOOLONG: + return(OS_FILE_NAME_TOO_LONG); } return(OS_FILE_ERROR_MAX + err); } @@ -3291,16 +3252,34 @@ os_file_status_posix( /* file exists, everything OK */ } else if (errno == ENOENT || errno == ENOTDIR) { + + if (exists != nullptr) { + *exists = false; + } + /* file does not exist */ *type = OS_FILE_TYPE_MISSING; return(true); + } else if (errno == ENAMETOOLONG) { + *type = OS_FILE_TYPE_NAME_TOO_LONG; + return(false); + } else if (errno == EACCES) { + *type = OS_FILE_PERMISSION_ERROR; + return(false); } else { + + *type = OS_FILE_TYPE_FAILED; + /* file exists, but stat call failed */ os_file_handle_error_no_exit(path, "stat", false); return(false); } + if (exists != nullptr) { + *exists = true; + } + if (S_ISDIR(statinfo.st_mode)) { *type = OS_FILE_TYPE_DIR; @@ -3826,17 +3805,24 @@ os_file_delete_if_exists_func( const char* name, bool* exist) { - if (exist != NULL) { + if (!os_file_can_delete(name)) { + return(false); + } + + if (exist != nullptr) { *exist = true; } int ret = unlink(name); if (ret != 0 && errno == ENOENT) { - if (exist != NULL) { + + if (exist != nullptr) { *exist = false; } + } else if (ret != 0 && errno != ENOENT) { + os_file_handle_error_no_exit(name, "delete", false); return(false); @@ -4117,16 +4103,16 @@ Dir_Walker::walk_posix(const Path& basedir, Function&& f) if (parent == nullptr) { - std::cerr - << std::endl + ib::info() << "Failed to walk directory" - << " '" << current.m_path << "'" - << std::endl; + << " '" << current.m_path << "'"; continue; } - f(current.m_path, current.m_depth); + if (!is_directory(current.m_path)) { + f(current.m_path, current.m_depth); + } struct dirent* dirent = nullptr; @@ -4146,7 +4132,12 @@ Dir_Walker::walk_posix(const Path& basedir, Function&& f) Path path(current.m_path); - path.append("/"); + if (path.back() != '/' && path.back() != '\\') { + path += OS_PATH_SEPARATOR; + } + + Fil_path::normalize(dirent->d_name); + path.append(dirent->d_name); if (is_directory(path)) { @@ -4291,30 +4282,50 @@ os_file_status_win32( bool* exists, os_file_type_t* type) { - int ret; struct _stat64 statinfo; - ret = _stat64(path, &statinfo); + int ret = _stat64(path, &statinfo); - *exists = !ret; + if (ret == 0) { - if (!ret) { /* file exists, everything OK */ } else if (errno == ENOENT || errno == ENOTDIR) { + + *type = OS_FILE_TYPE_MISSING; + /* file does not exist */ + + if (exists != nullptr) { + *exists = false; + } + return(true); + } else if (errno == EACCES) { + + *type = OS_FILE_PERMISSION_ERROR; + return(false); + } else { + + *type = OS_FILE_TYPE_FAILED; + /* file exists, but stat call failed */ os_file_handle_error_no_exit(path, "stat", false); return(false); } + if (exists != nullptr) { + *exists = true; + } + if (_S_IFDIR & statinfo.st_mode) { + *type = OS_FILE_TYPE_DIR; } else if (_S_IFREG & statinfo.st_mode) { + *type = OS_FILE_TYPE_FILE; } else { @@ -4854,8 +4865,7 @@ os_file_create_func( if (!read_only) { access |= GENERIC_WRITE; - } else if (type == OS_CLONE_LOG_FILE - || type == OS_CLONE_DATA_FILE) { + } else if (type == OS_CLONE_LOG_FILE || type == OS_CLONE_DATA_FILE) { /* Clone must allow concurrent write to file. */ share_mode |= FILE_SHARE_WRITE; @@ -4980,7 +4990,7 @@ os_file_create_simple_no_error_handling_func( access = GENERIC_READ; - /*!< A backup program has to give mysqld the maximum + /* A backup program has to give mysqld the maximum freedom to do what it likes with the file */ share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE; @@ -5012,16 +5022,18 @@ os_file_create_simple_no_error_handling_func( @param[out] exist indicate if file pre-exist @return true if success */ bool -os_file_delete_if_exists_func( - const char* name, - bool* exist) +os_file_delete_if_exists_func(const char* name, bool* exist) { - ulint count = 0; + if (!os_file_can_delete(name)) { + return(false); + } - if (exist != NULL) { + if (exist != nullptr) { *exist = true; } + ulint count = 0; + for (;;) { /* In Windows, deleting an .ibd file may fail if mysqlbackup is copying it */ @@ -5037,12 +5049,13 @@ os_file_delete_if_exists_func( if (lasterr == ERROR_FILE_NOT_FOUND || lasterr == ERROR_PATH_NOT_FOUND) { - /* the file does not exist, this not an error */ + /* The file does not exist, this not an error */ if (exist != NULL) { *exist = false; } return(true); + } ++count; @@ -5286,7 +5299,7 @@ os_file_get_status_win32( fh = CreateFile( (LPCTSTR) path, // File to open access, - 0, // No sharing + FILE_SHARE_READ, NULL, // Default security OPEN_EXISTING, // Existing file only FILE_ATTRIBUTE_NORMAL, // Normal file @@ -5383,7 +5396,9 @@ os_file_truncate_win32( LARGE_INTEGER length; length.QuadPart = size; - BOOL success = SetFilePointerEx(file.m_file, length, NULL, FILE_BEGIN); + + BOOL success = SetFilePointerEx( + file.m_file, length, NULL, FILE_BEGIN); if (!success) { os_file_handle_error_no_exit( @@ -5520,7 +5535,7 @@ Dir_Walker::walk_win32(const Path& basedir, Function&& f) if (h == INVALID_HANDLE_VALUE) { - ib::warn() + ib::info() << "Directory read failed:" << " '" << current.m_path << "' during scan"; @@ -5626,10 +5641,8 @@ os_file_io( } else { /* Skip encrypt log file header */ if (offset >= LOG_FILE_HDR_SIZE) { - block = os_file_encrypt_log(type, - buf, - encrypt_log_buf, - &n); + block = os_file_encrypt_log( + type, buf, encrypt_log_buf, &n); } } } @@ -6006,6 +6019,9 @@ os_file_handle_error_cond_exit( os_thread_sleep(100000); /* 100 ms */ return(true); + case OS_FILE_NAME_TOO_LONG: + return(false); + default: /* If it is an operation that can crash on error then it @@ -6188,14 +6204,14 @@ os_file_set_size( err = os_file_write( request, name, file, buf, current_size, n_bytes); -#else /* UNIV_HOTBACKUP */ - /* Using OS_AIO_SYNC mode on POSIX systems will result in +#else + /* Using AIO_mode::SYNC mode on POSIX systems will result in fall back to os_file_write/read. On Windows it will use special mechanism to wait before it returns back. */ err = os_aio( request, - OS_AIO_SYNC, name, + AIO_mode::SYNC, name, file, buf, current_size, n_bytes, read_only, NULL, NULL); #endif /* UNIV_HOTBACKUP */ @@ -6682,8 +6698,7 @@ os_file_get_status( return(ret); } -/** -Waits for an AIO operation to complete. This function is used to wait the +/** Waits for an AIO operation to complete. This function is used to wait for completed requests. The aio array of pending requests is divided into segments. The thread specifies which segment or slot it wants to wait for. NOTE: this function will also take care of freeing the aio slot, @@ -7211,7 +7226,6 @@ os_create_block_cache() void meb_free_block_cache() { - if (block_cache == NULL) { return; } @@ -7254,8 +7268,6 @@ os_aio_init( } #endif /* _WIN32 */ - os_create_block_cache(); - /* Get sector size for DIRECT_IO. In this case, we need to know the sector size for aligning the write buffer. */ #if !defined(NO_FALLOCATE) && defined(UNIV_LINUX) @@ -7697,24 +7709,24 @@ os_aio_simulated_wake_handler_threads() } /** Select the IO slot array -@param[in] type Type of IO, READ or WRITE +@param[in,out] type Type of IO, READ or WRITE @param[in] read_only true if running in read-only mode -@param[in] mode IO mode +@param[in] aio_mode IO mode @return slot array or NULL if invalid mode specified */ AIO* -AIO::select_slot_array(IORequest& type, bool read_only, ulint mode) +AIO::select_slot_array(IORequest& type, bool read_only, AIO_mode aio_mode) { AIO* array; ut_ad(type.validate()); - switch (mode) { - case OS_AIO_NORMAL: + switch (aio_mode) { + case AIO_mode::NORMAL: array = type.is_read() ? AIO::s_reads : AIO::s_writes; break; - case OS_AIO_IBUF: + case AIO_mode::IBUF: ut_ad(type.is_read()); /* Reduce probability of deadlock bugs in connection with ibuf: @@ -7725,12 +7737,12 @@ AIO::select_slot_array(IORequest& type, bool read_only, ulint mode) array = read_only ? AIO::s_reads : AIO::s_ibuf; break; - case OS_AIO_LOG: + case AIO_mode::LOG: array = read_only ? AIO::s_reads : AIO::s_log; break; - case OS_AIO_SYNC: + case AIO_mode::SYNC: array = AIO::s_sync; #if defined(LINUX_NATIVE_AIO) @@ -7905,7 +7917,9 @@ os_aio_windows_handler( BOOL ret; ret = GetOverlappedResult( - slot->file.m_file, &slot->control, &slot->n_bytes, TRUE); + slot->file.m_file, &slot->control, + &slot->n_bytes, TRUE); + n_bytes = ret ? slot->n_bytes : -1; } @@ -7926,7 +7940,7 @@ os_aio_windows_handler( NOTE! Use the corresponding macro os_aio(), not directly this function! Requests an asynchronous i/o operation. @param[in] type IO request context -@param[in] mode IO mode +@param[in] aio_mode IO mode @param[in] name Name of the file or path as NUL terminated string @param[in] file Open file handle @@ -7936,15 +7950,15 @@ Requests an asynchronous i/o operation. @param[in] read_only if true read only mode checks are enforced @param[in,out] m1 Message for the AIO handler, (can be used to identify a completed AIO operation); ignored - if mode is OS_AIO_SYNC + if mode is AIO_mode::SYNC @param[in,out] m2 message for the AIO handler (can be used to identify a completed AIO operation); ignored - if mode is OS_AIO_SYNC + if mode is AIO_mode::SYNC @return DB_SUCCESS or error code */ dberr_t os_aio_func( IORequest& type, - ulint mode, + AIO_mode aio_mode, const char* name, pfs_os_file_t file, void* buf, @@ -7969,7 +7983,7 @@ os_aio_func( ut_ad((n & 0xFFFFFFFFUL) == n); #endif /* WIN_ASYNC_IO */ - if (mode == OS_AIO_SYNC + if (aio_mode == AIO_mode::SYNC #ifdef WIN_ASYNC_IO && !srv_use_native_aio #endif /* WIN_ASYNC_IO */ @@ -7988,22 +8002,24 @@ os_aio_func( and os_file_write_func() */ if (type.is_read()) { - return(os_file_read_func(type, file.m_file, buf, offset, n)); + return(os_file_read_func( + type, file.m_file, buf, offset, n)); } ut_ad(type.is_write()); - return(os_file_write_func(type, name, file.m_file, buf, offset, n)); + return(os_file_write_func( + type, name, file.m_file, buf, offset, n)); } try_again: AIO* array; - array = AIO::select_slot_array(type, read_only, mode); + array = AIO::select_slot_array(type, read_only, aio_mode); Slot* slot; - slot = array->reserve_slot(type, m1, m2, file, name, buf, offset, n); + slot = array->reserve_slot(type, m1, m2, file, name, buf, offset, n); if (type.is_read()) { @@ -8052,9 +8068,9 @@ os_aio_func( if (srv_use_native_aio) { if ((ret && slot->len == slot->n_bytes) || (!ret && GetLastError() == ERROR_IO_PENDING)) { - /* aio was queued successfully! */ + /* AIO was queued successfully! */ - if (mode == OS_AIO_SYNC) { + if (aio_mode == AIO_mode::SYNC) { IORequest dummy_type; void* dummy_mess2; struct fil_node_t* dummy_mess1; @@ -9002,7 +9018,8 @@ Encryption::to_string(Type type) /** Generate random encryption value for key and iv. @param[in,out] value Encryption value */ -void Encryption::random_value(byte* value) +void +Encryption::random_value(byte* value) { ut_ad(value != NULL); @@ -9015,40 +9032,43 @@ void Encryption::create_master_key(byte** master_key) { #ifndef UNIV_HOTBACKUP - char* key_type = NULL; size_t key_len; + char* key_type = NULL; char key_name[ENCRYPTION_MASTER_KEY_NAME_MAX_LEN]; - int ret; /* If uuid does not match with current server uuid, set uuid as current server uuid. */ - if (strcmp(uuid, server_uuid) != 0) { - memcpy(uuid, server_uuid, ENCRYPTION_SERVER_UUID_LEN); + if (strcmp(s_uuid, server_uuid) != 0) { + strncpy(s_uuid, server_uuid, sizeof(s_uuid) - 1); } - memset(key_name, 0, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN); /* Generate new master key */ snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%s-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, - uuid, master_key_id + 1); + "%s-%s-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, + s_uuid, s_master_key_id + 1); /* We call key ring API to generate master key here. */ - ret = my_key_generate(key_name, "AES", - NULL, ENCRYPTION_KEY_LEN); + int ret = my_key_generate( + key_name, "AES", nullptr, ENCRYPTION_KEY_LEN); /* We call key ring API to get master key here. */ - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast(master_key), &key_len); + ret = my_key_fetch( + key_name, &key_type, nullptr, + reinterpret_cast(master_key), &key_len); + + if (ret != 0 || *master_key == nullptr) { + + ib::error() + << "Encryption can't find master key," + << " please check the keyring plugin is loaded." + << " ret=" << ret; - if (ret || *master_key == NULL) { - ib::error() << "Encryption can't find master key" - << ", please check the keyring plugin is loaded."; - *master_key = NULL; + *master_key = nullptr; } else { - master_key_id++; + ++s_master_key_id; } - if (key_type) { + if (key_type != nullptr) { my_free(key_type); } #endif /* !UNIV_HOTBACKUP */ @@ -9059,159 +9079,177 @@ Encryption::create_master_key(byte** master_key) @param[in] srv_uuid uuid of server instance @param[in,out] master_key master key */ void -Encryption::get_master_key(ulint master_key_id, - char* srv_uuid, - byte** master_key) +Encryption::get_master_key( + ulint master_key_id, + char* srv_uuid, + byte** master_key) { #ifndef UNIV_HOTBACKUP - char* key_type = NULL; - size_t key_len; + size_t key_len = 0; + char* key_type = nullptr; char key_name[ENCRYPTION_MASTER_KEY_NAME_MAX_LEN]; - int ret; - memset(key_name, 0, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN); + if (srv_uuid != nullptr) { + + ut_ad(strlen(srv_uuid) > 0); - if (srv_uuid != NULL) { snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%s-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, - srv_uuid, master_key_id); + "%s-%s-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, + srv_uuid, master_key_id); } else { + /* For compitable with 5.7.11, we need to get master key with server id. */ - memset(key_name, 0, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN); + snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%lu-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, - server_id, master_key_id); + "%s-%lu-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, + server_id, master_key_id); } /* We call key ring API to get master key here. */ - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast(master_key), - &key_len); + int ret = my_key_fetch( + key_name, &key_type, nullptr, + reinterpret_cast(master_key), &key_len); - if (key_type) { + if (key_type != nullptr) { my_free(key_type); } - if (ret) { - *master_key = NULL; - ib::error() << "Encryption can't find master key" - << ", please check the keyring plugin is loaded."; + if (ret != 0) { + + *master_key = nullptr; + + ib::error() + << "Encryption can't find master key," + << " please check the keyring plugin is loaded."; } #ifdef UNIV_ENCRYPT_DEBUG - if (!ret && *master_key) { - fprintf(stderr, "Fetched master key:%lu ", master_key_id); - ut_print_buf(stderr, *master_key, key_len); - fprintf(stderr, "\n"); + if (ret == 0 && *master_key != nullptr) { + std::ostringstream msg; + + ut_print_buf(msg, *master_key, key_len); + + ib::info() + << "Fetched master key: " << master_key_id + << "{" << msg.str() << "}"; } -#endif /* DEBUG_TDE */ +#endif /* UNIV_ENCRYPT_DEBUG */ #endif /* !UNIV_HOTBACKUP */ } /** Current master key id */ -ulint Encryption::master_key_id = 0; +ulint Encryption::s_master_key_id = 0; /** Current uuid of server instance */ -char Encryption::uuid[ENCRYPTION_SERVER_UUID_LEN + 1] = {0}; +char Encryption::s_uuid[ENCRYPTION_SERVER_UUID_LEN + 1] = {0}; /** Get current master key and master key id @param[in,out] master_key_id master key id @param[in,out] master_key master key @param[in,out] version encryption information version */ void -Encryption::get_master_key(ulint* master_key_id, - byte** master_key, - Encryption::Version* version) +Encryption::get_master_key( + ulint* master_key_id, + byte** master_key, + Version* version) { #ifndef UNIV_HOTBACKUP - char* key_type = NULL; + int ret; size_t key_len; + char* key_type = nullptr; char key_name[ENCRYPTION_MASTER_KEY_NAME_MAX_LEN]; - int ret; - memset(key_name, 0, ENCRYPTION_KEY_LEN); *version = Encryption::ENCRYPTION_VERSION_2; - if (Encryption::master_key_id == 0) { + if (s_master_key_id == 0) { + /* If m_master_key is 0, means there's no encrypted tablespace, we need to generate the first master key, and store it to key ring. */ - memset(uuid, 0, ENCRYPTION_SERVER_UUID_LEN + 1); - memcpy(uuid, server_uuid, ENCRYPTION_SERVER_UUID_LEN); + strncpy(s_uuid, server_uuid, sizeof(s_uuid) - 1); - /* Prepare the server uuid. */ + /* Prepare the server s_uuid. */ snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%s-1", ENCRYPTION_MASTER_KEY_PRIFIX, - uuid); + "%s-%s-1", ENCRYPTION_MASTER_KEY_PRIFIX, s_uuid); /* We call key ring API to generate master key here. */ - ret = my_key_generate(key_name, "AES", - NULL, ENCRYPTION_KEY_LEN); + ret = my_key_generate( + key_name, "AES", nullptr, ENCRYPTION_KEY_LEN); /* We call key ring API to get master key here. */ - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast(master_key), - &key_len); + ret = my_key_fetch( + key_name, &key_type, nullptr, + reinterpret_cast(master_key), &key_len); - if (!ret && *master_key != NULL) { - Encryption::master_key_id++; - *master_key_id = Encryption::master_key_id; + if (ret == 0 && *master_key != nullptr) { + ++s_master_key_id; + *master_key_id = s_master_key_id; } #ifdef UNIV_ENCRYPT_DEBUG - if (!ret && *master_key) { - fprintf(stderr, "Generated new master key:"); - ut_print_buf(stderr, *master_key, key_len); - fprintf(stderr, "\n"); + if (ret == 0 && *master_key != nullptr) { + std::ostringstream msg; + + ut_print_buf(msg, *master_key, key_len); + + ib::info() + <<"Generated new master key: {" + << msg.str() << "}"; } -#endif +#endif /* UNIV_ENCRYPT_DEBUG */ } else { - *master_key_id = Encryption::master_key_id; + *master_key_id = s_master_key_id; snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%s-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, - uuid, *master_key_id); + "%s-%s-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, + s_uuid, *master_key_id); /* We call key ring API to get master key here. */ - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast(master_key), - &key_len); - - /* For compitable with 5.7.11, we need to try to get master key with - server id when get master key with server uuid failure. */ - if (ret || *master_key == NULL) { - if (key_type) { + ret = my_key_fetch( + key_name, &key_type, nullptr, + reinterpret_cast(master_key), &key_len); + + /* For compitability with 5.7.11, we need to try to get master + key with server id when get master key with server uuid + failure. */ + if (ret != 0 || *master_key == nullptr) { + + if (key_type != nullptr) { my_free(key_type); } - memset(key_name, 0, - ENCRYPTION_MASTER_KEY_NAME_MAX_LEN); - snprintf(key_name, ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, - "%s-%lu-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, - server_id, *master_key_id); + snprintf(key_name, + ENCRYPTION_MASTER_KEY_NAME_MAX_LEN, + "%s-%lu-" ULINTPF, ENCRYPTION_MASTER_KEY_PRIFIX, + server_id, *master_key_id); + + ret = my_key_fetch( + key_name, &key_type, nullptr, + reinterpret_cast(master_key), &key_len); - ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast(master_key), - &key_len); *version = Encryption::ENCRYPTION_VERSION_1; } #ifdef UNIV_ENCRYPT_DEBUG - if (!ret && *master_key) { - fprintf(stderr, "Fetched master key:%lu ", - *master_key_id); - ut_print_buf(stderr, *master_key, key_len); - fprintf(stderr, "\n"); + if (ret == 0 && *master_key != nullptr) { + std::ostringstream msg; + + ut_print_buf(msg, *master_key, key_len); + + ib::info() + << "Fetched master key: " << *master_key_id + << ": {" << msg.str() << "}"; } -#endif +#endif /* UNIV_ENCRYPT_DEBUG */ } - if (ret) { - *master_key = NULL; - ib::error() << "Encryption can't find master key, please check" - " the keyring plugin is loaded."; + if (ret != 0) { + *master_key = nullptr; + ib::error() + << "Encryption can't find master key, please check" + << " the keyring plugin is loaded."; } - if (key_type) { + if (key_type != nullptr) { my_free(key_type); } #endif /* !UNIV_HOTBACKUP */ @@ -9223,84 +9261,88 @@ Encryption::get_master_key(ulint* master_key_id, @param[in,out] encrypt_info encryption information @param[in] is_boot if it's for bootstrap @return true if success */ -bool Encryption::fill_encryption_info(byte* key, - byte* iv, - byte* encrypt_info, - bool is_boot) -{ - byte* ptr; - lint elen; - ulint master_key_id; - byte* master_key; - byte key_info[ENCRYPTION_KEY_LEN * 2]; - ulint crc; - Version version; -#ifdef UNIV_ENCRYPT_DEBUG - const byte* data; - ulint i; -#endif +bool +Encryption::fill_encryption_info( + byte* key, + byte* iv, + byte* encrypt_info, + bool is_boot) +{ + Version version; + byte* master_key; + ulint master_key_id; /* Get master key from key ring. For bootstrap, we use a default master key which master_key_id is 0. */ if (is_boot) { master_key_id = 0; + master_key = static_cast(ut_zalloc_nokey( ENCRYPTION_KEY_LEN)); - memcpy(master_key, ENCRYPTION_DEFAULT_MASTER_KEY, - strlen(ENCRYPTION_DEFAULT_MASTER_KEY)); + + ut_ad(ENCRYPTION_KEY_LEN + >= sizeof(ENCRYPTION_DEFAULT_MASTER_KEY)); + + strcpy(reinterpret_cast(master_key), + ENCRYPTION_DEFAULT_MASTER_KEY); + version = ENCRYPTION_VERSION_2; } else { + get_master_key(&master_key_id, &master_key, &version); - if (master_key == NULL) { + + if (master_key == nullptr) { return(false); } } - memset(encrypt_info, 0, ENCRYPTION_INFO_SIZE_V2); - memset(key_info, 0, ENCRYPTION_KEY_LEN * 2); - /* Use the new master key to encrypt the key. */ - ut_ad(encrypt_info != NULL); - ptr = encrypt_info; + ut_ad(encrypt_info != nullptr); + auto ptr = encrypt_info; if (version == ENCRYPTION_VERSION_1) { memcpy(ptr, ENCRYPTION_KEY_MAGIC_V1, ENCRYPTION_MAGIC_SIZE); } else { memcpy(ptr, ENCRYPTION_KEY_MAGIC_V2, ENCRYPTION_MAGIC_SIZE); } + ptr += ENCRYPTION_MAGIC_SIZE; mach_write_to_4(ptr, master_key_id); + + /** FIXME: This should be uint32_t on all platforms. */ ptr += sizeof(ulint); if (version == ENCRYPTION_VERSION_2) { - memcpy(ptr, uuid, ENCRYPTION_SERVER_UUID_LEN); - ptr += ENCRYPTION_SERVER_UUID_LEN; + strncpy(reinterpret_cast(ptr), s_uuid, sizeof(s_uuid)); + ptr += sizeof(s_uuid) - 1; } + byte key_info[ENCRYPTION_KEY_LEN * 2]; + memcpy(key_info, key, ENCRYPTION_KEY_LEN); memcpy(key_info + ENCRYPTION_KEY_LEN, iv, ENCRYPTION_KEY_LEN); /* Encrypt key and iv. */ - elen = my_aes_encrypt(key_info, - ENCRYPTION_KEY_LEN * 2, - ptr, - master_key, - ENCRYPTION_KEY_LEN, - my_aes_256_ecb, - NULL, false); + auto elen = my_aes_encrypt( + key_info, sizeof(key_info), ptr, master_key, + ENCRYPTION_KEY_LEN, my_aes_256_ecb, nullptr, false); if (elen == MY_AES_BAD_DATA) { my_free(master_key); return(false); } - ptr += ENCRYPTION_KEY_LEN * 2; + ptr += sizeof(key_info); /* Write checksum bytes. */ - crc = ut_crc32(key_info, ENCRYPTION_KEY_LEN * 2); + auto crc = ut_crc32(key_info, sizeof(key_info)); + mach_write_to_4(ptr, crc); + ptr += sizeof(uint32_t); + + memset(ptr, 0x0, ENCRYPTION_INFO_SIZE_V2 - (ptr - encrypt_info)); if (is_boot) { ut_free(master_key); @@ -9311,37 +9353,27 @@ bool Encryption::fill_encryption_info(byte* key, return(true); } -/** Decoding the encryption info -from the first page of a tablespace. +/** Decoding the encryption info from the first page of a tablespace. @param[in,out] key key @param[in,out] iv iv @param[in] encryption_info encrytion info. @return true if success */ bool -Encryption::decode_encryption_info(byte* key, - byte* iv, - byte* encryption_info) -{ - byte* ptr; - ulint m_key_id; - byte* master_key = NULL; - lint elen; - byte key_info[ENCRYPTION_KEY_LEN * 2]; - ulint crc1; - ulint crc2; - char srv_uuid[ENCRYPTION_SERVER_UUID_LEN + 1]; - Version version; -#ifdef UNIV_ENCRYPT_DEBUG - const byte* data; - ulint i; -#endif +Encryption::decode_encryption_info( + byte* key, + byte* iv, + byte* encryption_info) +{ + byte* ptr; + Version version; + byte* master_key = nullptr; + char srv_uuid[ENCRYPTION_SERVER_UUID_LEN + 1]; ptr = encryption_info; /* For compatibility with 5.7.11, we need to handle the encryption information which created in this old version. */ - if (memcmp(ptr, ENCRYPTION_KEY_MAGIC_V1, - ENCRYPTION_MAGIC_SIZE) == 0) { + if (memcmp(ptr, ENCRYPTION_KEY_MAGIC_V1, ENCRYPTION_MAGIC_SIZE) == 0) { version = ENCRYPTION_VERSION_1; } else { version = ENCRYPTION_VERSION_2; @@ -9349,85 +9381,108 @@ Encryption::decode_encryption_info(byte* key, /* Check magic. */ if (version == ENCRYPTION_VERSION_2 - && memcmp(ptr, ENCRYPTION_KEY_MAGIC_V2, ENCRYPTION_MAGIC_SIZE) != 0) { - /* We ignore report error for recovery, - since the encryption info maybe hasn't writen - into datafile when the table is newly created. */ - if (!recv_recovery_is_on()) { - return(false); - } else { - return(true); - } + && memcmp(ptr, + ENCRYPTION_KEY_MAGIC_V2, ENCRYPTION_MAGIC_SIZE) != 0) { + + /* We don't report an error during recovery, since the + encryption info maybe hasn't writen into datafile when + the table is newly created. */ + + return(recv_recovery_is_on()); } ptr += ENCRYPTION_MAGIC_SIZE; /* Get master key id. */ - m_key_id = mach_read_from_4(ptr); + auto key_id = mach_read_from_4(ptr); + + /* FIXME: This is a bug should be uint32_t for all platforms. */ ptr += sizeof(ulint); /* Get server uuid. */ if (version == ENCRYPTION_VERSION_2) { - memset(srv_uuid, 0, ENCRYPTION_SERVER_UUID_LEN + 1); - memcpy(srv_uuid, ptr, ENCRYPTION_SERVER_UUID_LEN); - ptr += ENCRYPTION_SERVER_UUID_LEN; + + constexpr size_t len = sizeof(srv_uuid) - 1; + + srv_uuid[len] = 0; + memcpy(srv_uuid, ptr, len); + + ptr += len; } /* Get master key by key id. */ - memset(key_info, 0, ENCRYPTION_KEY_LEN * 2); + if (version == ENCRYPTION_VERSION_1) { - get_master_key(m_key_id, NULL, &master_key); + + get_master_key(key_id, nullptr, &master_key); + + } else if (key_id == 0) { + + /* When key_id is 0, which means it's the + default master key for bootstrap. */ + master_key = static_cast(ut_zalloc_nokey( + ENCRYPTION_KEY_LEN)); + + ut_ad(ENCRYPTION_KEY_LEN + >= sizeof(ENCRYPTION_DEFAULT_MASTER_KEY)); + + memcpy(reinterpret_cast(master_key), + ENCRYPTION_DEFAULT_MASTER_KEY, + sizeof(ENCRYPTION_DEFAULT_MASTER_KEY)); + } else { - if (m_key_id == 0) { - /* When m_key_id is 0, which means it's the - default master key for bootstrap. */ - master_key = static_cast(ut_zalloc_nokey( - ENCRYPTION_KEY_LEN)); - memcpy(master_key, ENCRYPTION_DEFAULT_MASTER_KEY, - strlen(ENCRYPTION_DEFAULT_MASTER_KEY)); - } else { - get_master_key(m_key_id, srv_uuid, &master_key); - } + get_master_key(key_id, srv_uuid, &master_key); } - if (master_key == NULL) { + if (master_key == nullptr) { return(false); } #ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "%lu ", m_key_id); - for (data = (const byte*) master_key, i = 0; - i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); -#endif + { + std::ostringstream msg; - /* Decrypt tablespace key and iv. */ - elen = my_aes_decrypt( - ptr, - ENCRYPTION_KEY_LEN * 2, - key_info, - master_key, - ENCRYPTION_KEY_LEN, - my_aes_256_ecb, NULL, false); + ut_print_buf_hex(msg, master_key, ENCRYPTION_KEY_LEN); - if (elen == MY_AES_BAD_DATA) { - if (m_key_id == 0) { - ut_free(master_key); - } else { - my_free(master_key); + ib::info() + << "Key ID: " << key_id + << " hex: {" << msg.str() << "}"; + } +#endif /* UNIV_ENCRYPT_DEBUG */ + + byte key_info[ENCRYPTION_KEY_LEN * 2]; + + { + /* Decrypt tablespace key and iv. */ + + auto len = my_aes_decrypt( + ptr, sizeof(key_info), key_info, + master_key, ENCRYPTION_KEY_LEN, my_aes_256_ecb, + nullptr, false); + + if (len == MY_AES_BAD_DATA) { + if (key_id == 0) { + ut_free(master_key); + } else { + my_free(master_key); + } + return(false); } - return(NULL); } /* Check checksum bytes. */ - ptr += ENCRYPTION_KEY_LEN * 2; + ptr += sizeof(key_info); + + auto crc1 = mach_read_from_4(ptr); + auto crc2 = ut_crc32(key_info, sizeof(key_info)); - crc1 = mach_read_from_4(ptr); - crc2 = ut_crc32(key_info, ENCRYPTION_KEY_LEN * 2); if (crc1 != crc2) { - ib::error() << "Failed to decrypt encryption information," + + ib::error() + << "Failed to decrypt encryption information," << " please check whether key file has been changed!"; - if (m_key_id == 0) { + + if (key_id == 0) { ut_free(master_key); } else { my_free(master_key); @@ -9439,30 +9494,33 @@ Encryption::decode_encryption_info(byte* key, memcpy(key, key_info, ENCRYPTION_KEY_LEN); /* Get tablespace iv */ - memcpy(iv, key_info + ENCRYPTION_KEY_LEN, - ENCRYPTION_KEY_LEN); + memcpy(iv, key_info + ENCRYPTION_KEY_LEN, ENCRYPTION_KEY_LEN); #ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, " "); - for (data = (const byte*) key, - i = 0; i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); - fprintf(stderr, " "); - for (data = (const byte*) iv, - i = 0; i < ENCRYPTION_KEY_LEN; i++) - fprintf(stderr, "%02lx", (ulong)*data++); - fprintf(stderr, "\n"); -#endif + { + std::ostringstream msg; + + ut_print_buf_hex(msg, key, ENCRYPTION_KEY_LEN); + + ib::info() << "Key: {" << msg.str() << "}"; + } + { + std::ostringstream msg; - if (m_key_id == 0) { + ut_print_buf_hex(msg, iv, ENCRYPTION_KEY_LEN); + ib::info() << "IV: {" << msg.str() << "}"; + } +#endif /* UNIV_ENCRYPT_DEBUG */ + + if (key_id == 0) { ut_free(master_key); } else { my_free(master_key); } - if (master_key_id < m_key_id) { - master_key_id = m_key_id; - memcpy(uuid, srv_uuid, ENCRYPTION_SERVER_UUID_LEN); + if (s_master_key_id < key_id) { + s_master_key_id = key_id; + memcpy(s_uuid, srv_uuid, sizeof(s_uuid) - 1); } return(true); @@ -9508,11 +9566,18 @@ Encryption::encrypt_log_block( byte remain_buf[MY_AES_BLOCK_SIZE * 2]; #ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "Encrypting block %lu.\n", - log_block_get_hdr_no(src_ptr)); - ut_print_buf_hex(stderr, src_ptr, OS_FILE_LOG_BLOCK_SIZE); - fprintf(stderr, "\n"); -#endif + { + std::ostringstream msg; + + ut_print_buf_hex(msg, src_ptr, OS_FILE_LOG_BLOCK_SIZE); + + ib::info() + << "Encrypting block: " + << log_block_get_hdr_no(src_ptr) + << "{" << msg.str() << "}"; + } +#endif /* UNIV_ENCRYPT_DEBUG */ + /* This is data size which need to encrypt. */ data_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; main_len = (data_len / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; @@ -9528,11 +9593,9 @@ Encryption::encrypt_log_block( ut_error; case Encryption::AES: { - lint elen; - ut_ad(m_klen == ENCRYPTION_KEY_LEN); - elen = my_aes_encrypt( + auto elen = my_aes_encrypt( src_ptr + LOG_BLOCK_HDR_SIZE, static_cast(main_len), dst_ptr + LOG_BLOCK_HDR_SIZE, @@ -9617,7 +9680,7 @@ Encryption::encrypt_log_block( } ut_free(buf2); ut_free(check_buf); -#endif +#endif /* UNIV_ENCRYPT_DEBUG */ /* Set the encrypted flag. */ log_block_set_encrypt_bit(dst_ptr, true); @@ -9675,7 +9738,7 @@ Encryption::encrypt_log( } ut_free(buf2); ut_free(check_buf); -#endif +#endif /* UNIV_ENCRYPT_DEBUG */ return(dst); } @@ -9716,7 +9779,7 @@ Encryption::encrypt( space_id, page_no, src_len); ut_print_buf(stderr, m_key, 32); ut_print_buf(stderr, m_iv, 32); -#endif +#endif /* UNIV_ENCRYPT_DEBUG */ /* Shouldn't encrypte an already encrypted page. */ ut_ad(page_type != FIL_PAGE_ENCRYPTED @@ -9850,7 +9913,8 @@ Encryption::encrypt( ut_free(buf2); ut_free(check_buf); fprintf(stderr, "Encrypted page:%lu.%lu\n", space_id, page_no); -#endif +#endif /* UNIV_ENCRYPT_DEBUG */ + *dst_len = src_len; return(dst); @@ -9995,12 +10059,21 @@ Encryption::decrypt_log( /* Encrypt the log blocks one by one. */ while (ptr != src + src_len) { + #ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "Decrypting block %lu.\n", - log_block_get_hdr_no(ptr)); - ut_print_buf_hex(stderr, ptr, OS_FILE_LOG_BLOCK_SIZE); - fprintf(stderr, "\n"); -#endif + { + std::ostringstream msg; + + ut_print_buf_hex(msg, ptr, OS_FILE_LOG_BLOCK_SIZE); + + ib::info() + << "Decrypting block: " + << log_block_get_hdr_no(ptr) << std::endl + << "data={" << std::endl + << msg.str << std::endl << "}"; + } +#endif /* UNIV_ENCRYPT_DEBUG */ + /* If it's not an encrypted block, skip it. */ if (!is_encrypted_log(ptr)) { ptr += OS_FILE_LOG_BLOCK_SIZE; @@ -10068,15 +10141,26 @@ Encryption::decrypt( } #ifdef UNIV_ENCRYPT_DEBUG - ulint space_id = - mach_read_from_4(src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - ulint page_no = mach_read_from_4(src + FIL_PAGE_OFFSET); + { + auto space_id = + mach_read_from_4(src + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - fprintf(stderr, "Decrypting page:%lu.%lu len:%lu\n", - space_id, page_no, src_len); - ut_print_buf(stderr, m_key, 32); - ut_print_buf(stderr, m_iv, 32); -#endif + auto page_no = mach_read_from_4(src + FIL_PAGE_OFFSET); + + std::ostringstream msg; + + msg << "key={" + ut_print_buf(msg, m_key, 32); + msg << "}" << std::endl << "iv= {"; + ut_print_buf(msg, m_iv, 32); + msg << "}"; + + ib::info() + << "Decrypting page: " + << space_id << "." << page_no, << " len: " << src_len + << std::endl << msg.str(); + } +#endif /* UNIV_ENCRYPT_DEBUG */ original_type = static_cast( mach_read_from_2(src + FIL_PAGE_ORIGINAL_TYPE_V1)); @@ -10197,8 +10281,8 @@ Encryption::decrypt( } #ifdef UNIV_ENCRYPT_DEBUG - fprintf(stderr, "Decrypted page:%lu.%lu\n", space_id, page_no); -#endif + ib::info() << "Decrypted page: " << space_id << "." << page_no; +#endif /* UNIV_ENCRYPT_DEBUG */ DBUG_EXECUTE_IF("ib_crash_during_decrypt_page", DBUG_SUICIDE();); @@ -10207,40 +10291,42 @@ Encryption::decrypt( #ifndef UNIV_HOTBACKUP /** Check if keyring plugin loaded. */ -bool Encryption::check_keyring() +bool +Encryption::check_keyring() { - char* key_type = NULL; size_t key_len; - char key_name[ENCRYPTION_MASTER_KEY_NAME_MAX_LEN]; - int my_ret; bool ret = false; - char* master_key = NULL; + char* key_type = nullptr; + char* master_key = nullptr; + char key_name[ENCRYPTION_MASTER_KEY_NAME_MAX_LEN]; - memset(key_name, 0, ENCRYPTION_KEY_LEN); - ut_strcpy(key_name, ENCRYPTION_DEFAULT_MASTER_KEY); + key_name[sizeof(ENCRYPTION_DEFAULT_MASTER_KEY)] = 0; + + strncpy(key_name, ENCRYPTION_DEFAULT_MASTER_KEY, sizeof(key_name)); /* We call key ring API to generate master key here. */ - my_ret = my_key_generate(key_name, "AES", - NULL, ENCRYPTION_KEY_LEN); + int my_ret = my_key_generate( + key_name, "AES", NULL, ENCRYPTION_KEY_LEN); /* We call key ring API to get master key here. */ - my_ret = my_key_fetch(key_name, &key_type, NULL, - reinterpret_cast(&master_key), - &key_len); + my_ret = my_key_fetch( + key_name, &key_type, nullptr, + reinterpret_cast(&master_key), &key_len); - if (my_ret) { - ib::error() << "Check keyring plugin fail, please check" - " the keyring plugin is loaded."; + if (my_ret != 0) { + ib::error() + << "Check keyring plugin fail, please check the" + << " keyring plugin is loaded."; } else { - my_key_remove(key_name, NULL); + my_key_remove(key_name, nullptr); ret = true; } - if (key_type != NULL) { + if (key_type != nullptr) { my_free(key_type); } - if (master_key != NULL) { + if (master_key != nullptr) { my_free(master_key); } @@ -10248,23 +10334,6 @@ bool Encryption::check_keyring() } #endif /* !UNIV_HOTBACKUP */ -/** Normalizes a directory path for the current OS: -On Windows, we convert '/' to '\', else we convert '\' to '/'. -@param[in,out] str A null-terminated directory and file path */ -void -os_normalize_path( - char* str) -{ - if (str != NULL) { - for (; *str; str++) { - if (*str == OS_PATH_SEPARATOR_ALT) { - *str = OS_PATH_SEPARATOR; - } - } - } -} - - /** Check if the path is a directory. The file/directory must exist. @param[in] path The path to check @return true if it is a directory */ @@ -10282,7 +10351,7 @@ Dir_Walker::is_directory(const Path& path) return(type == OS_FILE_TYPE_DIR); } - ut_ad(exists); + ut_ad(exists || type == OS_FILE_TYPE_FAILED); ut_ad(type != OS_FILE_TYPE_MISSING); return(false); diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index acd9c7de193f..ed03fc738611 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -346,12 +346,13 @@ page_create_low( { page_t* page; -#if PAGE_BTR_IBUF_FREE_LIST + FLST_BASE_NODE_SIZE > PAGE_DATA -# error "PAGE_BTR_IBUF_FREE_LIST + FLST_BASE_NODE_SIZE > PAGE_DATA" -#endif -#if PAGE_BTR_IBUF_FREE_LIST_NODE + FLST_NODE_SIZE > PAGE_DATA -# error "PAGE_BTR_IBUF_FREE_LIST_NODE + FLST_NODE_SIZE > PAGE_DATA" -#endif + static_assert( + PAGE_BTR_IBUF_FREE_LIST + FLST_BASE_NODE_SIZE <= PAGE_DATA, + "PAGE_BTR_IBUF_FREE_LIST + FLST_BASE_NODE_SIZE > PAGE_DATA"); + + static_assert( + PAGE_BTR_IBUF_FREE_LIST_NODE + FLST_NODE_SIZE <= PAGE_DATA, + "PAGE_BTR_IBUF_FREE_LIST_NODE + FLST_NODE_SIZE > PAGE_DATA"); buf_block_modify_clock_inc(block); diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 59302a318728..3fe474b7f0fc 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -243,9 +243,9 @@ page_zip_compress_write_log( /* Add the space occupied by BLOB pointers. */ trailer_size += page_zip->n_blobs * BTR_EXTERN_FIELD_REF_SIZE; ut_a(page_zip->m_end > PAGE_DATA); -#if FIL_PAGE_DATA > PAGE_DATA -# error "FIL_PAGE_DATA > PAGE_DATA" -#endif + + static_assert(FIL_PAGE_DATA <= PAGE_DATA, "FIL_PAGE_DATA > PAGE_DATA"); + ut_a(page_zip->m_end + trailer_size <= page_zip_get_size(page_zip)); log_ptr = mlog_write_initial_log_record_fast((page_t*) page, @@ -2799,9 +2799,9 @@ page_zip_write_header_log( ut_ad(offset < PAGE_DATA); ut_ad(offset + length < PAGE_DATA); -#if PAGE_DATA > 255 -# error "PAGE_DATA > 255" -#endif + + static_assert(PAGE_DATA <= 255, "PAGE_DATA > 255"); + ut_ad(length < 256); /* If no logging is requested, we may return now */ diff --git a/storage/innobase/row/row0import.cc b/storage/innobase/row/row0import.cc index 0a419acc0da2..eea667ae1a71 100644 --- a/storage/innobase/row/row0import.cc +++ b/storage/innobase/row/row0import.cc @@ -2323,7 +2323,8 @@ row_import_discard_changes( table->ibd_file_missing = TRUE; - fil_close_tablespace(trx, table->space); + err = fil_close_tablespace(trx, table->space); + ut_a(err == DB_SUCCESS || err == DB_TABLESPACE_NOT_FOUND); } /*****************************************************************//** @@ -3802,13 +3803,15 @@ row_import_for_mysql( dd_get_and_save_data_dir_path(table, table_def, true); if (DICT_TF_HAS_DATA_DIR(table->flags)) { - ut_a(table->data_dir_path); - filepath = fil_make_filepath( - table->data_dir_path, table->name.m_name, IBD, true); + ut_a(table->data_dir_path != nullptr); + + const auto dir = table->data_dir_path; + + filepath = Fil_path::make(dir, table->name.m_name, IBD, true); } else { - filepath = fil_make_filepath( - NULL, table->name.m_name, IBD, false); + filepath = Fil_path::make_ibd_from_table_name( + table->name.m_name); } DBUG_EXECUTE_IF( diff --git a/storage/innobase/row/row0merge.cc b/storage/innobase/row/row0merge.cc index a92f0e3a5c4a..e58bbe49972e 100644 --- a/storage/innobase/row/row0merge.cc +++ b/storage/innobase/row/row0merge.cc @@ -3640,23 +3640,24 @@ row_merge_file_create_low( performance schema */ struct PSI_file_locker* locker = NULL; PSI_file_locker_state state; + locker = PSI_FILE_CALL(get_thread_file_name_locker)( - &state, innodb_temp_file_key.m_value, - PSI_FILE_OPEN, - "Innodb Merge Temp File", &locker); + &state, innodb_temp_file_key.m_value, PSI_FILE_OPEN, + "Innodb Merge Temp File", &locker); + if (locker != NULL) { - PSI_FILE_CALL(start_file_open_wait)(locker, - __FILE__, - __LINE__); + PSI_FILE_CALL(start_file_open_wait)(locker, __FILE__, __LINE__); } -#endif +#endif /* UNIV_PFS_IO */ fd = innobase_mysql_tmpfile(path); #ifdef UNIV_PFS_IO if (locker != NULL) { + PSI_FILE_CALL(end_file_open_wait_and_bind_to_descriptor)( - locker, fd); - } -#endif + locker, fd); + + } +#endif /* UNIV_PFS_IO */ if (fd < 0) { ib::error() << "Cannot create temporary merge file"; @@ -3741,19 +3742,12 @@ row_make_new_pathname( dict_table_t* table, /*!< in: table to be renamed */ const char* new_name) /*!< in: new name */ { - char* new_path; - char* old_path; - ut_ad(dict_table_is_file_per_table(table)); - old_path = fil_space_get_first_path(table->space); - ut_a(old_path); - - new_path = os_file_make_new_pathname(old_path, new_name); - - ut_free(old_path); + auto old_path = fil_space_get_first_path(table->space); + auto new_path = Fil_path::make_new_ibd(old_path, new_name); - return(new_path); + return(mem_strdup(new_path.c_str())); } /** Create the index and load in to the dictionary. diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 88e9ee379359..e31530035803 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -4102,21 +4102,32 @@ row_drop_single_table_tablespace( dberr_t err = DB_SUCCESS; /* If the tablespace is not in the cache, just delete the file. */ - if (!fil_space_for_table_exists_in_mem( - space_id, tablename, true, false, NULL, 0)) { + if (!fil_space_exists_in_mem( + space_id, tablename, true, false, NULL, 0)) { /* Force a delete of any discarded or temporary files. */ - fil_delete_file(filepath); + if (fil_delete_file(filepath)) { - ib::info() << "Removed datafile " << filepath; + ib::info() << "Removed datafile " << filepath; - } else if (fil_delete_tablespace(space_id, BUF_REMOVE_FLUSH_NO_WRITE) - != DB_SUCCESS) { + } else { + ib::info() + << "Failed to delete the datafile '" + << filepath << "'!"; + } + + } else { - ib::error() << "We are not able to delete the tablespace " - << space_id << " file " << filepath << "!"; + err = fil_delete_tablespace( + space_id, BUF_REMOVE_FLUSH_NO_WRITE); - err = DB_ERROR; + if (err != DB_SUCCESS && err != DB_TABLESPACE_NOT_FOUND) { + + ib::error() + << "Failed to delete the datafile of" + << " tablespace " << space_id + << ", file '" << filepath << "'!"; + } } return(err); @@ -4521,14 +4532,13 @@ row_drop_table_for_mysql( /* Determine the tablespace filename before we drop dict_table_t. Free this memory before returning. */ if (DICT_TF_HAS_DATA_DIR(table->flags)) { - ut_a(table->data_dir_path); - filepath = fil_make_filepath( - table->data_dir_path, - table_name, IBD, true); + auto dir = dict_table_get_datadir(table); + + filepath = Fil_path::make(dir, table_name, IBD, true); + } else if (!shared_tablespace) { - filepath = fil_make_filepath( - NULL, table_name, IBD, false); + filepath = Fil_path::make_ibd_from_table_name(table_name); } /* Free the dict_table_t object. */ @@ -4544,11 +4554,16 @@ row_drop_table_for_mysql( /* Do not attempt to drop known-to-be-missing tablespaces, nor system or shared general tablespaces. */ - if (is_discarded || ibd_file_missing || is_temp || shared_tablespace + if (is_discarded + || ibd_file_missing + || is_temp + || shared_tablespace || fsp_is_system_or_temp_tablespace(space_id)) { + /* For encrypted table, if ibd file can not be decrypt, we also set ibd_file_missing. We still need to try to remove the ibd file for this. */ + if (is_discarded || !is_encrypted || !ibd_file_missing) { goto funct_exit; } diff --git a/storage/innobase/srv/srv0srv.cc b/storage/innobase/srv/srv0srv.cc index e95a4d76b341..f661135e2c58 100644 --- a/storage/innobase/srv/srv0srv.cc +++ b/storage/innobase/srv/srv0srv.cc @@ -2398,7 +2398,7 @@ srv_enable_undo_encryption_if_set() /* If the undo log space is using default key, rotate it. We need the server_uuid initialized, otherwise, the keyname will not contains server uuid. */ - if (Encryption::master_key_id != 0 + if (Encryption::s_master_key_id != 0 || srv_read_only_mode || strlen(server_uuid) == 0) { return; diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 3d2fba5752c9..ff3fedd98a88 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -109,12 +109,6 @@ Created 2/16/1996 Heikki Tuuri # include "ut0crc32.h" # include "ut0new.h" -#ifdef HAVE_LZO1X -#include - -extern bool srv_lzo_disabled; -#endif /* HAVE_LZO1X */ - /** fil_space_t::flags for hard-coded tablespaces */ extern ulint predefined_flags; @@ -276,7 +270,7 @@ srv_file_check_mode( } /** I/o-handler thread function. -@param[in] segment The AIO segment the thread will work on */ +@param[in] segment The AIO segment the thread will work on */ static void io_handler_thread(ulint segment) @@ -407,8 +401,9 @@ create_log_files( "innodb_redo_log", dict_sys_t::s_log_space_first_id, fsp_flags_set_page_size(0, univ_page_size), FIL_TYPE_LOG); - ut_a(fil_validate()); - ut_a(log_space != NULL); + + ut_ad(fil_validate()); + ut_a(log_space != nullptr); /* Once the redo log is set to be encrypted, initialize encryption information. */ @@ -432,18 +427,20 @@ create_log_files( logfile0 = fil_node_create( logfilename, static_cast(srv_log_file_size), log_space, false, false); - ut_a(logfile0); + + ut_a(logfile0 != nullptr); for (unsigned i = 1; i < srv_n_log_files; i++) { sprintf(logfilename + dirnamelen, "ib_logfile%u", i); - if (!fil_node_create(logfilename, - static_cast(srv_log_file_size), - log_space, false, false)) { + if (fil_node_create( + logfilename, + static_cast(srv_log_file_size), + log_space, false, false) == nullptr) { ib::error() - << "Cannot create file node for log file " + << "Cannot create file for log file " << logfilename; return(DB_ERROR); @@ -457,7 +454,6 @@ create_log_files( } fil_open_log_and_system_tablespace_files(); - fil_tablespace_open_create(); /* Create a log checkpoint. */ log_mutex_enter(); @@ -467,12 +463,12 @@ create_log_files( /* Write encryption information into the first log file header if redo log is set with encryption. */ - if (FSP_FLAGS_GET_ENCRYPTION(log_space->flags)) { - if (!log_write_encryption(log_space->encryption_key, - log_space->encryption_iv, - true)) { - return(DB_ERROR); - } + if (FSP_FLAGS_GET_ENCRYPTION(log_space->flags) + && !log_write_encryption( + log_space->encryption_key, log_space->encryption_iv, + true)) { + + return(DB_ERROR); } return(DB_SUCCESS); @@ -491,7 +487,7 @@ create_log_files_rename( { /* If innodb_flush_method=O_DSYNC, we need to explicitly flush the log buffers. */ - fil_flush(dict_sys_t::s_log_space_first_id); + fil_flush_file_redo(); /* Close the log files, so that we can rename the first one. */ fil_close_log_files(false); @@ -788,7 +784,7 @@ srv_undo_tablespace_fixup( } /** Open an undo tablespace. -@param[in] space_id tablespace ID +@param[in] space_id Undo tablespace ID @return DB_SUCCESS or error code */ static dberr_t @@ -798,21 +794,23 @@ srv_undo_tablespace_open(space_id_t space_id) bool success; ulint flags; bool atomic_write; + std::string scanned_name; dberr_t err = DB_ERROR; undo::Tablespace undo_space(space_id); char* undo_name = undo_space.space_name(); char* file_name = undo_space.file_name(); - fil_space_t* space; - fil_node_t* node = nullptr; /* See if the previous name in the file map is correct. */ - std::string recover_name = fil_system_open_fetch(space_id); - if (recover_name.length() != 0 - && !fil_paths_equal(file_name, recover_name.c_str())) { + scanned_name = fil_system_open_fetch(space_id); + + if (scanned_name.length() != 0 + && !Fil_path::equal(file_name, scanned_name)) { + /* Make sure that this space_id is used by the correctly named undo tablespace. */ - ib::error() << "Cannot create " << file_name - << " because " << recover_name.c_str() + ib::info() + << "Cannot create " << file_name + << " because " << scanned_name << " already uses Space ID=" << space_id << "! Did you change innodb_undo_directory?"; @@ -820,11 +818,14 @@ srv_undo_tablespace_open(space_id_t space_id) } /* Check if it was already opened during redo recovery. */ - space = fil_space_get(space_id); + fil_space_t* space = fil_space_get(space_id); + if (space != nullptr) { - node = UT_LIST_GET_FIRST(space->chain); - ut_ad(fil_paths_equal(recover_name.c_str(), node->name)); +#ifdef UNIV_DEBUG + const auto& file = space->files.front(); + ut_ad(Fil_path::equal(scanned_name, file.name)); +#endif /* UNIV_DEBUG */ fil_flush(space_id); @@ -874,22 +875,32 @@ srv_undo_tablespace_open(space_id_t space_id) space = fil_space_create( undo_name, space_id, flags, FIL_TYPE_TABLESPACE); - ut_a(space); - ut_a(fil_validate()); + ut_a(space != nullptr); + ut_ad(fil_validate()); os_offset_t size = os_file_get_size(fh); ut_a(size != (os_offset_t)-1); page_no_t n_pages = static_cast( size / UNIV_PAGE_SIZE); - if (nullptr == fil_node_create(file_name, n_pages, space, - false, atomic_write)) { + if (fil_node_create( + file_name, n_pages, space, false, atomic_write) + == nullptr) { + os_file_close(fh); - ib::error() << "Error creating file node for " << undo_name; + + ib::error() + << "Error creating file for " + << undo_name; + return(DB_ERROR); } + } else { - node->atomic_write = atomic_write; + + auto& file = space->files.front(); + + file.atomic_write = atomic_write; } /* Read the encryption metadata in this undo tablespace. @@ -911,7 +922,8 @@ srv_undo_tablespace_open(space_id_t space_id) But if it is under construction, we cannot open it until the header page has been written. */ if (!undo::is_under_construction(space_id)) { - ut_a(fil_space_open(space_id)); + bool success = fil_space_open(space_id); + ut_a(success); } return(DB_SUCCESS); @@ -928,7 +940,7 @@ dberr_t srv_undo_tablespaces_open( ulong target_undo_spaces) { - dberr_t err; + dberr_t err; /* Build a list of existing undo tablespaces from the references in the TRX_SYS page. (not including the system tablespace) */ @@ -1123,8 +1135,10 @@ srv_undo_tablespaces_create( undo::spaces->add(space_id); + ++cur_undo_spaces; + /* Quit when we have enough. */ - if (++cur_undo_spaces >= target_undo_spaces) { + if (cur_undo_spaces >= target_undo_spaces) { break; } } @@ -1156,7 +1170,6 @@ srv_undo_tablespaces_construct(bool create_new_db) ut_a(!srv_read_only_mode); ut_a(!srv_force_recovery); - Space_Ids::const_iterator it; for (auto space_id : undo::s_under_construction) { /* Enable undo log encryption if it's ON. */ @@ -1180,7 +1193,7 @@ srv_undo_tablespaces_construct(bool create_new_db) mtr_start(&mtr); - mtr_x_lock(fil_space_get_latch(space_id, NULL), &mtr); + mtr_x_lock(fil_space_get_latch(space_id), &mtr); if (!fsp_header_init(space_id, SRV_UNDO_TABLESPACE_SIZE_IN_PAGES, @@ -1278,6 +1291,7 @@ srv_undo_tablespaces_upgrade() /** Downgrade undo tablespaces by deleting the new undo tablespaces which are not referenced by the TRX_SYS page. @return error code */ +static void srv_undo_tablespaces_downgrade() { @@ -1738,9 +1752,9 @@ srv_prepare_to_delete_redo_log_files( log_write_up_to(flushed_lsn, true); - /* If innodb_flush_method=O_DSYNC, - we need to explicitly flush the log buffers. */ - fil_flush(dict_sys_t::s_log_space_first_id); + /* If innodb_flush_method=O_DSYNC, we need to explicitly + flush the log buffers. */ + fil_flush_file_redo(); ut_ad(flushed_lsn == log_get_lsn()); @@ -1772,7 +1786,7 @@ srv_prepare_to_delete_redo_log_files( recovery "dir1;dir2; ... dirN" @return DB_SUCCESS or error code */ dberr_t -srv_start(bool create_new_db, const char* scan_directories) +srv_start(bool create_new_db, const std::string& scan_directories) { lsn_t flushed_lsn; page_no_t sum_of_data_file_sizes; @@ -1899,15 +1913,14 @@ srv_start(bool create_new_db, const char* scan_directories) ib::info() << (ut_crc32_cpu_enabled ? "Using" : "Not using") << " CPU crc32 instructions"; - if (!create_new_db - && scan_directories != nullptr - && strlen(scan_directories) > 0) { + os_create_block_cache(); - err = fil_scan_for_tablespaces(scan_directories); + fil_init(srv_max_n_open_files); - if (err != DB_SUCCESS) { - return(srv_init_abort(err)); - } + err = fil_scan_for_tablespaces(scan_directories); + + if (err != DB_SUCCESS) { + return(srv_init_abort(err)); } if (!srv_read_only_mode) { @@ -1919,12 +1932,12 @@ srv_start(bool create_new_db, const char* scan_directories) srv_monitor_file_name = static_cast( ut_malloc_nokey( - strlen(fil_path_to_mysql_datadir) + MySQL_datadir_path.len() + 20 + sizeof "/innodb_status.")); sprintf(srv_monitor_file_name, "%s/innodb_status." ULINTPF, - fil_path_to_mysql_datadir, + static_cast(MySQL_datadir_path), os_proc_get_number()); srv_monitor_file = fopen(srv_monitor_file_name, "w+"); @@ -1987,8 +2000,6 @@ srv_start(bool create_new_db, const char* scan_directories) return(srv_init_abort(DB_ERROR)); } - fil_init(srv_file_per_table ? 50000 : 5000, srv_max_n_open_files); - double size; char unit; @@ -2060,19 +2071,19 @@ srv_start(bool create_new_db, const char* scan_directories) if (t < start) { if (t == 0) { - os_thread_create( + os_thread_create( io_ibuf_thread_key, io_handler_thread, t); } else { ut_ad(t == 1); - os_thread_create( + os_thread_create( io_log_thread_key, io_handler_thread, t); } } else if (t >= start && t < (start + srv_n_read_io_threads)) { - os_thread_create( + os_thread_create( io_read_thread_key, io_handler_thread, t); @@ -2080,11 +2091,11 @@ srv_start(bool create_new_db, const char* scan_directories) && t < (start + srv_n_read_io_threads + srv_n_write_io_threads)) { - os_thread_create( + os_thread_create( io_write_thread_key, io_handler_thread, t); } else { - os_thread_create( + os_thread_create( io_handler_thread_key, io_handler_thread, t); } @@ -2264,8 +2275,8 @@ srv_start(bool create_new_db, const char* scan_directories) fsp_flags_set_page_size(0, univ_page_size), FIL_TYPE_LOG); - ut_a(fil_validate()); - ut_a(log_space); + ut_ad(fil_validate()); + ut_a(log_space != nullptr); /* srv_log_file_size is measured in pages; if page size is 16KB, then we have a limit of 64TB on 32 bit systems */ @@ -2274,10 +2285,11 @@ srv_start(bool create_new_db, const char* scan_directories) for (unsigned j = 0; j < i; j++) { sprintf(logfilename + dirnamelen, "ib_logfile%u", j); - if (!fil_node_create( + if (fil_node_create( logfilename, static_cast(srv_log_file_size), - log_space, false, false)) { + log_space, false, false) == nullptr) { + return(srv_init_abort(DB_ERROR)); } } @@ -2289,17 +2301,16 @@ srv_start(bool create_new_db, const char* scan_directories) /* Read the first log file header to get the encryption information if it exist. */ - if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { - if (!log_read_encryption()) { + if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO + && !log_read_encryption()) { + return(srv_init_abort(DB_ERROR)); - } } } files_checked: /* Open all log files and data files in the system - tablespace: we keep them open until database - shutdown */ + tablespace: we keep them open until database shutdown */ fil_open_log_and_system_tablespace_files(); @@ -2347,7 +2358,8 @@ srv_start(bool create_new_db, const char* scan_directories) flushed_lsn = log_get_lsn(); - fil_write_flushed_lsn(flushed_lsn); + err = fil_write_flushed_lsn(flushed_lsn); + ut_a(err == DB_SUCCESS); create_log_files_rename( logfilename, dirnamelen, flushed_lsn, logfile0); @@ -2421,15 +2433,15 @@ srv_start(bool create_new_db, const char* scan_directories) && fil_check_missing_tablespaces()) { ib::error() - << "Use --innodb-scan-directories to find the" - << " the tablespace files. If that fails then use" - << " --innodb-force-recovery=1 to ignore" + << "Use --innodb-directories to find the" + << " tablespace files. If that fails then use" + << " --innodb-force-recvovery=1 to ignore" << " this and to permanently lose all changes" << " to the missing tablespace(s)"; /* Set the abort flag to true. */ - void* ptr = recv_recovery_from_checkpoint_finish(true); - ut_a(ptr == nullptr); + auto p = recv_recovery_from_checkpoint_finish(true); + ut_a(p == nullptr); return(srv_init_abort(DB_ERROR)); } @@ -2525,7 +2537,8 @@ srv_start(bool create_new_db, const char* scan_directories) RECOVERY_CRASH(3); /* Stamp the LSN to the data files. */ - fil_write_flushed_lsn(flushed_lsn); + err = fil_write_flushed_lsn(flushed_lsn); + ut_a(err == DB_SUCCESS); RECOVERY_CRASH(4); @@ -2713,9 +2726,14 @@ srv_start(bool create_new_db, const char* scan_directories) server could crash in middle of key rotation. Some tablespace didn't complete key rotation. Here, we will resume the rotation. */ - if (!srv_read_only_mode && !create_new_db + if (!srv_read_only_mode + && !create_new_db && srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { - fil_encryption_rotate(); + + if (!fil_encryption_rotate()) { + + ib::info() << "fil_encryption_rotate() failed!"; + } } srv_is_being_started = false; @@ -2799,15 +2817,15 @@ srv_start(bool create_new_db, const char* scan_directories) /** Applier of dynamic metadata */ struct metadata_applier { - /** Default constructor */ - metadata_applier() {} - /** Visitor. - @param[in] table table to visit */ - void operator()(dict_table_t* table) const - { - ut_ad(dict_sys->dynamic_metadata != NULL); + /** Default constructor */ + metadata_applier() {} + /** Visitor. + @param[in] table table to visit */ + void operator()(dict_table_t* table) const + { + ut_ad(dict_sys->dynamic_metadata != NULL); ib_uint64_t autoinc = table->autoinc; - dict_table_load_dynamic_metadata(table); + dict_table_load_dynamic_metadata(table); /* For those tables which were not opened by ha_innobase::open() and not initialized by innobase_initialize_autoinc(), the next counter should be @@ -2815,7 +2833,7 @@ struct metadata_applier if (autoinc != table->autoinc && table->autoinc != ~0ULL) { ++table->autoinc; } - } + } }; /** Apply the dynamic metadata to all tables */ @@ -2823,15 +2841,15 @@ static void apply_dynamic_metadata() { - const metadata_applier applier; + const metadata_applier applier; - dict_sys->for_each_table(applier); + dict_sys->for_each_table(applier); - if (srv_dict_metadata != NULL) { - srv_dict_metadata->apply(); - UT_DELETE(srv_dict_metadata); - srv_dict_metadata = NULL; - } + if (srv_dict_metadata != NULL) { + srv_dict_metadata->apply(); + UT_DELETE(srv_dict_metadata); + srv_dict_metadata = NULL; + } } /** On a restart, initialize the remaining InnoDB subsystems so that @@ -2899,8 +2917,10 @@ srv_start_threads( return; } - if (!bootstrap && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO + if (!bootstrap + && srv_force_recovery < SRV_FORCE_NO_TRX_UNDO && trx_sys_need_rollback()) { + /* Rollback all recovered transactions that are not in committed nor in XA PREPARE state. */ trx_rollback_or_clean_is_active = true; @@ -2953,7 +2973,7 @@ srv_fts_close(void) for (table = UT_LIST_GET_FIRST(dict_sys->table_LRU); table; table = UT_LIST_GET_NEXT(table_LRU, table)) { - fts_t* fts = table->fts; + fts_t* fts = table->fts; if (fts != NULL) { fts_sync_table(table); @@ -2962,7 +2982,7 @@ srv_fts_close(void) for (table = UT_LIST_GET_FIRST(dict_sys->table_non_LRU); table; table = UT_LIST_GET_NEXT(table_LRU, table)) { - fts_t* fts = table->fts; + fts_t* fts = table->fts; if (fts != NULL) { fts_sync_table(table); @@ -3247,28 +3267,19 @@ srv_get_encryption_data_filename( char* filename, ulint max_len) { - ulint len; - char* path; - /* Make sure the data_dir_path is set. */ dd_get_and_save_data_dir_path(table, NULL, false); - if (DICT_TF_HAS_DATA_DIR(table->flags)) { - ut_a(table->data_dir_path); + std::string path = dict_table_get_datadir(table); - path = fil_make_filepath( - table->data_dir_path, table->name.m_name, CFP, true); - } else { - path = fil_make_filepath(NULL, table->name.m_name, CFP, false); - } + auto filepath = Fil_path::make(path, table->name.m_name, CFP, true); - ut_a(path); - len = ut_strlen(path); + size_t len = strlen(filepath); ut_a(max_len >= len); - strcpy(filename, path); + strcpy(filename, filepath); - ut_free(path); + ut_free(filepath); } /** Call exit(3) */ diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 29e1f1f2fa13..5ecb9317c69e 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -472,6 +472,7 @@ LatchDebug::LatchDebug() LEVEL_MAP_INSERT(SYNC_LOCK_FREE_HASH); LEVEL_MAP_INSERT(SYNC_MONITOR_MUTEX); LEVEL_MAP_INSERT(SYNC_ANY_LATCH); + LEVEL_MAP_INSERT(SYNC_FIL_SHARD); LEVEL_MAP_INSERT(SYNC_DOUBLEWRITE); LEVEL_MAP_INSERT(SYNC_BUF_FLUSH_LIST); LEVEL_MAP_INSERT(SYNC_BUF_FLUSH_STATE); @@ -807,6 +808,7 @@ LatchDebug::check_order( case SYNC_POOL_MANAGER: case SYNC_RECV_WRITER: case SYNC_PARSER: + case SYNC_DICT: basic_check(latches, level, level); break; @@ -846,12 +848,14 @@ LatchDebug::check_order( } break; + case SYNC_FIL_SHARD: case SYNC_BUF_FLUSH_LIST: case SYNC_BUF_LRU_LIST: case SYNC_BUF_FREE_LIST: case SYNC_BUF_ZIP_FREE: case SYNC_BUF_ZIP_HASH: case SYNC_BUF_FLUSH_STATE: + case SYNC_RSEG_ARRAY_HEADER: /* We can have multiple mutexes of this type therefore we can only check whether the greater than condition holds. */ @@ -900,10 +904,6 @@ LatchDebug::check_order( || basic_check(latches, level, SYNC_FSP)); break; - case SYNC_RSEG_ARRAY_HEADER: - ut_a(basic_check(latches, level, level - 1)); - break; - case SYNC_TRX_UNDO_PAGE: /* Purge is allowed to read in as many UNDO pages as it likes. @@ -1013,10 +1013,6 @@ LatchDebug::check_order( ut_a(find(latches, SYNC_PERSIST_AUTOINC) == NULL); break; - case SYNC_DICT: - basic_check(latches, level, SYNC_DICT); - break; - case SYNC_MUTEX: case SYNC_UNKNOWN: case SYNC_LEVEL_VARYING: @@ -1413,7 +1409,7 @@ sync_latch_meta_init() LATCH_ADD_MUTEX(PARSER, SYNC_PARSER, parser_mutex_key); - LATCH_ADD_MUTEX(FIL_SYSTEM, SYNC_ANY_LATCH, fil_system_mutex_key); + LATCH_ADD_MUTEX(FIL_SHARD, SYNC_FIL_SHARD, fil_system_mutex_key); LATCH_ADD_MUTEX(FLUSH_LIST, SYNC_BUF_FLUSH_LIST, flush_list_mutex_key); diff --git a/storage/innobase/trx/trx0purge.cc b/storage/innobase/trx/trx0purge.cc index 32c8d59d73b7..d0d6b2e12727 100644 --- a/storage/innobase/trx/trx0purge.cc +++ b/storage/innobase/trx/trx0purge.cc @@ -990,8 +990,8 @@ trx_purge_mark_undo_for_truncate( ut_a(undo::spaces->size() >= FSP_MIN_UNDO_TABLESPACES); /* Return immediately if - * truncate is disabled or - * an undo tablespace is currently marked for truncate. */ + 1. truncate is disabled or + 2. an undo tablespace is currently marked for truncate. */ if (!srv_undo_log_truncate || undo_trunc->is_marked()) { return; } @@ -1067,7 +1067,8 @@ trx_purge_cleanse_purge_queue( ++it2) { if ((*it2)->space_id - == undo_trunc->get_marked_space_id()) { + == undo_trunc->get_marked_space_id()) { + it->erase(it2); break; } diff --git a/storage/innobase/trx/trx0roll.cc b/storage/innobase/trx/trx0roll.cc index 5d1c0f91da1c..1221e971aafd 100644 --- a/storage/innobase/trx/trx0roll.cc +++ b/storage/innobase/trx/trx0roll.cc @@ -56,7 +56,7 @@ rollback */ static const ulint TRX_ROLL_TRUNC_THRESHOLD = 1; /** true if trx_rollback_or_clean_all_recovered() thread is active */ -bool trx_rollback_or_clean_is_active; +bool trx_rollback_or_clean_is_active; /** In crash recovery, the current trx to be rolled back; NULL otherwise */ static const trx_t* trx_roll_crash_recv_trx = NULL; @@ -819,11 +819,11 @@ void trx_recovery_rollback_thread() { #ifdef UNIV_PFS_THREAD - THD* thd = create_thd(false, true, true, - trx_recovery_rollback_thread_key.m_value); + THD* thd = create_thd( + false, true, true, trx_recovery_rollback_thread_key.m_value); #else THD* thd = create_thd(false, true, true, 0); -#endif +#endif /* UNIV_PFS_THREAD */ my_thread_init(); @@ -831,7 +831,7 @@ trx_recovery_rollback_thread() trx_rollback_or_clean_recovered(TRUE); - trx_rollback_or_clean_is_active = false; + trx_rollback_or_clean_is_active = false; destroy_thd(thd); diff --git a/storage/innobase/trx/trx0rseg.cc b/storage/innobase/trx/trx0rseg.cc index b2fad0ba7304..0fb7e799f723 100644 --- a/storage/innobase/trx/trx0rseg.cc +++ b/storage/innobase/trx/trx0rseg.cc @@ -61,7 +61,7 @@ trx_rseg_header_create( buf_block_t* block; ut_ad(mtr); - ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space_id, NULL), + ut_ad(mtr_memo_contains(mtr, fil_space_get_latch(space_id), MTR_MEMO_X_LOCK)); /* Allocate a new file segment for the rollback segment */ diff --git a/storage/innobase/trx/trx0undo.cc b/storage/innobase/trx/trx0undo.cc index a735bce744ce..f82059b26287 100644 --- a/storage/innobase/trx/trx0undo.cc +++ b/storage/innobase/trx/trx0undo.cc @@ -2024,7 +2024,7 @@ trx_undo_truncate_tablespace( in truncated tablespaces. */ mtr_start(&mtr); mtr_set_log_mode(&mtr, MTR_LOG_NO_REDO); - mtr_x_lock(fil_space_get_latch(space_id, NULL), &mtr); + mtr_x_lock(fil_space_get_latch(space_id), &mtr); for (auto rseg : *undo_trunc->rsegs()) { trx_rsegf_t* rseg_header;