WL#8619 - InnoDB: Provide offline database portability without ISL files

Summary of changes.

1. Introduce --innodb-directories := "dir1;...;dirN". This is renamed
   from --innodb-scan-directories which was introduced in WL#9499.
   Currently this is not dynamic.
2. Revert to 5.6 behaviour where we scan directories and find .ibd files.
   If you move tablespaces to a new location, add the path to
   innodb-data-directories so that InnoDB can "discover" the files on
   startup.
3. Tablespaces can only be created under known directories. This
   eliminates the need for .isl files.
4. Upgrade redo log format, not backward compatible
5. Change redo log type IDs
6. Code cleanup
7. Shard the fil system data structures for performance
8. Remove WL#9499 files that were used to track open tablespaces.

RB#16842 Approved by: Jimmy and Kevin.

mysql-test/r/mysqltest.result

@@ -1228,6 +1228,7 @@ ERROR HY000: MySQL server has gone away
 #
 # Bug#23743035: ADD A COPY_FILES_WILDCARD COMMAND
 #
+README_tablespace_portable_linux.txt
 bug20683959loaddata.txt
 bug30435_10k_items.txt
 bug30435_5k.txt
@@ -1252,6 +1253,7 @@ numbers.txt
 wl5766_data.txt
 words.dat
 words2.dat
+README_tablespace_portable_linux.txt
 bug20683959loaddata.txt
 bug30435_10k_items.txt
 bug30435_5k.txt

mysql-test/r/partition_innodb.result

@@ -1114,6 +1114,7 @@ t1	CREATE TABLE `t1` (
 t1#p#p1.ibd
 t1#p#p0.ibd
 DROP TABLE t1;
+# restart: --innodb-directories=MYSQL_TMP_DIR
 CREATE TABLESPACE ts1 ADD DATAFILE 'MYSQL_TMP_DIR/ts1.ibd' ENGINE=Innodb;
 CREATE TABLE `t1` (
 `f1` INT(10) UNSIGNED NOT NULL AUTO_INCREMENT,

mysql-test/r/tablespace.result

@@ -577,6 +577,7 @@ SET AUTOCOMMIT=ON;
 #
 # Test ALTER TABLESPACE RENAME with partition tables
 CREATE TABLESPACE ts1 ADD DATAFILE 'ts1.ibd' Engine=InnoDB;
+# restart: --innodb-directories=MYSQL_TMP_DIR
 CREATE TABLE tab (a INT, b INT) ENGINE = InnoDB ROW_FORMAT=DYNAMIC
 PARTITION BY RANGE(a) SUBPARTITION BY KEY(b) (
 PARTITION p1 VALUES LESS THAN (100) TABLESPACE=ts1,

mysql-test/suite/auth_sec/r/key_rotation_qa.result

@@ -1,22 +1,3 @@
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Encryption can't find master key, please check the keyring plugin is loaded.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Plugin keyring_file reported: 'keyring_file initialization failure.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Function 'keyring_file' already exists");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Couldn't load plugin named 'keyring_file' with soname 'keyring_file.so'.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Couldn't load plugin named 'keyring_file' with soname 'keyring_file.dll'.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Can't generate new master key, please check the keyring plugin is loaded.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Encryption information in datafile");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Operating system error number 2 in a file operation.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: The error means the system cannot find the path specified.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Could not find a valid tablespace file for");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Can't generate new master key for tablespace encryption, please check the keyring plugin is loaded.");
-call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Ignoring tablespace `test/.*` because it could not be opened.");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Failed to find tablespace for table");
-call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Can't read encryption key from file");
-call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot open table .* from the internal data dictionary of InnoDB though the .frm file for the table exists.*");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Table .* in the InnoDB data dictionary has tablespace id .*, but tablespace with that id or name does not exist");
-call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Please refer to .* for how to resolve the issue");
-call mtr.add_suppression("Could not flush keys to keyring's backup");
 ALTER INSTANCE ROTATE MYISAM MASTER KEY;
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'MYISAM MASTER KEY' at line 1
 ALTER INSTANCE ROTATE INNODB;

mysql-test/suite/auth_sec/t/key_rotation_qa.test

@@ -4,6 +4,7 @@
 # and key rotation testing.
 
 --source include/no_valgrind_without_big.inc
+--disable_query_log
 call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Encryption can't find master key, please check the keyring plugin is loaded.");
 call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Plugin keyring_file reported: 'keyring_file initialization failure.");
 call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] Function 'keyring_file' already exists");
@@ -23,6 +24,8 @@ call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Cannot open table .*
 call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: Table .* in the InnoDB data dictionary has tablespace id .*, but tablespace with that id or name does not exist");
 call mtr.add_suppression("\\[Warning\\] \\[[^]]*\\] InnoDB: Please refer to .* for how to resolve the issue");
 call mtr.add_suppression("Could not flush keys to keyring's backup");
+call mtr.add_suppression("\\[ERROR\\].*InnoDB: Tablespace [0-9]+, name 'test.t[0-9]+', unable to open file '.*test.t[0-9]+.ibd' - Data structure corruption");
+--enable_query_log
 
 # Invalid syntax
 --error ER_PARSE_ERROR

mysql-test/suite/innodb/r/alter_kill.result

@@ -63,7 +63,7 @@ XA END 'x';
 XA PREPARE 'x';
 # Kill the server
 # Attempt to start without an *.ibd file.
-Pattern "\[ERROR\] \[[^]]*\] InnoDB:.*tablespace \d+ .*not found:.*test.bug16735660.ibd.*" found
+Pattern "\[ERROR\].*InnoDB: Could not find any file associated with the tablespace ID:.*" found
 # restart
 SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 SELECT * FROM bug16735660;

mysql-test/suite/innodb/r/create_tablespace.result

@@ -3,6 +3,10 @@
 #
 SET DEFAULT_STORAGE_ENGINE=InnoDB;
 SET NAMES utf8;
+# restart:--innodb-directories=MYSQL_TMP_DIR
+SHOW VARIABLES LIKE 'innodb_directories';
+Variable_name	Value
+innodb_directories	MYSQL_TMP_DIR
 # Strict-mode has no effect on CREATE TABLESPACE.
 # It rejects all invalid input, as if strict mode is always ON.
 SHOW VARIABLES LIKE 'innodb_strict_mode';
@@ -43,10 +47,14 @@ CREATE TABLESPACE test/s_bad ADD DATAFILE 's_bad.ibd';
 ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '/s_bad ADD DATAFILE 's_bad.ibd'' at line 1
 CREATE TABLESPACE `test/s_bad` ADD DATAFILE 's_bad.ibd';
 ERROR 42000: InnoDB: A general tablespace name cannot contain '/'.
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'delete' returned OS error .*");
-call mtr.add_suppression("\\[ERROR\\] \\[[^]]*\\] InnoDB: File .* 'stat' returned OS error .*");
 CREATE TABLESPACE `s_too_long_file_name` ADD DATAFILE '../xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ibd';
-ERROR HY000: Failed to create TABLESPACE s_too_long_file_name
+ERROR HY000: Incorrect File Name '../xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ibd'.
+SHOW WARNINGS;
+Level	Code	Message
+Error	3121	Incorrect File Name '../xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx.ibd'.
+Error	3121	CREATE TABLESPACE data file must be in one of these directories './;MYSQL_TMP_DIR/'.
+Error	1528	Failed to create TABLESPACE s_too_long_file_name
+Error	1031	Table storage engine for 's_too_long_file_name' doesn't have this option
 CREATE TABLESPACE s_bad ADD DATAFILE 's_bad.ibd' FILE_BLOCK_SIZE=1k FILE_BLOCK_SIZE=2k;
 ERROR HY000: It is not allowed to specify FILE_BLOCK_SIZE more than once
 CREATE TABLESPACE s_bad ADD DATAFILE 's_bad.ibd' FILE_BLOCK_SIZE=3k;
@@ -274,10 +282,9 @@ Error	3121	Incorrect File Name 's_dir/s_subdir/.ibd'.
 Error	1528	Failed to create TABLESPACE s_bad
 Error	1031	Table storage engine for 's_bad' doesn't have this option
 CREATE TABLESPACE s_bad ADD DATAFILE 's_dir/s_bad.ibs';
-ERROR HY000: Incorrect File Name 's_dir/s_bad.ibs'.
+ERROR HY000: An IBD filepath must end with `.ibd`.
 SHOW WARNINGS;
 Level	Code	Message
-Error	3121	Incorrect File Name 's_dir/s_bad.ibs'.
 Error	3121	An IBD filepath must end with `.ibd`.
 Error	1528	Failed to create TABLESPACE s_bad
 Error	1031	Table storage engine for 's_bad' doesn't have this option
@@ -1642,7 +1649,7 @@ SET GLOBAL innodb_strict_mode=default;
 #
 # Check CREATE TABLESPACE is handled properly in read-only mode
 CREATE TABLESPACE s1 ADD DATAFILE 's1.ibd' ENGINE InnoDB;
-# restart: --innodb_read_only
+# restart: --innodb_read_only --innodb-directories=MYSQL_TMP_DIR
 CREATE TABLESPACE s2 ADD DATAFILE 's2.ibd' ENGINE InnoDB;
 ERROR HY000: Table 'tablespaces' is read only
 SHOW WARNINGS;
@@ -1656,7 +1663,7 @@ Level	Code	Message
 Error	1529	Failed to drop TABLESPACE s1
 Error	1874	InnoDB is in read only mode.
 # Restart in normal mode for cleanup.
-# restart
+# restart --innodb-directories=MYSQL_TMP_DIR
 DROP TABLESPACE s1;
 #
 # Bug#26199233 create like ignores innodb_file_per_table

mysql-test/suite/innodb/r/create_tablespace_8k.result

@@ -867,6 +867,8 @@ test/t_zip4k_to_file_per_table	test/t_zip4k_to_file_per_table	39	5	Compressed	40
 DROP TABLE `t_zip1k_to_file_per_table`;
 DROP TABLE `t_zip2k_to_file_per_table`;
 DROP TABLE `t_zip4k_to_file_per_table`;
+call mtr.add_suppression("\\[ERROR\\].*InnoDB: '.*s_.k.ibd' read failed! - attempted to read 4 bytes, read only 0 bytes at offset 8226");
+call mtr.add_suppression("\\[Warning\\].*InnoDB: Ignoring '.*s_.k.ibd' invalid tablespace ID in the header");
 #
 # Clean-up.
 #