Merge 10.1 into 10.2

Disable the test encryption.innodb_encryption-page-compression
because the wait_condition would seem to time out deterministically.
MDEV-14814 has to be addressed in 10.2 separately.

Datafile::validate_first_page(): Do not invoke
page_size_t::page_size_t(flags) before validating the tablespace flags.
This avoids a crash in MDEV-15333 innodb.restart test case.
FIXME: Reduce the number of error messages. The first one is enough.

mysql-test/suite/encryption/disabled.def

@@ -13,3 +13,4 @@
 innodb_scrub            : MDEV-8139 scrubbing does not work reliably
 innodb_scrub_background : MDEV-8139 scrubbing does not work reliably
 innodb-redo-badkey : MDEV-13893 / MDEV-12699 Improve crash recovery of corrupted data pages
+innodb_encryption-page-compression : MDEV-14814 wait condition timeout

mysql-test/suite/encryption/r/innodb_encryption-page-compression.result

@@ -137,15 +137,9 @@ count(*)
 SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
 variable_value > 0
 1
-SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted';
-variable_value >= 0
-1
 SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed';
 variable_value > 0
 1
-SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
-variable_value >= 0
-1
 SET GLOBAL innodb_encryption_threads = 4;
 SET GLOBAL innodb_encrypt_tables = off;
 update innodb_page_compressed1 set c1 = c1 + 1;
@@ -157,17 +151,17 @@ update innodb_page_compressed6 set c1 = c1 + 1;
 update innodb_page_compressed7 set c1 = c1 + 1;
 update innodb_page_compressed8 set c1 = c1 + 1;
 update innodb_page_compressed9 set c1 = c1 + 1;
-SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
-variable_value >= 0
+SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
+variable_value > 0
 1
 SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted';
 variable_value > 0
 1
 SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed';
 variable_value > 0
 1
-SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
-variable_value >= 0
+SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
+variable_value > 0
 1
 drop procedure innodb_insert_proc;
 drop table innodb_normal;

mysql-test/suite/encryption/r/innodb_encryption_filekeys.result

@@ -1,4 +1,3 @@
-call mtr.add_suppression("trying to do an operation on a dropped tablespace .*");
 SET GLOBAL innodb_encrypt_tables = OFF;
 SET GLOBAL innodb_encryption_threads = 4;
 CREATE TABLE t1 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB;
@@ -12,24 +11,21 @@ t1	CREATE TABLE `t1` (
 CREATE TABLE t2 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=YES;
 CREATE TABLE t3 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=NO;
 CREATE TABLE t4 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNODB ENCRYPTED=YES ENCRYPTION_KEY_ID=4;
-INSERT INTO t2 select * from t1;
-INSERT INTO t3 select * from t1;
-INSERT INTO t4 select * from t1;
 SET GLOBAL innodb_encrypt_tables = on;
 # Wait max 10 min for key encryption threads to encrypt required all spaces
 # Success!
 SELECT COUNT(1) FROM t1;
 COUNT(1)
-400
+10
 SELECT COUNT(1) FROM t2;
 COUNT(1)
-400
+10
 SELECT COUNT(1) FROM t3;
 COUNT(1)
-400
+10
 SELECT COUNT(1) FROM t4;
 COUNT(1)
-400
+10
 SET GLOBAL innodb_encrypt_tables = off;
 # Wait max 10 min for key encryption threads to decrypt all required spaces
 # Success!
@@ -48,18 +44,17 @@ INSERT INTO t5 select * from t1;
 # Success!
 SELECT COUNT(1) FROM t1;
 COUNT(1)
-400
+10
 SELECT COUNT(1) FROM t2;
 COUNT(1)
-400
+10
 SELECT COUNT(1) FROM t3;
 COUNT(1)
-400
+10
 SELECT COUNT(1) FROM t4;
 COUNT(1)
-400
+10
 SELECT COUNT(1) FROM t5;
 COUNT(1)
-400
+10
 drop table t1,t2,t3,t4, t5;
-set GLOBAL innodb_default_encryption_key_id=1;

mysql-test/suite/encryption/t/innodb_encryption-page-compression.test

@@ -1,9 +1,6 @@
 -- source include/have_innodb.inc
 -- source include/have_example_key_management_plugin.inc
-
-let $innodb_compression_algorithm_orig=`SELECT @@innodb_compression_algorithm`;
-let $innodb_encrypt_tables_orig = `SELECT @@innodb_encrypt_tables`;
-let $innodb_encryption_threads_orig = `SELECT @@innodb_encryption_threads`;
+-- source include/not_embedded.inc
 
 SET GLOBAL innodb_encryption_threads = 4;
 SET GLOBAL innodb_encrypt_tables = on;
@@ -78,11 +75,13 @@ select count(*) from innodb_page_compressed9 where c1 < 500000;
 
 let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED';
 --source include/wait_condition.inc
+let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED';
+--source include/wait_condition.inc
 
 SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
-SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted';
 SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed';
-SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
+
+--source include/restart_mysqld.inc
 
 SET GLOBAL innodb_encryption_threads = 4;
 SET GLOBAL innodb_encrypt_tables = off;
@@ -97,13 +96,15 @@ update innodb_page_compressed7 set c1 = c1 + 1;
 update innodb_page_compressed8 set c1 = c1 + 1;
 update innodb_page_compressed9 set c1 = c1 + 1;
 
-let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_DECRYPTED';
+let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_ENCRYPTED';
+--source include/wait_condition.inc
+let $wait_condition= select variable_value > 0 from information_schema.global_status where variable_name = 'INNODB_NUM_PAGES_PAGE_COMPRESSED';
 --source include/wait_condition.inc
 
-SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
+SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_encrypted';
 SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_decrypted';
 SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_compressed';
-SELECT variable_value >= 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
+SELECT variable_value > 0 FROM information_schema.global_status WHERE variable_name = 'innodb_num_pages_page_decompressed';
 
 drop procedure innodb_insert_proc;
 drop table innodb_normal;
@@ -116,10 +117,3 @@ drop table innodb_page_compressed6;
 drop table innodb_page_compressed7;
 drop table innodb_page_compressed8;
 drop table innodb_page_compressed9;
-
-# reset system
---disable_query_log
-EVAL SET GLOBAL innodb_compression_algorithm = $innodb_compression_algorithm_orig;
-EVAL SET GLOBAL innodb_encrypt_tables = $innodb_encrypt_tables_orig;
-EVAL SET GLOBAL innodb_encryption_threads = $innodb_encryption_threads_orig;
---enable_query_log

mysql-test/suite/encryption/t/innodb_encryption_filekeys.test

@@ -1,12 +1,11 @@
 -- source include/have_innodb.inc
 -- source include/have_file_key_management_plugin.inc
-# embedded does not support restart
--- source include/not_embedded.inc
-
-call mtr.add_suppression("trying to do an operation on a dropped tablespace .*");
 
+--disable_query_log
 let $encrypt_tables = `SELECT @@innodb_encrypt_tables`;
 let $threads = `SELECT @@innodb_encryption_threads`;
+let $key_id = `SELECT @@innodb_default_encryption_key_id`;
+--enable_query_log
 
 SET GLOBAL innodb_encrypt_tables = OFF;
 SET GLOBAL innodb_encryption_threads = 4;
@@ -19,21 +18,20 @@ CREATE TABLE t4 (pk INT PRIMARY KEY AUTO_INCREMENT, c VARCHAR(256)) ENGINE=INNOD
 
 --disable_warnings
 --disable_query_log
-set autocommit=0;
-let $i = 400;
+begin;
+let $i = 10;
 while ($i)
 {
 INSERT INTO t1 values(NULL, substring(MD5(RAND()), -128));
 dec $i;
 }
-commit;
-set autocommit=1;
---enable_warnings
---enable_query_log
 
 INSERT INTO t2 select * from t1;
 INSERT INTO t3 select * from t1;
 INSERT INTO t4 select * from t1;
+commit;
+--enable_warnings
+--enable_query_log
 
 SET GLOBAL innodb_encrypt_tables = on;
 
@@ -54,7 +52,7 @@ while ($cnt)
 }
 if (!$success)
 {
-    SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
+    SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
     SHOW STATUS LIKE 'innodb_encryption%';
     -- die Timeout waiting for encryption threads
 }
@@ -84,7 +82,7 @@ while ($cnt)
 }
 if (!$success)
 {
-    SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
+    SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
     SHOW STATUS LIKE 'innodb_encryption%';
     -- die Timeout waiting for encryption threads
 }
@@ -113,7 +111,7 @@ while ($cnt)
 }
 if (!$success)
 {
-    SELECT * FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
+    SELECT NAME,ENCRYPTION_SHCEME,MIN_KEY_VERSION, ROTATING_OR_FLUSHING FROM INFORMATION_SCHEMA.INNODB_TABLESPACES_ENCRYPTION;
     SHOW STATUS LIKE 'innodb_encryption%';
     -- die Timeout waiting for encryption threads
 }
@@ -125,12 +123,11 @@ SELECT COUNT(1) FROM t3;
 SELECT COUNT(1) FROM t4;
 SELECT COUNT(1) FROM t5;
 
+drop table t1,t2,t3,t4, t5;
+
 # reset system
 --disable_query_log
 EVAL SET GLOBAL innodb_encrypt_tables = $encrypt_tables;
 EVAL SET GLOBAL innodb_encryption_threads = $threads;
+EVAL set GLOBAL innodb_default_encryption_key_id = $key_id;
 --enable_query_log
-
-drop table t1,t2,t3,t4, t5;
-
-set GLOBAL innodb_default_encryption_key_id=1;

mysql-test/suite/innodb/r/restart.result

@@ -0,0 +1,29 @@
+#
+# MDEV-15333 MariaDB (still) slow start
+#
+# FIXME: Unlike MySQL, maybe MariaDB should not read the .ibd files
+# of tables with .isl file or DATA DIRECTORY attribute.
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Invalid flags 0x7a207879 in .*td\\.ibd");
+# FIXME: This is much more noisy than MariaDB 10.1!
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot read first page in datafile: .*td\\.ibd, Space ID:2048948345, Flags: 2048948345");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Operating system error number .* in a file operation\\.");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: The error means the system cannot find the path specified\\.");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them\\.");
+call mtr.add_suppression("\\[Warning\\] InnoDB: Ignoring tablespace for `test`\\.`td` because it could not be opened\\.");
+CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT
+PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9;
+CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC
+STATS_PERSISTENT=0 DATA DIRECTORY='MYSQL_TMP_DIR';
+SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
+WHERE engine = 'innodb'
+AND support IN ('YES', 'DEFAULT', 'ENABLED');
+COUNT(*)
+1
+SELECT * FROM tr;
+a
+SELECT * FROM tc;
+a
+SELECT * FROM td;
+a
+DROP TABLE tr,tc,td;

mysql-test/suite/innodb/t/restart.test

@@ -0,0 +1,80 @@
+--source include/innodb_page_size.inc
+--source include/not_embedded.inc
+
+let datadir= `select @@datadir`;
+let page_size= `select @@innodb_page_size`;
+
+--echo #
+--echo # MDEV-15333 MariaDB (still) slow start
+--echo #
+
+# Ensure that on normal startup, no data files are read.
+# Note: just like in MySQL, all .ibd files will still be
+# opened at least once.
+
+--echo # FIXME: Unlike MySQL, maybe MariaDB should not read the .ibd files
+--echo # of tables with .isl file or DATA DIRECTORY attribute.
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Invalid flags 0x7a207879 in .*td\\.ibd");
+--echo # FIXME: This is much more noisy than MariaDB 10.1!
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Cannot read first page in datafile: .*td\\.ibd, Space ID:2048948345, Flags: 2048948345");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: Operating system error number .* in a file operation\\.");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: The error means the system cannot find the path specified\\.");
+call mtr.add_suppression("\\[ERROR\\] InnoDB: If you are installing InnoDB, remember that you must create directories yourself, InnoDB does not create them\\.");
+call mtr.add_suppression("\\[Warning\\] InnoDB: Ignoring tablespace for `test`\\.`td` because it could not be opened\\.");
+
+CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT
+PAGE_COMPRESSED=1 PAGE_COMPRESSION_LEVEL=9;
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
+eval CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC
+STATS_PERSISTENT=0 DATA DIRECTORY='$MYSQL_TMP_DIR';
+
+--source include/shutdown_mysqld.inc
+
+--move_file $datadir/test/tr.ibd $datadir/test/tr0.ibd
+--move_file $datadir/test/tc.ibd $datadir/test/tc0.ibd
+--move_file $MYSQL_TMP_DIR/test/td.ibd $datadir/test/td0.ibd
+# TODO: test that MariaDB does not even attempt to open the files
+#--mkdir $datadir/test/tr.ibd
+#--mkdir $datadir/test/tc.ibd
+#--mkdir $MYSQL_TMP_DIR/test/td.ibd
+
+perl;
+die unless open OUT, ">", "$ENV{datadir}/test/tr.ibd";
+print OUT "foo " x $ENV{page_size};
+close OUT or die;
+die unless open OUT, ">", "$ENV{datadir}/test/tc.ibd";
+print OUT "bar " x $ENV{page_size};
+close OUT or die;
+die unless open OUT, ">", "$ENV{MYSQL_TMP_DIR}/test/td.ibd";
+print OUT "xyz " x $ENV{page_size};
+close OUT or die;
+EOF
+
+--let $restart_parameters= --skip-innodb-buffer-pool-load-at-startup
+--source include/start_mysqld.inc
+--let $restart_parameters=
+
+SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES
+WHERE engine = 'innodb'
+AND support IN ('YES', 'DEFAULT', 'ENABLED');
+
+--source include/shutdown_mysqld.inc
+
+# TODO: test that MariaDB does not even attempt to open the files
+#--rmdir $datadir/test/tr.ibd
+#--rmdir $datadir/test/tc.ibd
+#--rmdir $MYSQL_TMP_DIR/test/td.ibd
+--remove_file $datadir/test/tr.ibd
+--remove_file $datadir/test/tc.ibd
+--remove_file $MYSQL_TMP_DIR/test/td.ibd
+
+--move_file $datadir/test/tr0.ibd $datadir/test/tr.ibd
+--move_file $datadir/test/tc0.ibd $datadir/test/tc.ibd
+--move_file $datadir/test/td0.ibd $MYSQL_TMP_DIR/test/td.ibd
+
+--source include/start_mysqld.inc
+SELECT * FROM tr;
+SELECT * FROM tc;
+SELECT * FROM td;
+DROP TABLE tr,tc,td;

storage/innobase/fil/fil0fil.cc

@@ -4215,7 +4215,8 @@ fil_ibd_open(
 			err = DB_ERROR;
 		}
 
-		if (purpose != FIL_TYPE_IMPORT && !srv_read_only_mode) {
+		if (err == DB_SUCCESS && validate
+		    && purpose != FIL_TYPE_IMPORT && !srv_read_only_mode) {
 			df_remote.close();
 			df_dict.close();
 			df_default.close();