Merge remote-tracking branch 'origin/10.1' into 10.1

Extra merge due to more extra commits during merge process

libmysqld/lib_sql.cc

@@ -434,8 +434,8 @@ static void emb_free_embedded_thd(MYSQL *mysql)
   thread_count--;
   thd->store_globals();
   thd->unlink();
-  delete thd;
   mysql_mutex_unlock(&LOCK_thread_count);
+  delete thd;
   my_pthread_setspecific_ptr(THR_THD,  0);
   mysql->thd=0;
 }

mysql-test/r/mysqld--help.result

@@ -457,6 +457,10 @@ The following options may be given as the first argument:
  --max-seeks-for-key=# 
  Limit assumed max number of seeks when looking up rows
  based on a key
+ --max-session-mem-used=# 
+ Amount of memory a single user session is allowed to
+ allocate. This limits the value of the session variable
+ MEM_USED
  --max-sort-length=# The number of bytes to use when sorting BLOB or TEXT
  values (only the first max_sort_length bytes of each
  value are used; the rest are ignored)
@@ -1266,6 +1270,7 @@ max-long-data-size 4194304
 max-prepared-stmt-count 16382
 max-relay-log-size 1073741824
 max-seeks-for-key 18446744073709551615
+max-session-mem-used 9223372036854775807
 max-sort-length 1024
 max-sp-recursion-depth 0
 max-statement-time 0

mysql-test/r/trigger_null-8605.result

@@ -354,3 +354,13 @@ show columns from t1;
 Field	Type	Null	Key	Default	Extra
 a	int(11)	NO	PRI	NULL	
 drop table t1;
+create table t1 (
+pk int primary key,
+i int,
+v1 int as (i) virtual,
+v2 int as (i) virtual
+);
+create trigger tr before update on t1 for each row set @a = 1;
+insert into t1 (pk, i) values (null, null);
+ERROR 23000: Column 'pk' cannot be null
+drop table t1;

mysql-test/suite/encryption/disabled.def

@@ -10,4 +10,5 @@
 #
 ##############################################################################
 
-innodb_scrub_background : MDEV-8139 background scrubbing does not work reliably
+innodb_scrub            : MDEV-8139 scrubbing does not work reliably
+innodb_scrub_background : MDEV-8139 scrubbing does not work reliably

mysql-test/suite/innodb/include/ibd_convert.pl

@@ -0,0 +1,25 @@
+#!/usr/bin/perl
+# Convert tablespace flags to the format understood by MariaDB 10.1.0..10.1.20,
+# with the assumption that the flags were correct.
+
+sub convert_to_mariadb_101
+{
+    my ($file, $page_size) = @_;
+    open(FILE, "+<", $file) or die "Unable to open $file\n";
+    sysread(FILE, $_, $page_size)==$page_size||die "Unable to read $file\n";
+    sysseek(FILE, 0, 0)||die "Unable to seek $file\n";
+
+    # FIL_PAGE_DATA + FSP_SPACE_FLAGS = 38 + 16 = 54 bytes from the start
+    my($flags) = unpack "x[54]N", $_;
+    my $badflags = ($flags & 0x3f);
+    my $compression_level=6;
+    $badflags |= 1<<6|$compression_level<<7 if ($flags & 1 << 16);
+    $badflags |= ($flags & 15 << 6) << 7; # PAGE_SSIZE
+
+    substr ($_, 54, 4) = pack("N", $badflags);
+    # Replace the innodb_checksum_algorithm=none checksum
+    substr ($_, 0, 4) = pack("N", 0xdeadbeef);
+    substr ($_, $page_size - 8, 4) = pack("N", 0xdeadbeef);
+    syswrite(FILE, $_, $page_size)==$page_size||die "Unable to write $file\n";
+    close(FILE);
+}

mysql-test/suite/innodb/r/101_compatibility.result

@@ -0,0 +1,48 @@
+#
+# MDEV-11623 MariaDB 10.1 fails to start datadir created with
+# MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K
+#
+call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of tablespace");
+SET GLOBAL innodb_file_per_table=1;
+SET GLOBAL innodb_file_format=Barracuda;
+CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT;
+CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+CREATE TABLE tz(a INT)ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1;
+CREATE TABLE tdd(a INT) ENGINE=InnoDB, DATA DIRECTORY='MYSQL_TMP_DIR';
+CREATE TABLE tp(a INT) ENGINE=InnoDB page_compressed=1;
+CREATE TABLE ti(a INT) ENGINE=InnoDB;
+FLUSH TABLES ti FOR EXPORT;
+backup: ti
+UNLOCK TABLES;
+ALTER TABLE ti DISCARD TABLESPACE;
+restore: ti .ibd and .cfg files
+ALTER TABLE ti IMPORT TABLESPACE;
+BEGIN;
+INSERT INTO tr VALUES(1);
+INSERT INTO tc VALUES(1);
+INSERT INTO td VALUES(1);
+INSERT INTO tz VALUES(1);
+INSERT INTO tdd VALUES(1);
+INSERT INTO tp VALUES(1);
+INSERT INTO ti VALUES(1);
+# Kill the server
+CHECK TABLE tr,tc,td,tz,tdd,tp,ti;
+Table	Op	Msg_type	Msg_text
+test.tr	check	status	OK
+test.tc	check	status	OK
+test.td	check	status	OK
+test.tz	check	status	OK
+test.tdd	check	status	OK
+test.tp	check	status	OK
+test.ti	check	status	OK
+CHECK TABLE tr,tc,td,tz,tdd,tp,ti;
+Table	Op	Msg_type	Msg_text
+test.tr	check	status	OK
+test.tc	check	status	OK
+test.td	check	status	OK
+test.tz	check	status	OK
+test.tdd	check	status	OK
+test.tp	check	status	OK
+test.ti	check	status	OK
+DROP TABLE tr,tc,td,tz,tdd,tp,ti;

mysql-test/suite/innodb/r/doublewrite.result

@@ -39,6 +39,8 @@ set global innodb_buf_flush_list_now = 1;
 # Kill the server
 # Make the first page (page_no=0) of the user tablespace
 # full of zeroes.
+#
+# MDEV-11623: Use old FSP_SPACE_FLAGS in the doublewrite buffer.
 check table t1;
 Table	Op	Msg_type	Msg_text
 test.t1	check	status	OK

mysql-test/suite/innodb/r/group_commit_crash.result

@@ -1,4 +1,3 @@
-call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed");
 CREATE TABLE t1(a CHAR(255),
 b CHAR(255),
 c CHAR(255),

mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result

@@ -1,4 +1,3 @@
-call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed");
 CREATE TABLE t1(a CHAR(255),
 b CHAR(255),
 c CHAR(255),

mysql-test/suite/innodb/r/innodb-alter-tempfile.result

@@ -4,10 +4,6 @@
 # Temporary tablename will be unique. This makes sure that future
 # in-place ALTERs of the same table will not be blocked due to
 # temporary tablename.
-call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed
-");
-call mtr.add_suppression("InnoDB: file read of space .* page .*");
-call mtr.add_suppression("InnoDB: Trying to recover it from the doublewrite buffer.");
 # Crash the server in ha_innobase::commit_inplace_alter_table()
 CREATE TABLE t1 (f1 INT NOT NULL, f2 INT NOT NULL) ENGINE=innodb;
 SET debug='d,innodb_alter_commit_crash_before_commit';

mysql-test/suite/innodb/r/innodb-wl5522-debug-zip.result

@@ -1,4 +1,5 @@
-call mtr.add_suppression("InnoDB: Page for tablespace .* ");
+call mtr.add_suppression("InnoDB: Page for tablespace ");
+call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=0x");
 FLUSH TABLES;
 SET GLOBAL innodb_file_per_table = 1;
 SELECT @@innodb_file_per_table;
@@ -565,7 +566,7 @@ ERROR HY000: Tablespace has been discarded for table 't1'
 restore: t1 .ibd and .cfg files
 SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure";
 ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE;
-ERROR HY000: Internal error: Cannot reset LSNs in table '"test_wl5522"."t1"' : Unsupported
+ERROR HY000: Internal error: Cannot reset LSNs in table '"test_wl5522"."t1"' : Data structure corruption
 SET SESSION debug_dbug="-d,fsp_flags_is_valid_failure";
 DROP TABLE test_wl5522.t1;
 unlink: t1.ibd

mysql-test/suite/innodb/r/innodb-wl5522-debug.result

@@ -1,4 +1,5 @@
 call mtr.add_suppression("InnoDB: Page for tablespace .* ");
+call mtr.add_suppression("InnoDB: Invalid FSP_SPACE_FLAGS=");
 FLUSH TABLES;
 SET GLOBAL innodb_file_per_table = 1;
 SELECT @@innodb_file_per_table;
@@ -912,7 +913,7 @@ ERROR HY000: Tablespace has been discarded for table 't1'
 restore: t1 .ibd and .cfg files
 SET SESSION debug_dbug="+d,fsp_flags_is_valid_failure";
 ALTER TABLE test_wl5522.t1 IMPORT TABLESPACE;
-ERROR HY000: Internal error: Cannot reset LSNs in table '"test_wl5522"."t1"' : Unsupported
+ERROR HY000: Internal error: Cannot reset LSNs in table '"test_wl5522"."t1"' : Data structure corruption
 SET SESSION debug_dbug="-d,fsp_flags_is_valid_failure";
 DROP TABLE test_wl5522.t1;
 unlink: t1.ibd

mysql-test/suite/innodb/r/innodb_bug14147491.result

@@ -1,6 +1,5 @@
 call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed.*");
 CALL mtr.add_suppression("InnoDB: Error: Unable to read tablespace .* page no .* into the buffer pool after 100 attempts");
-CALL mtr.add_suppression("InnoDB: Warning: database page corruption or a failed");
 CALL mtr.add_suppression("InnoDB: Database page corruption on disk or a failed");
 CALL mtr.add_suppression("InnoDB: Space .* file test/t1 read of page .*");
 CALL mtr.add_suppression("InnoDB: You may have to recover from a backup.");

mysql-test/suite/innodb/r/xa_recovery.result

@@ -1,11 +1,11 @@
-call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed");
 CREATE TABLE t1 (a INT) ENGINE=InnoDB;
 INSERT INTO t1 VALUES (1);
 XA START 'x';
 UPDATE t1 set a=2;
 XA END 'x';
 XA PREPARE 'x';
 call mtr.add_suppression("Found 1 prepared XA transactions");
+# Kill the server
 SELECT * FROM t1 LOCK IN SHARE MODE;
 SET TRANSACTION ISOLATION LEVEL READ UNCOMMITTED;
 SELECT * FROM t1;

mysql-test/suite/innodb/t/101_compatibility.test

@@ -0,0 +1,101 @@
+--source include/have_innodb.inc
+--source include/not_embedded.inc
+
+-- echo #
+-- echo # MDEV-11623 MariaDB 10.1 fails to start datadir created with
+-- echo # MariaDB 10.0/MySQL 5.6 using innodb-page-size!=16K
+-- echo #
+
+# This is actually testing the opposite: starting the fixed 10.1 with
+# buggy 10.1 files (by manually converting the flags in the files).
+
+call mtr.add_suppression("InnoDB: adjusting FSP_SPACE_FLAGS of tablespace");
+SET GLOBAL innodb_file_per_table=1;
+SET GLOBAL innodb_file_format=Barracuda;
+let INNODB_PAGE_SIZE=`select @@innodb_page_size`;
+let MYSQLD_DATADIR=`select @@datadir`;
+
+CREATE TABLE tr(a INT)ENGINE=InnoDB ROW_FORMAT=REDUNDANT;
+CREATE TABLE tc(a INT)ENGINE=InnoDB ROW_FORMAT=COMPACT;
+CREATE TABLE td(a INT)ENGINE=InnoDB ROW_FORMAT=DYNAMIC;
+--disable_warnings
+CREATE TABLE tz(a INT)ENGINE=InnoDB ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1;
+--enable_warnings
+
+--replace_result $MYSQL_TMP_DIR MYSQL_TMP_DIR
+EVAL CREATE TABLE tdd(a INT) ENGINE=InnoDB, DATA DIRECTORY='$MYSQL_TMP_DIR';
+
+CREATE TABLE tp(a INT) ENGINE=InnoDB page_compressed=1;
+CREATE TABLE ti(a INT) ENGINE=InnoDB;
+FLUSH TABLES ti FOR EXPORT;
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_backup_tablespaces("test", "ti");
+EOF
+UNLOCK TABLES;
+ALTER TABLE ti DISCARD TABLESPACE;
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/innodb-util.pl";
+ib_discard_tablespaces("test", "ti");
+ib_restore_tablespaces("test", "ti");
+do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl";
+my $ps = $ENV{INNODB_PAGE_SIZE};
+my $dd = $ENV{MYSQLD_DATADIR};
+convert_to_mariadb_101("$dd/test/ti.ibd", $ps);
+EOF
+
+ALTER TABLE ti IMPORT TABLESPACE;
+
+BEGIN;
+INSERT INTO tr VALUES(1);
+INSERT INTO tc VALUES(1);
+INSERT INTO td VALUES(1);
+INSERT INTO tz VALUES(1);
+INSERT INTO tdd VALUES(1);
+INSERT INTO tp VALUES(1);
+INSERT INTO ti VALUES(1);
+
+--source include/kill_mysqld.inc
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl";
+my $ps = $ENV{INNODB_PAGE_SIZE};
+my $dd = $ENV{MYSQLD_DATADIR};
+
+convert_to_mariadb_101("$dd/ibdata1", $ps);
+convert_to_mariadb_101("$dd/test/tr.ibd", $ps);
+convert_to_mariadb_101("$dd/test/tc.ibd", $ps);
+convert_to_mariadb_101("$dd/test/td.ibd", $ps);
+convert_to_mariadb_101("$dd/test/tz.ibd", 1024) if $ps<32768;
+convert_to_mariadb_101("$dd/test/tp.ibd", $ps);
+convert_to_mariadb_101("$dd/test/ti.ibd", $ps);
+convert_to_mariadb_101("$ENV{MYSQL_TMP_DIR}/test/tdd.ibd", $ps);
+EOF
+
+--source include/start_mysqld.inc
+CHECK TABLE tr,tc,td,tz,tdd,tp,ti;
+--source include/shutdown_mysqld.inc
+
+perl;
+do "$ENV{MTR_SUITE_DIR}/include/ibd_convert.pl";
+my $ps = $ENV{INNODB_PAGE_SIZE};
+my $dd = $ENV{MYSQLD_DATADIR};
+
+convert_to_mariadb_101("$dd/ibdata1", $ps);
+convert_to_mariadb_101("$dd/test/tr.ibd", $ps);
+convert_to_mariadb_101("$dd/test/tc.ibd", $ps);
+convert_to_mariadb_101("$dd/test/td.ibd", $ps);
+convert_to_mariadb_101("$dd/test/tz.ibd", 1024) if $ps<32768;
+convert_to_mariadb_101("$dd/test/tp.ibd", $ps);
+convert_to_mariadb_101("$dd/test/ti.ibd", $ps);
+convert_to_mariadb_101("$ENV{MYSQL_TMP_DIR}/test/tdd.ibd", $ps);
+EOF
+
+--let $restart_parameters=--innodb-read-only
+--source include/start_mysqld.inc
+CHECK TABLE tr,tc,td,tz,tdd,tp,ti;
+--source include/shutdown_mysqld.inc
+
+--let $restart_parameters=
+--source include/start_mysqld.inc
+DROP TABLE tr,tc,td,tz,tdd,tp,ti;

mysql-test/suite/innodb/t/doublewrite.test

@@ -17,7 +17,7 @@ call mtr.add_suppression("space header page consists of zero bytes.*test.t1");
 call mtr.add_suppression("checksum mismatch in tablespace.*test.t1");
 call mtr.add_suppression("Current page size .* !=  page size on page");
 call mtr.add_suppression("innodb-page-size mismatch in tablespace.*test.t1");
-call mtr.add_suppression("Database page corruption");
+call mtr.add_suppression("Trying to recover page.*from the doublewrite buffer");
 --enable_query_log
 
 let INNODB_PAGE_SIZE=`select @@innodb_page_size`;
@@ -65,14 +65,48 @@ set global innodb_buf_flush_list_now = 1;
 
 --echo # Make the first page (page_no=0) of the user tablespace
 --echo # full of zeroes.
+--echo #
+--echo # MDEV-11623: Use old FSP_SPACE_FLAGS in the doublewrite buffer.
+
 perl;
 use IO::Handle;
 my $fname= "$ENV{'MYSQLD_DATADIR'}test/t1.ibd";
+my $page_size = $ENV{INNODB_PAGE_SIZE};
+my $page;
 open(FILE, "+<", $fname) or die;
-FILE->autoflush(1);
-binmode FILE;
-print FILE chr(0) x ($ENV{'INNODB_PAGE_SIZE'});
+sysread(FILE, $page, $page_size)==$page_size||die "Unable to read $name\n";
+sysseek(FILE, 0, 0)||die "Unable to seek $fname\n";
+die unless syswrite(FILE, chr(0) x $page_size, $page_size) == $page_size;
 close FILE;
+
+open(FILE, "+<", "$ENV{MYSQLD_DATADIR}ibdata1")||die "cannot open ibdata1\n";
+sysseek(FILE, 6 * $page_size - 190, 0)||die "Unable to seek ibdata1\n";
+sysread(FILE, $_, 12) == 12||die "Unable to read TRX_SYS\n";
+my($magic,$d1,$d2)=unpack "NNN", $_;
+die "magic=$magic, $d1, $d2\n" unless $magic == 536853855 && $d2 >= $d1 + 64;
+sysseek(FILE, $d1 * $page_size, 0)||die "Unable to seek ibdata1\n";
+# Find the page in the doublewrite buffer
+for (my $d = $d1; $d < $d2 + 64; $d++)
+{
+    sysread(FILE, $_, $page_size)==$page_size||die "Cannot read doublewrite\n";
+    next unless $_ eq $page;
+    sysseek(FILE, $d * $page_size, 0)||die "Unable to seek ibdata1\n";
+    # Write buggy MariaDB 10.1.x FSP_SPACE_FLAGS to the doublewrite buffer
+    my($flags) = unpack "x[54]N", $_;
+    my $badflags = ($flags & 0x3f);
+    my $compression_level=6;
+    $badflags |= 1<<6|$compression_level<<7 if ($flags & 1 << 16);
+    $badflags |= ($flags & 15 << 6) << 7; # PAGE_SSIZE
+
+    substr ($_, 54, 4) = pack("N", $badflags);
+    # Replace the innodb_checksum_algorithm=none checksum
+    substr ($_, 0, 4) = pack("N", 0xdeadbeef);
+    substr ($_, $page_size - 8, 4) = pack("N", 0xdeadbeef);
+    syswrite(FILE, $_, $page_size)==$page_size||die;
+    close(FILE);
+    exit 0;
+}
+die "Did not find the page in the doublewrite buffer ($d1,$d2)\n";
 EOF
 
 --source include/start_mysqld.inc

mysql-test/suite/innodb/t/group_commit_crash.test

@@ -9,8 +9,6 @@
 --source include/have_debug.inc
 --source include/have_log_bin.inc
 
-call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed");
-
 let $file_format_max=`SELECT @@innodb_file_format_max`;
 CREATE TABLE t1(a CHAR(255),
                 b CHAR(255),

mysql-test/suite/innodb/t/group_commit_crash_no_optimize_thread.test

@@ -9,8 +9,6 @@
 --source include/have_debug.inc
 --source include/have_log_bin.inc
 
-call mtr.add_suppression("InnoDB: Warning: database page corruption or a failed");
-
 let $file_format_max=`SELECT @@innodb_file_format_max`;
 CREATE TABLE t1(a CHAR(255),
                 b CHAR(255),