MDEV-23691 S3 storage engine: delayed slave can drop the table

This commit fixed the problems with S3 after the "DROP TABLE FORCE" changes.
It also fixes all failing replication S3 tests.

A slave is delayed if it is trying to execute replicated queries on a
table that is already converted to S3 by the master later in the binlog.

Fixes for replication events on S3 tables for delayed slaves:
- INSERT and INSERT ... SELECT and CREATE TABLE are ignored but written
  to the binary log.   UPDATE & DELETE will be fixed in a future commit.

Other things:
- On slaves with --s3-slave-ignore-updates set, allow S3 tables to be
  opened in read-write mode. This was done to be able to
  ignore-but-replicate queries like insert.  Without this change any
  open of an S3 table failed with 'Table is read only' which is too
  early to be able to replicate the original query.
- Errors are now printed if handler::extra() call fails in
  wait_while_tables_are_used().
- Error message for row changes are changed from HA_ERR_WRONG_COMMAND
  to HA_ERR_TABLE_READONLY.
- Disable some maria_extra() calls for S3 tables. This could cause
  S3 tables to fail in some cases.
- Added missing thr_lock_delete() to ma_open() in case of failure.
- Removed from mysql_prepare_insert() the not needed argument 'table'.

mysql-test/suite/s3/alter.result

@@ -25,6 +25,14 @@ t1	CREATE TABLE `t1` (
   `a` int(11) DEFAULT NULL,
   `b` int(11) DEFAULT NULL
 ) ENGINE=Aria DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 COMMENT='hello'
+alter table t1 engine=s3;
+alter table t1 engine=innodb;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) DEFAULT NULL
+) ENGINE=InnoDB DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1 COMMENT='hello'
 select count(*), sum(a), sum(b) from t1;
 count(*)	sum(a)	sum(b)
 1000	500500	510500

mysql-test/suite/s3/alter.test

@@ -1,5 +1,6 @@
 --source include/have_s3.inc
 --source include/have_sequence.inc
+--source include/have_innodb.inc
 
 #
 # Create unique database for running the tests
@@ -21,6 +22,9 @@ alter table t1 comment="hello";
 show create table t1;
 alter table t1 engine=aria;
 show create table t1;
+alter table t1 engine=s3;
+alter table t1 engine=innodb;
+show create table t1;
 select count(*), sum(a), sum(b) from t1;
 drop table t1;
 

mysql-test/suite/s3/partition.result

@@ -63,7 +63,7 @@ count(*)
 ALTER TABLE t2 CHECK PARTITION p3;
 Table	Op	Msg_type	Msg_text
 s3.t2	check	error	Subpartition p3sp0 returned error
-s3.t2	check	error	Unknown - internal error 131 during operation
+s3.t2	check	error	Unknown - internal error 165 during operation
 SELECT count(*) FROM t2;
 count(*)
 6

mysql-test/suite/s3/replication.inc

@@ -20,6 +20,8 @@ connection master;
 
 create table t1 (a int, b int) engine=aria;
 insert into t1 select seq,seq+10 from seq_1_to_10;
+sync_slave_with_master;
+connection master;
 alter table t1 engine=s3;
 show create table t1;
 
@@ -116,12 +118,18 @@ connection slave;
 stop slave;
 connection master;
 rename table t1 to t2;
-create table t1 (a int, b int) engine=aria;
+# Check the different create options with the table
+create table t1 (a int) engine=aria;
+drop table t1;
+create table if not exists t1 (a int, b int) engine=aria;
+drop table t1;
+create or replace table t1 (a int, b int, c int) engine=aria;
 alter table t1 engine=s3;
 connection slave;
 start slave;
 connection master;
 sync_slave_with_master;
+show create table t1;
 select * from t1 limit 2;
 select * from t2 limit 2;
 connection master;

mysql-test/suite/s3/replication_delayed.cnf

@@ -0,0 +1,3 @@
+!include ../rpl/my.cnf
+!include ./my.cnf
+!include ./slave.cnf

mysql-test/suite/s3/replication_delayed.result

@@ -0,0 +1,124 @@
+include/master-slave.inc
+[connection master]
+set binlog_format=mixed;
+RESET MASTER;
+connection slave;
+set binlog_format=mixed;
+RESET MASTER;
+connection master;
+connection slave;
+use database;
+connection master;
+#
+# MDEV-23691 S3 storage engine: delayed slave can drop the table
+#
+connection slave;
+stop slave;
+connection master;
+create /*or replace*/ table t100 (
+pk varchar(100)
+) engine = 'innodb';
+insert into t100 values ('old data');
+alter table t100 engine=s3;
+drop table t100;
+create /*or replace*/ table t100 (
+pk varchar(100)
+) engine= innodb;
+insert into t100 select 'new data' from seq_1_to_10;
+alter table t100 engine=s3;
+select count(*), 'before slave start' from t100;
+count(*)	before slave start
+10	before slave start
+connection slave;
+start slave;
+connection master;
+connection slave;
+connection master;
+flush tables;
+select count(*), 'after slave start' from t100;
+count(*)	after slave start
+10	after slave start
+show create table t100;
+Table	Create Table
+t100	CREATE TABLE `t100` (
+  `pk` varchar(100) DEFAULT NULL
+) ENGINE=S3 DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1
+connection slave;
+select count(*) from t100;
+count(*)
+10
+connection master;
+drop table t100;
+#
+# Test delayed slave with inserts
+#
+connection slave;
+stop slave;
+connection master;
+create table t1 (a int) engine=innodb;
+insert into t1 values (1),(2),(3);
+insert into t1 select * from seq_4_to_6;
+alter table t1 engine=s3;
+connection slave;
+start slave;
+connection master;
+connection slave;
+select * from t1;
+a
+1
+2
+3
+4
+5
+6
+connection master;
+drop table t1;
+#
+# Check slave binary log
+#
+connection slave;
+include/show_binlog_events.inc
+Log_name	Pos	Event_type	Server_id	End_log_pos	Info
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	create database database
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; create /*or replace*/ table t100 (
+pk varchar(100)
+) engine = 'innodb'
+slave-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; insert into t100 values ('old data')
+slave-bin.000001	#	Query	#	#	COMMIT
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; alter table t100 engine=s3
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; DROP TABLE IF EXISTS `t100` /* generated by server */
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; create /*or replace*/ table t100 (
+pk varchar(100)
+) engine= innodb
+slave-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; insert into t100 select 'new data' from seq_1_to_10
+slave-bin.000001	#	Query	#	#	COMMIT
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; alter table t100 engine=s3
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; flush tables
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; DROP TABLE IF EXISTS `t100` /* generated by server */
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; create table t1 (a int) engine=innodb
+slave-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; insert into t1 values (1),(2),(3)
+slave-bin.000001	#	Query	#	#	COMMIT
+slave-bin.000001	#	Gtid	#	#	BEGIN GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; insert into t1 select * from seq_4_to_6
+slave-bin.000001	#	Query	#	#	COMMIT
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; alter table t1 engine=s3
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; DROP TABLE IF EXISTS `t1` /* generated by server */
+connection master;
+#
+# clean up
+#
+include/rpl_end.inc

mysql-test/suite/s3/replication_delayed.test

@@ -0,0 +1,115 @@
+--source include/have_s3.inc
+--source include/have_innodb.inc
+--source include/have_binlog_format_mixed.inc
+--source include/master-slave.inc
+--source include/have_sequence.inc
+
+# First clear the binlog
+set binlog_format=mixed;
+RESET MASTER;
+connection slave;
+set binlog_format=mixed;
+RESET MASTER;
+connection master;
+
+#
+# Create unique database for running the tests
+#
+--source create_database.inc
+sync_slave_with_master;
+--replace_result $database database
+--eval use $database
+connection master;
+
+--echo #
+--echo # MDEV-23691 S3 storage engine: delayed slave can drop the table
+--echo #
+
+connection slave;
+stop slave;
+connection master;
+
+#
+# Create version 1 of the table
+#
+
+create /*or replace*/ table t100 (
+  pk varchar(100)
+) engine = 'innodb';
+
+insert into t100 values ('old data');
+alter table t100 engine=s3;
+
+#
+# Create version 2 of the table
+#
+drop table t100;
+create /*or replace*/ table t100 (
+  pk varchar(100)
+) engine= innodb;
+insert into t100 select 'new data' from seq_1_to_10;
+alter table t100 engine=s3;
+
+select count(*), 'before slave start' from t100;
+
+#
+#  Now, start the slave
+#
+connection slave;
+start slave;
+connection master;
+sync_slave_with_master;
+#select count(*) from t100;
+connection master;
+
+flush tables;
+select count(*), 'after slave start' from t100;
+show create table t100;
+
+connection slave;
+
+select count(*) from t100;
+
+connection master;
+
+drop table t100;
+
+--echo #
+--echo # Test delayed slave with inserts
+--echo #
+
+
+# Stop slave
+
+connection slave;
+stop slave;
+connection master;
+
+# Create tables with data while slave is stopped
+create table t1 (a int) engine=innodb;
+insert into t1 values (1),(2),(3);
+insert into t1 select * from seq_4_to_6;
+alter table t1 engine=s3;
+
+connection slave;
+start slave;
+connection master;
+sync_slave_with_master;
+select * from t1;
+connection master;
+drop table t1;
+
+--echo #
+--echo # Check slave binary log
+--echo #
+
+sync_slave_with_master;
+--let $binlog_database=$database
+--source include/show_binlog_events.inc
+connection master;
+
+--echo #
+--echo # clean up
+--echo #
+--source drop_database.inc
+--source include/rpl_end.inc

mysql-test/suite/s3/replication_mixed.result

@@ -13,6 +13,8 @@ connection master;
 #
 create table t1 (a int, b int) engine=aria;
 insert into t1 select seq,seq+10 from seq_1_to_10;
+connection slave;
+connection master;
 alter table t1 engine=s3;
 show create table t1;
 Table	Create Table
@@ -134,14 +136,25 @@ connection slave;
 stop slave;
 connection master;
 rename table t1 to t2;
-create table t1 (a int, b int) engine=aria;
+create table t1 (a int) engine=aria;
+drop table t1;
+create table if not exists t1 (a int, b int) engine=aria;
+drop table t1;
+create or replace table t1 (a int, b int, c int) engine=aria;
 alter table t1 engine=s3;
 connection slave;
 start slave;
 connection master;
 connection slave;
+show create table t1;
+Table	Create Table
+t1	CREATE TABLE `t1` (
+  `a` int(11) DEFAULT NULL,
+  `b` int(11) DEFAULT NULL,
+  `c` int(11) DEFAULT NULL
+) ENGINE=S3 DEFAULT CHARSET=latin1 PAGE_CHECKSUM=1
 select * from t1 limit 2;
-a	b
+a	b	c
 select * from t2 limit 2;
 a	b	f
 1	11	NULL
@@ -238,7 +251,15 @@ slave-bin.000001	#	Query	#	#	use `database`; set @@sql_if_exists=1; alter table
 slave-bin.000001	#	Gtid	#	#	GTID #-#-#
 slave-bin.000001	#	Query	#	#	use `database`; set @@sql_if_exists=1; rename table t1 to t2
 slave-bin.000001	#	Gtid	#	#	GTID #-#-#
-slave-bin.000001	#	Query	#	#	use `database`; create table t1 (a int, b int) engine=aria
+slave-bin.000001	#	Query	#	#	use `database`; create table t1 (a int) engine=aria
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; DROP TABLE IF EXISTS `t1` /* generated by server */
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; create table if not exists t1 (a int, b int) engine=aria
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; DROP TABLE IF EXISTS `t1` /* generated by server */
+slave-bin.000001	#	Gtid	#	#	GTID #-#-#
+slave-bin.000001	#	Query	#	#	use `database`; create or replace table t1 (a int, b int, c int) engine=aria
 slave-bin.000001	#	Gtid	#	#	GTID #-#-#
 slave-bin.000001	#	Query	#	#	use `database`; alter table t1 engine=s3
 slave-bin.000001	#	Gtid	#	#	GTID #-#-#