MDEV-23844 Atomic DROP TABLE (single table)

Logging logic:
- Log tables, just before they are dropped, to the ddl log
- After the last table for the statement is dropped, log an xid for the
  whole ddl log event

In case of crash:
- Remove first any active DROP TABLE events from the ddl log that matches
  xids found in binary log (this mean the drop was successful and was
  propery logged).
- Loop over all active DROP TABLE events
  - Ensure that the table is completely dropped
- Write a DROP TABLE entry to the binary log with the dropped tables.

Other things:
- Added code to ha_drop_table() to be able to tell the difference if
  a get_new_handler() failed because of out-of-memory or because the
  handler refused/was not able to create a a handler. This was needed
  to get sequences to work as sequences needs a share object to be passed
  to get_new_handler()
- TC_LOG_BINLOG::recover() was changed to always collect Xid's from the
  binary log and always call ddl_log_close_binlogged_events(). This was
  needed to be able to collect DROP TABLE events with embedded Xid's
  (used by ddl log).
- Added a new variable "$grep_script" to binlog filter to be able to find
  only rows that matches a regexp.
- Had to adjust some test that changed because drop statements are a bit
  larger in the binary log than before (as we have to store the xid)

Other things:
- MDEV-25588 Atomic DDL: Binlog query event written upon recovery is corrupt
  fixed (in the original commit).

mysql-test/include/filter_file.inc

@@ -56,6 +56,9 @@
 #
 #   $filter_script
 #     If set, rows matching this regexp will be filtered out
+#
+#   $grep_script
+#     If set, only include rows matching this regexp
 
 --let $include_filename= filter_file.inc
 --source include/begin_include_file.inc
@@ -71,6 +74,7 @@ if ($rpl_debug)
 --let _FF_PRE_SCRIPT= $pre_script
 --let _FF_SCRIPT= $script
 --let _FF_FILTER_SCRIPT= $filter_script
+--let _FF_GREP_SCRIPT= $grep_script
 --let _FF_INPUT_FILE= $input_file
 --let _FF_OUTPUT_FILE= $output_file
 --let _FF_SELECT_COLUMNS= $select_columns
@@ -85,6 +89,7 @@ perl;
   $pre_script =~ s/DOLLAR/\$/g;
   my $script = $ENV{'_FF_SCRIPT'};
   my $filter_script = $ENV{'_FF_FILTER_SCRIPT'};
+  my $grep_script = $ENV{'_FF_GREP_SCRIPT'};
   $script =~ s/DOLLAR/\$/g;
   my $input_file = $ENV{'_FF_INPUT_FILE'};
   my $output_file = $ENV{'_FF_OUTPUT_FILE'};
@@ -129,7 +134,8 @@ perl;
     {
       ' . $script . '
     }
-    if (!$filter_script || ! m/$filter_script/)
+    if ((!$filter_script || ! m/$filter_script/) &&
+       (!$grep_script || m/$grep_script/))
     {
       $filtered_contents .= $_."\n";
     }

mysql-test/suite/atomic/drop_sequence.result

@@ -0,0 +1,104 @@
+call mtr.add_suppression("InnoDB: .* does not exist in the InnoDB internal");
+"engine: aria  crash point: ddl_log_drop_before_delete_table  position: 1"
+t2.MAD
+t2.MAI
+t2.TRG
+t2.frm
+t2_trg.TRN
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_delete_table  position: 2"
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_delete_table  position: 3"
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2`,`test`.`ts` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_after_delete_table  position: 1"
+t2.MAD
+t2.MAI
+t2.TRG
+t2.frm
+t2_trg.TRN
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_after_delete_table  position: 2"
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_after_delete_table  position: 3"
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2`,`test`.`ts` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_drop_trigger  position: 1"
+t2.MAD
+t2.MAI
+t2.TRG
+t2.frm
+t2_trg.TRN
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_drop_trigger  position: 2"
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_drop_trigger  position: 3"
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2`,`test`.`ts` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_drop_trigger2  position: 1"
+t2.MAD
+t2.MAI
+t2.TRG
+t2.frm
+t2_trg.TRN
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_drop_trigger2  position: 2"
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_drop_trigger2  position: 3"
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2`,`test`.`ts` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_after_drop_trigger  position: 1"
+t2.MAD
+t2.MAI
+t2.TRG
+t2.frm
+t2_trg.TRN
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_after_drop_trigger  position: 2"
+ts.MAD
+ts.MAI
+ts.frm
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_after_drop_trigger  position: 3"
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2`,`test`.`ts` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_binlog  position: 1"
+master-bin.000002	#	Query	#	#	DROP TABLE IF EXISTS `test`.`t1`,`test`.`t2`,`test`.`ts` /* generated by ddl recovery */
+"engine: aria  crash point: ddl_log_drop_before_binlog  position: 2"
+"No crash!"
+master-bin.000001	#	Query	#	#	use `test`; DROP TABLE `t1`,`t2`,`ts` /* generated by server */
+"engine: aria  crash point: ddl_log_drop_before_binlog  position: 3"
+"No crash!"
+master-bin.000001	#	Query	#	#	use `test`; DROP TABLE `t1`,`t2`,`ts` /* generated by server */
+"engine: aria  crash point: ddl_log_drop_after_binlog  position: 1"
+master-bin.000001	#	Query	#	#	use `test`; DROP TABLE `t1`,`t2`,`ts` /* generated by server */
+"engine: aria  crash point: ddl_log_drop_after_binlog  position: 2"
+"No crash!"
+master-bin.000001	#	Query	#	#	use `test`; DROP TABLE `t1`,`t2`,`ts` /* generated by server */
+"engine: aria  crash point: ddl_log_drop_after_binlog  position: 3"
+"No crash!"
+master-bin.000001	#	Query	#	#	use `test`; DROP TABLE `t1`,`t2`,`ts` /* generated by server */
+Warnings:
+Note	1051	Unknown table 'test.t1,test.t2,test.ts'

mysql-test/suite/atomic/drop_sequence.test

@@ -0,0 +1,115 @@
+--source include/have_debug.inc
+--source include/have_log_bin.inc
+--source include/not_valgrind.inc
+
+#
+# Testing of atomic drop with crashes in a lot of different places
+#
+
+call mtr.add_suppression("InnoDB: .* does not exist in the InnoDB internal");
+let $MYSQLD_DATADIR= `SELECT @@datadir`;
+
+let $engine_count=1;
+let $engines='aria';
+
+let $crash_count=7;
+let $crash_points='ddl_log_drop_before_delete_table', 'ddl_log_drop_after_delete_table', 'ddl_log_drop_before_drop_trigger', 'ddl_log_drop_before_drop_trigger2', 'ddl_log_drop_after_drop_trigger', 'ddl_log_drop_before_binlog', 'ddl_log_drop_after_binlog';
+
+# Number of drops in the tested statement
+let $drops=3;
+
+let $old_debug=`select @@debug_dbug`;
+
+let $e=0;
+let $keep_include_silent=1;
+let $grep_script=DROP TABLE;
+--disable_query_log
+
+while ($e < $engine_count)
+{
+  inc $e;
+  let $engine=`select ELT($e, $engines)`;
+  let $default_engine=$engine;
+  let $extra_option=;
+
+  if ($engine == "aria")
+  {
+    let $extra_option=transactional=1;
+  }
+  if ($engine == "aria_notrans")
+  {
+    let $default_engine="aria";
+    let $extra_option=transactional=0;
+  }
+
+  let $c=0;
+  while ($c < $crash_count)
+  {
+    inc $c;
+    let $crash=`select ELT($c, $crash_points)`;
+    let $r=0;
+    while ($r < $drops)
+    {
+      inc $r;
+     --eval set @@default_storage_engine=$default_engine
+     --eval create table t1 (a int not null) $extra_option;
+     --eval create table t2 (b int not null) $extra_option;
+      create sequence ts;
+      insert into t1 values(1);
+      insert into t2 values(2);
+      flush tables;
+
+      delimiter |;
+      create trigger t1_trg before insert on t1 for each row
+      begin
+        if isnull(new.a) then
+          set new.a:= 1000;
+        end if;
+      end|
+      create trigger t2_trg before insert on t2 for each row
+      begin
+        if isnull(new.b) then
+          set new.b:= 2000;
+        end if;
+      end|
+      delimiter ;|
+
+      RESET MASTER;
+
+      echo "engine: $engine  crash point: $crash  position: $r";
+      --exec echo "restart" > $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+      --disable_reconnect
+      --eval set @@debug_dbug="+d,$crash",@debug_crash_counter=$r
+      let $errno=0;
+      --error 0,2013
+      drop table t1,t2,ts;
+      let $error=$errno;
+      --enable_reconnect
+      --source include/wait_until_connected_again.inc
+      --disable_query_log
+      --eval set @@debug_dbug="$old_debug"
+
+      if ($error == 0)
+      {
+        echo "No crash!";
+      }
+      # Check which tables still exists
+      --list_files $MYSQLD_DATADIR/test t*
+
+      --let $binlog_file=master-bin.000001
+      --source include/show_binlog_events.inc
+      if ($error)
+      {
+        --let $binlog_file=master-bin.000002
+        --source include/show_binlog_events.inc
+      }
+      # Really drop the tables. The warnings will show what was dropped
+      --disable_warnings
+      drop table if exists t1,t2,ts;
+      --enable_warnings
+    }
+  }
+}
+drop table if exists t1,t2,ts;
+
+--enable_query_log