MDEV-18450 Slaves wait shutdown

The patches features an optional shutdown behavior to hold on until
after all connected slaves have been sent the last binlogged event.
The connected slave is one whose START SLAVE has been acknowledged and
that was not stopped since that though it could be technically
reconnecting in background.

The solution therefore disallows killing the dump thread until is has
found EOF of the latest binlog file.  It is up to the shutdown
requester (DBA) to set up a sufficiently large shutdown timeout value
for shudown to wait patiently until lagging behind slaves have been
synchronized. On the other hand if a specific slave needs exclusion
from synchronization the DBA would have to stop it manually which
would terminate its dump thread.

`mysqladmin shutdown' is extended with a `--wait_for_all_slaves' option
which translates to `SHUTDOW WAIT FOR ALL SLAVES' sql query
to enable the feature on the client side.

The patch also performs a small refactoring of the server shutdown
around close_connections() to introduce kill thread phases which
are two as of current.

client/client_priv.h

@@ -100,6 +100,7 @@ enum options_client
   OPT_SKIP_ANNOTATE_ROWS_EVENTS,
   OPT_SSL_CRL, OPT_SSL_CRLPATH,
   OPT_PRINT_ROW_COUNT, OPT_PRINT_ROW_EVENT_POSITIONS,
+  OPT_SHUTDOWN_WAIT_FOR_SLAVES,
   OPT_MAX_CLIENT_OPTION /* should be always the last */
 };
 

client/mysqladmin.cc

@@ -40,7 +40,8 @@ ulonglong last_values[MAX_MYSQL_VAR+100];
 static int interval=0;
 static my_bool option_force=0,interrupted=0,new_line=0,
                opt_compress= 0, opt_local= 0, opt_relative= 0, opt_verbose= 0,
-               opt_vertical= 0, tty_password= 0, opt_nobeep;
+               opt_vertical= 0, tty_password= 0, opt_nobeep,
+               opt_shutdown_wait_for_slaves= 0;
 static my_bool debug_info_flag= 0, debug_check_flag= 0;
 static uint tcp_port = 0, option_wait = 0, option_silent=0, nr_iterations;
 static uint opt_count_iterations= 0, my_end_arg;
@@ -218,6 +219,11 @@ static struct my_option my_long_options[] =
   {"shutdown_timeout", OPT_SHUTDOWN_TIMEOUT, "", &opt_shutdown_timeout,
    &opt_shutdown_timeout, 0, GET_ULONG, REQUIRED_ARG,
    SHUTDOWN_DEF_TIMEOUT, 0, 3600*12, 0, 1, 0},
+  {"wait_for_all_slaves", OPT_SHUTDOWN_WAIT_FOR_SLAVES,
+   "Defers shutdown until after all binlogged events have been sent to "
+   "all connected slaves", &opt_shutdown_wait_for_slaves,
+   &opt_shutdown_wait_for_slaves, 0, GET_BOOL, NO_ARG, 0, 0, 0,
+   0, 0, 0},
   {"plugin_dir", OPT_PLUGIN_DIR, "Directory for client-side plugins.",
     &opt_plugin_dir, &opt_plugin_dir, 0,
    GET_STR, REQUIRED_ARG, 0, 0, 0, 0, 0, 0},
@@ -693,7 +699,17 @@ static int execute_commands(MYSQL *mysql,int argc, char **argv)
 	  !stat(pidfile, &pidfile_status))
 	last_modified= pidfile_status.st_mtime;
 
-      if (mysql_shutdown(mysql, SHUTDOWN_DEFAULT))
+      if (opt_shutdown_wait_for_slaves)
+      {
+        sprintf(buff, "SHUTDOWN WAIT FOR ALL SLAVES");
+        if (mysql_query(mysql, buff))
+        {
+          my_printf_error(0, "%s failed; error: '%-.200s'",
+                          error_flags, buff, mysql_error(mysql));
+          return -1;
+        }
+      }
+      else if (mysql_shutdown(mysql, SHUTDOWN_DEFAULT))
       {
 	my_printf_error(0, "shutdown failed; error: '%s'", error_flags,
 			mysql_error(mysql));

mysql-test/mysql-test-run.pl

@@ -4550,7 +4550,8 @@ ($$)
      qr/InnoDB: See also */,
      qr/InnoDB: Cannot open .*ib_buffer_pool.* for reading: No such file or directory*/,
      qr/InnoDB: Table .*mysql.*innodb_table_stats.* not found./,
-     qr/InnoDB: User stopword table .* does not exist./
+     qr/InnoDB: User stopword table .* does not exist./,
+     qr/Dump thread [0-9]+ last sent to server [0-9]+ binlog file:pos .+/
 
     );
 

mysql-test/suite/rpl/include/rpl_shutdown_wait_slaves.inc

@@ -0,0 +1,83 @@
+--connection server_1
+
+CREATE TABLE t1 (a INT) ENGINE=innodb;
+
+--save_master_pos
+
+--connection server_2
+--sync_with_master
+
+--connection server_3
+--sync_with_master
+
+--connection server_4
+--source include/stop_slave.inc
+
+--connection server_1
+--disable_query_log
+--let $count=1000
+while ($count)
+{
+ INSERT INTO t1 SET a=1;
+ --dec $count
+}
+--enable_query_log
+--save_master_pos
+
+# Shutdown master and restart server_4 who will be waiting for the master
+# to start replication at its shutdown beginning phase.
+# The being forked out server_4 dump thread must relate to a record
+# in slave_list, and it won't start sending out binlog events
+# until has received a signal from the shutdown thread.
+# This also proves delivery to a  started-in-middle-of-shutdown slave.
+--connection server_1
+SET @@GLOBAL.debug_dbug="+d,simulate_delay_at_shutdown";
+
+--connection server_4
+--source include/start_slave.inc
+
+--connection server_1
+--write_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+wait
+EOF
+# --shutdown_server 60
+--send SHUTDOWN WAIT FOR ALL SLAVES
+--reap
+--source include/wait_until_disconnected.inc
+#
+# MDEV-18450 liveness condition:
+# Despite shutdown even "late" slave #4 is in sync
+#
+--connection server_4
+--sync_with_master
+
+--connection server_3
+--sync_with_master
+
+--connection server_2
+--sync_with_master
+
+--connection server_1
+--append_file $MYSQLTEST_VARDIR/tmp/mysqld.1.expect
+restart
+EOF
+
+--connection default
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+--connection server_1
+--enable_reconnect
+--source include/wait_until_connected_again.inc
+
+#
+# Cleanup
+#
+--connection server_1
+DROP TABLE t1;
+
+--connection server_2
+--source include/start_slave.inc
+--connection server_3
+--source include/start_slave.inc
+--connection server_4
+--source include/start_slave.inc

mysql-test/suite/rpl/r/rpl_shutdown_wait_semisync_slaves.result

@@ -0,0 +1,58 @@
+include/rpl_init.inc [topology=1->2, 1->3, 1->4]
+connection server_1;
+call mtr.add_suppression("Timeout waiting for reply of binlog");
+SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
+connection server_2;
+set global rpl_semi_sync_slave_enabled = 1;
+include/stop_slave.inc
+include/start_slave.inc
+set global rpl_semi_sync_slave_enabled = 1;
+connection server_3;
+set global rpl_semi_sync_slave_enabled = 1;
+include/stop_slave.inc
+include/start_slave.inc
+set global rpl_semi_sync_slave_enabled = 1;
+connection server_1;
+CREATE TABLE t1 (a INT) ENGINE=innodb;
+connection server_2;
+connection server_3;
+connection server_4;
+include/stop_slave.inc
+connection server_1;
+connection server_1;
+SET @@GLOBAL.debug_dbug="+d,simulate_delay_at_shutdown";
+connection server_4;
+include/start_slave.inc
+connection server_1;
+SHUTDOWN WAIT FOR ALL SLAVES;
+connection server_4;
+connection server_3;
+connection server_2;
+connection server_1;
+connection default;
+connection server_1;
+connection server_1;
+DROP TABLE t1;
+connection server_2;
+include/start_slave.inc
+Warnings:
+Note	1254	Slave is already running
+connection server_3;
+include/start_slave.inc
+Warnings:
+Note	1254	Slave is already running
+connection server_4;
+include/start_slave.inc
+Warnings:
+Note	1254	Slave is already running
+connection server_2;
+include/stop_slave.inc
+include/start_slave.inc
+SET @@GLOBAL.rpl_semi_sync_slave_enabled = 0;;
+connection server_3;
+include/stop_slave.inc
+include/start_slave.inc
+SET @@GLOBAL.rpl_semi_sync_slave_enabled = 0;;
+connection server_1;
+SET @@GLOBAL.rpl_semi_sync_master_enabled = 0;
+include/rpl_end.inc

mysql-test/suite/rpl/r/rpl_shutdown_wait_slaves.result

@@ -0,0 +1,35 @@
+include/rpl_init.inc [topology=1->2, 1->3, 1->4]
+connection server_1;
+CREATE TABLE t1 (a INT) ENGINE=innodb;
+connection server_2;
+connection server_3;
+connection server_4;
+include/stop_slave.inc
+connection server_1;
+connection server_1;
+SET @@GLOBAL.debug_dbug="+d,simulate_delay_at_shutdown";
+connection server_4;
+include/start_slave.inc
+connection server_1;
+SHUTDOWN WAIT FOR ALL SLAVES;
+connection server_4;
+connection server_3;
+connection server_2;
+connection server_1;
+connection default;
+connection server_1;
+connection server_1;
+DROP TABLE t1;
+connection server_2;
+include/start_slave.inc
+Warnings:
+Note	1254	Slave is already running
+connection server_3;
+include/start_slave.inc
+Warnings:
+Note	1254	Slave is already running
+connection server_4;
+include/start_slave.inc
+Warnings:
+Note	1254	Slave is already running
+include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_shutdown_wait_semisync_slaves.cnf

@@ -0,0 +1,16 @@
+!include ../my.cnf
+
+[mysqld.1]
+log_warnings=3
+[mysqld.2]
+
+[mysqld.3]
+
+[mysqld.4]
+
+[ENV]
+SERVER_MYPORT_3=		@mysqld.3.port
+SERVER_MYSOCK_3=		@mysqld.3.socket
+
+SERVER_MYPORT_4=		@mysqld.4.port
+SERVER_MYSOCK_4=		@mysqld.4.socket

mysql-test/suite/rpl/t/rpl_shutdown_wait_semisync_slaves.test

@@ -0,0 +1,46 @@
+#
+# MDEV-18450 "Slow" shutdown to wait for slaves that are to be fed
+# with everything in the master binlog before shutdown completes.
+#
+# This is a semisync version of basic tests.
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--let $rpl_topology=1->2, 1->3, 1->4
+--source include/rpl_init.inc
+
+--connection server_1
+call mtr.add_suppression("Timeout waiting for reply of binlog");
+--let $sav_enabled_master=`SELECT @@GLOBAL.rpl_semi_sync_master_enabled`
+SET @@GLOBAL.rpl_semi_sync_master_enabled = 1;
+
+--let slaves= 3
+--let i= 2
+while (`SELECT $i <= $slaves`)
+{
+  --connection server_$i
+  --let $sav_enabled_slave=`SELECT @@GLOBAL.rpl_semi_sync_slave_enabled`
+  set global rpl_semi_sync_slave_enabled = 1;
+
+  source include/stop_slave.inc;
+  source include/start_slave.inc;
+  set global rpl_semi_sync_slave_enabled = 1;
+
+  --inc $i
+}
+
+--source include/rpl_shutdown_wait_slaves.inc
+--let i= 2
+while (`SELECT $i <= $slaves`)
+{
+  --connection server_$i
+  source include/stop_slave.inc;
+  source include/start_slave.inc;
+  --eval SET @@GLOBAL.rpl_semi_sync_slave_enabled = $sav_enabled_slave;
+
+  --inc $i
+}
+
+--connection server_1
+--eval SET @@GLOBAL.rpl_semi_sync_master_enabled = $sav_enabled_master
+
+--source include/rpl_end.inc

mysql-test/suite/rpl/t/rpl_shutdown_wait_slaves.cnf

@@ -0,0 +1,16 @@
+!include ../my.cnf
+
+[mysqld.1]
+log_warnings=3
+[mysqld.2]
+
+[mysqld.3]
+
+[mysqld.4]
+
+[ENV]
+SERVER_MYPORT_3=		@mysqld.3.port
+SERVER_MYSOCK_3=		@mysqld.3.socket
+
+SERVER_MYPORT_4=		@mysqld.4.port
+SERVER_MYSOCK_4=		@mysqld.4.socket

mysql-test/suite/rpl/t/rpl_shutdown_wait_slaves.test

@@ -0,0 +1,11 @@
+#
+# MDEV-18450 "Slow" shutdown to wait for slaves that are to be fed
+# with everything in the master binlog before shutdown completes.
+#
+--source include/have_innodb.inc
+--source include/have_debug.inc
+--let $rpl_topology=1->2, 1->3, 1->4
+--source include/rpl_init.inc
+
+--source include/rpl_shutdown_wait_slaves.inc
+--source include/rpl_end.inc