MDEV-25975 innodb_disallow_writes causes shutdown to hang

We will remove the parameter innodb_disallow_writes because it is badly
designed and implemented. The parameter was never allowed at startup.
It was only internally used by Galera snapshot transfer.
If a user executed
SET GLOBAL innodb_disallow_writes=ON;
the server could hang even on subsequent read operations.

During Galera snapshot transfer, we will block writes
to implement an rsync friendly snapshot, as follows:

sst_flush_tables() will acquire a global lock by executing
FLUSH TABLES WITH READ LOCK, which will block any writes
at the high level.

sst_disable_innodb_writes(), invoked via ha_disable_internal_writes(true),
will suspend or disable InnoDB background tasks or threads that could
initiate writes. As part of this, log_make_checkpoint() will be invoked
to ensure that anything in the InnoDB buf_pool.flush_list will be written
to the data files. This has the nice side effect that the Galera joiner
will avoid crash recovery.

The changes to sql/wsrep.cc and to the tests are based on a prototype
that was developed by Jan Lindström.

Reviewed by: Jan Lindström

extra/mariabackup/xtrabackup.cc

@@ -4372,11 +4372,6 @@ static bool xtrabackup_backup_func()
 	crc_init();
 	recv_sys_init();
 
-#ifdef WITH_INNODB_DISALLOW_WRITES
-	srv_allow_writes_event = os_event_create(0);
-	os_event_set(srv_allow_writes_event);
-#endif
-
 	xb_filters_init();
 
 	xb_fil_io_init();
@@ -5837,10 +5832,6 @@ static bool xtrabackup_prepare_func(char** argv)
 		log_sys.create();
 		recv_recovery_on = true;
 
-#ifdef WITH_INNODB_DISALLOW_WRITES
-		srv_allow_writes_event = os_event_create(0);
-		os_event_set(srv_allow_writes_event);
-#endif
 		dberr_t err = xb_data_files_init();
 		if (err != DB_SUCCESS) {
 			msg("mariabackup: error: xb_data_files_init() failed "
@@ -5862,9 +5853,6 @@ static bool xtrabackup_prepare_func(char** argv)
 		xb_filter_hash_free(inc_dir_tables_hash);
 
 		fil_system.close();
-#ifdef WITH_INNODB_DISALLOW_WRITES
-		os_event_destroy(srv_allow_writes_event);
-#endif
 		innodb_free_param();
 		log_sys.close();
 		sync_check_close();

mysql-test/suite/galera/r/galera_bf_abort_shutdown.result

@@ -0,0 +1,11 @@
+connection node_1;
+connection node_2;
+connection node_1;
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+connection node_2;
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*");
+SET DEBUG_SYNC = 'wsrep_before_certification WAIT_FOR continue';
+INSERT INTO t1 VALUES (1);
+connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2;
+connection node_1;
+DROP TABLE t1;

mysql-test/suite/galera/r/galera_drop_database.result

@@ -46,3 +46,6 @@ SHOW TABLES;
 Tables_in_fts
 DROP DATABASE fts;
 connection node_2;
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:");
+Warnings:
+Note	1049	Unknown database 'fts'

mysql-test/suite/galera/r/galera_events2.result

@@ -109,6 +109,7 @@ f1	f2
 SELECT EVENT_CATALOG, EVENT_SCHEMA, EVENT_NAME, DEFINER, EVENT_BODY, EVENT_DEFINITION, EVENT_TYPE, EXECUTE_AT, INTERVAL_VALUE, INTERVAL_FIELD, STATUS,ON_COMPLETION, EVENT_COMMENT FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_NAME='one_event';
 EVENT_CATALOG	EVENT_SCHEMA	EVENT_NAME	DEFINER	EVENT_BODY	EVENT_DEFINITION	EVENT_TYPE	EXECUTE_AT	INTERVAL_VALUE	INTERVAL_FIELD	STATUS	ON_COMPLETION	EVENT_COMMENT
 def	test	one_event	root@localhost	SQL	SELECT 123	RECURRING	NULL	10	SECOND	SLAVESIDE_DISABLED	NOT PRESERVE	
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*");
 connection node_1;
 SELECT * FROM t1;
 f1	f2

mysql-test/suite/galera/r/galera_restart_on_unknown_option.result

@@ -39,6 +39,7 @@ f1	f2
 connection node_2;
 Starting server ...
 Starting server ...
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:");
 SELECT * FROM t1;
 f1	f2
 1	a

mysql-test/suite/galera/t/galera_bf_abort_shutdown.test

@@ -0,0 +1,34 @@
+#
+# This test verifies that the server can be shut down even if
+# some of the wsrep transactions are in QUERY_COMMITTING state.
+# In this case the shutdown sequence may do a BF abort for the
+# connection.
+#
+
+--source include/have_innodb.inc
+--source include/galera_cluster.inc
+--source include/have_debug_sync.inc
+
+# Save original auto_increment_offset values.
+--let $node_1=node_1
+--let $node_2=node_2
+--source include/auto_increment_offset_save.inc
+
+--connection node_1
+CREATE TABLE t1 (f1 INT PRIMARY KEY);
+
+--connection node_2
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*");
+SET DEBUG_SYNC = 'wsrep_before_certification WAIT_FOR continue';
+--send INSERT INTO t1 VALUES (1)
+
+--connect node_2a, 127.0.0.1, root, , test, $NODE_MYPORT_2
+--source include/restart_mysqld.inc
+
+# Restore original auto_increment_offset values.
+--let $node_2=node_2a
+--source include/auto_increment_offset_restore.inc
+
+--connection node_1
+
+DROP TABLE t1;

mysql-test/suite/galera/t/galera_drop_database.test

@@ -56,6 +56,7 @@ SHOW TABLES;
 DROP DATABASE fts;
 
 --connection node_2
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:");
 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'fts_t1';
 --source include/wait_condition.inc
 --let $wait_condition = SELECT COUNT(*) = 0 FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_NAME = 'fts_t2';

mysql-test/suite/galera/t/galera_events2.test

@@ -137,6 +137,8 @@ SELECT * FROM t1;
 --echo # node_2 Event should be SERVERSIDE_DISABLED
 SELECT EVENT_CATALOG, EVENT_SCHEMA, EVENT_NAME, DEFINER, EVENT_BODY, EVENT_DEFINITION, EVENT_TYPE, EXECUTE_AT, INTERVAL_VALUE, INTERVAL_FIELD, STATUS,ON_COMPLETION, EVENT_COMMENT FROM INFORMATION_SCHEMA.EVENTS WHERE EVENT_NAME='one_event';
 
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:.*");
+
 --connection node_1
 SELECT * FROM t1;
 --echo # node_1 Event should be ENABLED

mysql-test/suite/galera/t/galera_restart_on_unknown_option.test

@@ -125,6 +125,7 @@ SELECT * FROM t1;
 
 # Sanity check (node 2 is running now and can perform SQL operators):
 
+call mtr.add_suppression("WSREP: Failed to scan the last segment to the end. Last events may be missing. Last recovered event:");
 SELECT * FROM t1;
 
 --connection node_1

mysql-test/suite/sys_vars/r/sysvars_innodb.result

@@ -2,9 +2,7 @@ select VARIABLE_NAME, SESSION_VALUE, DEFAULT_VALUE, VARIABLE_SCOPE, VARIABLE_TYP
 where variable_name like 'innodb%' and
 variable_name not in (
 'innodb_version',                   # always the same as the server version
-'innodb_disallow_writes',           # only available WITH_WSREP
 'innodb_numa_interleave',           # only available WITH_NUMA
-'innodb_sched_priority_cleaner',    # linux only
 'innodb_evict_tables_on_commit_debug', # one may want to override this
 'innodb_use_native_aio',            # default value depends on OS
 'innodb_buffer_pool_load_pages_abort')            # debug build only, and is only for testing

mysql-test/suite/sys_vars/t/sysvars_innodb.test

@@ -9,9 +9,7 @@ select VARIABLE_NAME, SESSION_VALUE, DEFAULT_VALUE, VARIABLE_SCOPE, VARIABLE_TYP
   where variable_name like 'innodb%' and
   variable_name not in (
     'innodb_version',                   # always the same as the server version
-    'innodb_disallow_writes',           # only available WITH_WSREP
     'innodb_numa_interleave',           # only available WITH_NUMA
-    'innodb_sched_priority_cleaner',    # linux only
     'innodb_evict_tables_on_commit_debug', # one may want to override this
     'innodb_use_native_aio',            # default value depends on OS
     'innodb_buffer_pool_load_pages_abort')            # debug build only, and is only for testing