MDEV-24596 : Assertion `state_ == s_exec || state_ == s_quitting' failed in wsrep::client_state::disable_streaming

Jan Lindström · Jan Lindström · commit be5fce16a0d4 · 2021-01-21T11:41:29.000+02:00
There were multiple problems here
* wsrep_trx_fragment_size should not be set when wsrep is disabled or provider is not loaded
* wsrep_trx_fragment_unit should not be set when wsrep is disabled or provider is not loaded
* wsrep_debug has no effect if wsrep is disabled or provider is not loaded
* wsrep_start_position should not be set when wsrep is disabled or provider is not loaded any other value than default
* wsrep_start_position should be changed only when we are joiner or initialized
* wsrep_start_position should be allowed to set only a value that exits, thus
we need to add error handling to wsrep_sst_complete
diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
@@ -23,6 +23,7 @@ galera_binlog_stmt_autoinc : MDEV-19959 Galera test failure on galera_binlog_stm
 galera_encrypt_tmp_files : Get error failed to enable encryption of temporary files
 galera_ftwrl : MDEV-21525 galera.galera_ftwrl
 galera_gcache_recover_manytrx : MDEV-18834 Galera test failure
+galera_ist_mysqldump : MDEV-24463 galera.galera_sst_mysqldump_with_key MTR failed: 'INSERT failed: 1213: Deadlock found when trying to get lock
 galera_kill_largechanges : MDEV-18179 Galera test failure on galera.galera_kill_largechanges
 galera_kill_nochanges : MDEV-18280 Galera test failure on galera_split_brain and galera_kill_nochanges
 galera_many_tables_nopk : MDEV-18182 Galera test failure on galera.galera_many_tables_nopk
@@ -33,6 +34,8 @@ galera_shutdown_nonprim : MDEV-21493 galera.galera_shutdown_nonprim
 galera_split_brain : MDEV-18280 Galera test failure on galera_split_brain and galera_kill_nochanges
 galera_ssl_upgrade : MDEV-19950 Galera test failure on galera_ssl_upgrade
 galera_sst_mariabackup_encrypt_with_key : MDEV-21484 galera_sst_mariabackup_encrypt_with_key
+galera_sst_mysqldump : MDEV-24463 galera.galera_sst_mysqldump_with_key MTR failed: 'INSERT failed: 1213: Deadlock found when trying to get lock
+galera_sst_mysqldump_with_key : MDEV-24463 galera.galera_sst_mysqldump_with_key MTR failed: 'INSERT failed: 1213: Deadlock found when trying to get lock
 galera_toi_ddl_nonconflicting : MDEV-21518 galera.galera_toi_ddl_nonconflicting
 galera_toi_truncate : MDEV-22996 Hang on galera_toi_truncate test case
 galera_var_node_address : MDEV-20485 Galera test failure
diff --git a/mysql-test/suite/galera/r/galera_var_wsrep_start_position.result b/mysql-test/suite/galera/r/galera_var_wsrep_start_position.result
@@ -1,6 +1,9 @@
+connection node_2;
+connection node_1;
 #
 # wsrep_start_position
 #
+CALL mtr.add_suppression("WSREP: SST failed for position .*");
 SET @wsrep_start_position_global_saved = @@global.wsrep_start_position;
 # default
 SELECT @@global.wsrep_start_position;
@@ -15,20 +18,22 @@ SELECT @@global.wsrep_start_position;
 00000000-0000-0000-0000-000000000000:-1
 
 # valid values
-SET @@global.wsrep_start_position='12345678-1234-1234-1234-123456789012:100';
+SET @@global.wsrep_start_position='00000000-0000-0000-0000-000000000000:-1';
 SELECT @@global.wsrep_start_position;
 @@global.wsrep_start_position
-12345678-1234-1234-1234-123456789012:100
+00000000-0000-0000-0000-000000000000:-1
+
 SET @@global.wsrep_start_position='00000000-0000-0000-0000-000000000000:0';
+ERROR 42000: Variable 'wsrep_start_position' can't be set to the value of '00000000-0000-0000-0000-000000000000:0'
 SELECT @@global.wsrep_start_position;
 @@global.wsrep_start_position
-00000000-0000-0000-0000-000000000000:0
-SET @@global.wsrep_start_position='00000000-0000-0000-0000-000000000000:-1';
+00000000-0000-0000-0000-000000000000:-1
+# invalid values
+SET @@global.wsrep_start_position='12345678-1234-1234-1234-123456789012:100';
+ERROR 42000: Variable 'wsrep_start_position' can't be set to the value of '12345678-1234-1234-1234-123456789012:100'
 SELECT @@global.wsrep_start_position;
 @@global.wsrep_start_position
 00000000-0000-0000-0000-000000000000:-1
-
-# invalid values
 SET @@global.wsrep_start_position='00000000-0000-0000-0000-000000000000:-2';
 ERROR 42000: Variable 'wsrep_start_position' can't be set to the value of '00000000-0000-0000-0000-000000000000:-2'
 SELECT @@global.wsrep_start_position;
diff --git a/mysql-test/suite/galera/t/galera_var_wsrep_start_position.test b/mysql-test/suite/galera/t/galera_var_wsrep_start_position.test
@@ -1,11 +1,12 @@
---source include/have_wsrep.inc
+--source include/galera_cluster.inc
 
 --echo #
 --echo # wsrep_start_position
 --echo #
 
-SET @wsrep_start_position_global_saved = @@global.wsrep_start_position;
+CALL mtr.add_suppression("WSREP: SST failed for position .*");
 
+SET @wsrep_start_position_global_saved = @@global.wsrep_start_position;
 --echo # default
 SELECT @@global.wsrep_start_position;
 
@@ -15,19 +16,20 @@ SELECT @@global.wsrep_start_position;
 SELECT @@session.wsrep_start_position;
 SELECT @@global.wsrep_start_position;
 
-
 --echo
 --echo # valid values
-SET @@global.wsrep_start_position='12345678-1234-1234-1234-123456789012:100';
-SELECT @@global.wsrep_start_position;
-SET @@global.wsrep_start_position='00000000-0000-0000-0000-000000000000:0';
-SELECT @@global.wsrep_start_position;
 SET @@global.wsrep_start_position='00000000-0000-0000-0000-000000000000:-1';
 SELECT @@global.wsrep_start_position;
 
 --echo
+--error ER_WRONG_VALUE_FOR_VAR
+SET @@global.wsrep_start_position='00000000-0000-0000-0000-000000000000:0';
+SELECT @@global.wsrep_start_position;
 --echo # invalid values
 --error ER_WRONG_VALUE_FOR_VAR
+SET @@global.wsrep_start_position='12345678-1234-1234-1234-123456789012:100';
+SELECT @@global.wsrep_start_position;
+--error ER_WRONG_VALUE_FOR_VAR
 SET @@global.wsrep_start_position='00000000-0000-0000-0000-000000000000:-2';
 SELECT @@global.wsrep_start_position;
 --error ER_WRONG_VALUE_FOR_VAR
diff --git a/mysql-test/suite/sys_vars/r/wsrep_debug_basic.result b/mysql-test/suite/sys_vars/r/wsrep_debug_basic.result
@@ -16,19 +16,23 @@ SELECT @@global.wsrep_debug;
 @@global.wsrep_debug
 NONE
 SET @@global.wsrep_debug=1;
+Warnings:
+Warning	1231	Setting 'wsrep_debug' has no effect because wsrep is switched off
 SELECT @@global.wsrep_debug;
 @@global.wsrep_debug
-SERVER
+NONE
 
 # valid values
 SET @@global.wsrep_debug=NONE;
 SELECT @@global.wsrep_debug;
 @@global.wsrep_debug
 NONE
 SET @@global.wsrep_debug=SERVER;
+Warnings:
+Warning	1231	Setting 'wsrep_debug' has no effect because wsrep is switched off
 SELECT @@global.wsrep_debug;
 @@global.wsrep_debug
-SERVER
+NONE
 SET @@global.wsrep_debug=default;
 SELECT @@global.wsrep_debug;
 @@global.wsrep_debug
diff --git a/mysql-test/suite/wsrep/r/wsrep_variables_no_provider.result b/mysql-test/suite/wsrep/r/wsrep_variables_no_provider.result
@@ -0,0 +1,44 @@
+SELECT @@wsrep_on;
+@@wsrep_on
+1
+SET @wsrep_slave_threads_global_saved = @@global.wsrep_slave_threads;
+SET @wsrep_debug_saved = @@global.wsrep_debug;
+SET @wsrep_provider_options_saved= @@global.wsrep_provider_options;
+SET @wsrep_cluster_address_saved= @@global.wsrep_cluster_address;
+SET GLOBAL wsrep_provider=none;
+SET SESSION wsrep_trx_fragment_size=DEFAULT;
+ERROR HY000: Incorrect arguments to SET
+SELECT @@session.wsrep_trx_fragment_size;
+@@session.wsrep_trx_fragment_size
+0
+SET GLOBAL wsrep_start_position='12345678-1234-1234-1234-123456789012:100';
+ERROR 42000: Variable 'wsrep_start_position' can't be set to the value of '12345678-1234-1234-1234-123456789012:100'
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1231	Cannot set 'wsrep_start_position' because wsrep is switched off or provider is not loaded
+Error	1231	Variable 'wsrep_start_position' can't be set to the value of '12345678-1234-1234-1234-123456789012:100'
+SELECT @@global.wsrep_start_position;
+@@global.wsrep_start_position
+00000000-0000-0000-0000-000000000000:-1
+SET GLOBAL wsrep_debug=1;
+Warnings:
+Warning	1231	Setting 'wsrep_debug' has no effect because wsrep is switched off
+SELECT @@global.wsrep_debug;
+@@global.wsrep_debug
+NONE
+SET GLOBAL wsrep_slave_threads=5;
+SELECT @@global.wsrep_slave_threads;
+@@global.wsrep_slave_threads
+5
+SET GLOBAL wsrep_desync=1;
+ERROR HY000: WSREP (galera) not started
+SELECT @@global.wsrep_desync;
+@@global.wsrep_desync
+0
+SET SESSION wsrep_trx_fragment_unit='rows';
+ERROR HY000: Incorrect arguments to SET
+SELECT @@session.wsrep_trx_fragment_unit;
+@@session.wsrep_trx_fragment_unit
+rows
+SET @@global.wsrep_slave_threads = @wsrep_slave_threads_global_saved;
+SET @@global.wsrep_debug = @wsrep_debug_saved;
diff --git a/mysql-test/suite/wsrep/r/wsrep_variables_wsrep_off.result b/mysql-test/suite/wsrep/r/wsrep_variables_wsrep_off.result
@@ -0,0 +1,39 @@
+SELECT @@wsrep_on;
+@@wsrep_on
+0
+SET @wsrep_slave_threads_global_saved = @@global.wsrep_slave_threads;
+SET @wsrep_debug_saved = @@global.wsrep_debug;
+SET SESSION wsrep_trx_fragment_size=DEFAULT;
+ERROR HY000: Incorrect arguments to SET
+SELECT @@session.wsrep_trx_fragment_size;
+@@session.wsrep_trx_fragment_size
+0
+SET GLOBAL wsrep_start_position='12345678-1234-1234-1234-123456789012:100';
+ERROR 42000: Variable 'wsrep_start_position' can't be set to the value of '12345678-1234-1234-1234-123456789012:100'
+SHOW WARNINGS;
+Level	Code	Message
+Warning	1231	Cannot set 'wsrep_start_position' because wsrep is switched off or provider is not loaded
+Error	1231	Variable 'wsrep_start_position' can't be set to the value of '12345678-1234-1234-1234-123456789012:100'
+SELECT @@global.wsrep_start_position;
+@@global.wsrep_start_position
+00000000-0000-0000-0000-000000000000:-1
+SET GLOBAL wsrep_debug=1;
+Warnings:
+Warning	1231	Setting 'wsrep_debug' has no effect because wsrep is switched off
+SELECT @@global.wsrep_debug;
+@@global.wsrep_debug
+NONE
+SET GLOBAL wsrep_slave_threads=5;
+SELECT @@global.wsrep_slave_threads;
+@@global.wsrep_slave_threads
+5
+SET GLOBAL wsrep_desync=1;
+ERROR HY000: WSREP (galera) not started
+SELECT @@global.wsrep_desync;
+@@global.wsrep_desync
+0
+SET SESSION wsrep_trx_fragment_unit='rows';
+ERROR HY000: Incorrect arguments to SET
+SELECT @@session.wsrep_trx_fragment_unit;
+@@session.wsrep_trx_fragment_unit
+rows
diff --git a/mysql-test/suite/wsrep/t/wsrep_variables_no_provider.cnf b/mysql-test/suite/wsrep/t/wsrep_variables_no_provider.cnf
@@ -0,0 +1,12 @@
+# Use default setting for mysqld processes
+!include include/default_mysqld.cnf
+
+[mysqld.1]
+wsrep-on=ON
+binlog-format=ROW
+wsrep-provider=@ENV.WSREP_PROVIDER
+wsrep-cluster-address='gcomm://'
+#galera_port=@OPT.port
+#ist_port=@OPT.port
+#sst_port=@OPT.port
+
diff --git a/mysql-test/suite/wsrep/t/wsrep_variables_no_provider.test b/mysql-test/suite/wsrep/t/wsrep_variables_no_provider.test
@@ -0,0 +1,38 @@
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+SELECT @@wsrep_on;
+
+SET @wsrep_slave_threads_global_saved = @@global.wsrep_slave_threads;
+SET @wsrep_debug_saved = @@global.wsrep_debug;
+SET @wsrep_provider_options_saved= @@global.wsrep_provider_options;
+SET @wsrep_cluster_address_saved= @@global.wsrep_cluster_address;
+
+SET GLOBAL wsrep_provider=none;
+
+--error ER_WRONG_ARGUMENTS
+SET SESSION wsrep_trx_fragment_size=DEFAULT;
+SELECT @@session.wsrep_trx_fragment_size;
+--error ER_WRONG_VALUE_FOR_VAR
+SET GLOBAL wsrep_start_position='12345678-1234-1234-1234-123456789012:100';
+SHOW WARNINGS;
+SELECT @@global.wsrep_start_position;
+SET GLOBAL wsrep_debug=1;
+SELECT @@global.wsrep_debug;
+SET GLOBAL wsrep_slave_threads=5;
+SELECT @@global.wsrep_slave_threads;
+--error ER_WRONG_ARGUMENTS
+SET GLOBAL wsrep_desync=1;
+SELECT @@global.wsrep_desync;
+--error ER_WRONG_ARGUMENTS
+SET SESSION wsrep_trx_fragment_unit='rows';
+SELECT @@session.wsrep_trx_fragment_unit;
+
+--disable_query_log
+eval SET GLOBAL wsrep_provider= '$WSREP_PROVIDER';
+SET GLOBAL wsrep_cluster_address= @wsrep_cluster_address_saved;
+SET GLOBAL wsrep_provider_options= @wsrep_provider_options_saved;
+--source include/galera_wait_ready.inc
+SET @@global.wsrep_slave_threads = @wsrep_slave_threads_global_saved;
+SET @@global.wsrep_debug = @wsrep_debug_saved;
+--enable_query_log
diff --git a/mysql-test/suite/wsrep/t/wsrep_variables_wsrep_off.cnf b/mysql-test/suite/wsrep/t/wsrep_variables_wsrep_off.cnf
@@ -0,0 +1,12 @@
+# Use default setting for mysqld processes
+!include include/default_mysqld.cnf
+
+[mysqld]
+wsrep-on=OFF
+
+[mysqld.1]
+wsrep-on=OFF
+#galera_port=@OPT.port
+#ist_port=@OPT.port
+#sst_port=@OPT.port
+
diff --git a/mysql-test/suite/wsrep/t/wsrep_variables_wsrep_off.test b/mysql-test/suite/wsrep/t/wsrep_variables_wsrep_off.test
@@ -0,0 +1,30 @@
+--source include/have_wsrep.inc
+--source include/have_innodb.inc
+
+SELECT @@wsrep_on;
+
+SET @wsrep_slave_threads_global_saved = @@global.wsrep_slave_threads;
+SET @wsrep_debug_saved = @@global.wsrep_debug;
+
+--error ER_WRONG_ARGUMENTS
+SET SESSION wsrep_trx_fragment_size=DEFAULT;
+SELECT @@session.wsrep_trx_fragment_size;
+--error ER_WRONG_VALUE_FOR_VAR
+SET GLOBAL wsrep_start_position='12345678-1234-1234-1234-123456789012:100';
+SHOW WARNINGS;
+SELECT @@global.wsrep_start_position;
+SET GLOBAL wsrep_debug=1;
+SELECT @@global.wsrep_debug;
+SET GLOBAL wsrep_slave_threads=5;
+SELECT @@global.wsrep_slave_threads;
+--error ER_WRONG_ARGUMENTS
+SET GLOBAL wsrep_desync=1;
+SELECT @@global.wsrep_desync;
+--error ER_WRONG_ARGUMENTS
+SET SESSION wsrep_trx_fragment_unit='rows';
+SELECT @@session.wsrep_trx_fragment_unit;
+
+--disable_query_log
+SET @@global.wsrep_slave_threads = @wsrep_slave_threads_global_saved;
+SET @@global.wsrep_debug = @wsrep_debug_saved;
+--enable_query_log
diff --git a/sql/wsrep_priv.h b/sql/wsrep_priv.h
@@ -42,7 +42,7 @@ extern wsrep_seqno_t local_seqno;
 extern Wsrep_schema* wsrep_schema;
 
 // a helper function
-void wsrep_sst_received(THD*, const wsrep_uuid_t&, wsrep_seqno_t,
+bool wsrep_sst_received(THD*, const wsrep_uuid_t&, wsrep_seqno_t,
                         const void*, size_t);
 
 void wsrep_notify_status(enum wsrep::server_state::state status,
diff --git a/sql/wsrep_sst.cc b/sql/wsrep_sst.cc
@@ -308,12 +308,37 @@ bool wsrep_before_SE()
 }
 
 // Signal end of SST
-static void wsrep_sst_complete (THD*                thd,
-                                int const           rcode)
+static bool wsrep_sst_complete (THD*                thd,
+                                int const           rcode,
+                                wsrep::gtid const   sst_gtid)
 {
   Wsrep_client_service client_service(thd, thd->wsrep_cs());
-  Wsrep_server_state::instance().sst_received(client_service, rcode);
+  Wsrep_server_state& server_state= Wsrep_server_state::instance();
+  enum wsrep::server_state::state state= server_state.state();
+  bool failed= false;
+
+  // Do not call sst_received if we are not in joiner or
+  // initialized state on server. This is because it
+  // assumes we are on those states. Give error if we are
+  // in incorrect state.
+  if ((state == Wsrep_server_state::s_joiner ||
+       state == Wsrep_server_state::s_initialized))
+    Wsrep_server_state::instance().sst_received(client_service,
+       rcode);
+  else
+  {
+    char start_pos_buf[FN_REFLEN];
+    ssize_t len= wsrep::print_to_c_str(sst_gtid, start_pos_buf, FN_REFLEN-1);
+    start_pos_buf[len]='\0';
+    WSREP_ERROR("SST failed for position %s initialized %d server_state %s",
+                start_pos_buf,
+                server_state.is_initialized(),
+                wsrep::to_c_string(state));
+    failed= true;
+  }
+
   wsrep_joiner_monitor_end();
+  return failed;
 }
 
   /*
@@ -325,13 +350,15 @@ static void wsrep_sst_complete (THD*                thd,
   @param seqno     [IN]               Initial state sequence number
   @param state     [IN]               Always NULL, also ignored by wsrep provider (?)
   @param state_len [IN]               Always 0, also ignored by wsrep provider (?)
+  @return true when successful, false if error
 */
-void wsrep_sst_received (THD*                thd,
+bool wsrep_sst_received (THD*                thd,
                          const wsrep_uuid_t& uuid,
                          wsrep_seqno_t const seqno,
                          const void* const   state,
                          size_t const        state_len)
 {
+  bool error= false;
   /*
     To keep track of whether the local uuid:seqno should be updated. Also, note
     that local state (uuid:seqno) is updated/checkpointed only after we get an
@@ -371,8 +398,10 @@ void wsrep_sst_received (THD*                thd,
     if (WSREP_ON)
     {
       int const rcode(seqno < 0 ? seqno : 0);
-      wsrep_sst_complete(thd,rcode);
+      error= wsrep_sst_complete(thd,rcode, sst_gtid);
     }
+
+    return error;
 }
 
 static int sst_scan_uuid_seqno (const char* str,
@@ -653,7 +682,7 @@ static void* sst_joiner_thread (void* a)
     /* Read committed isolation to avoid gap locking */
     thd->variables.tx_isolation= ISO_READ_COMMITTED;
 
-    wsrep_sst_complete (thd, -err);
+    wsrep_sst_complete (thd, -err, ret_gtid);
 
     delete thd;
     my_thread_end();
diff --git a/sql/wsrep_var.cc b/sql/wsrep_var.cc
