MDEV-24097: galera[_3nodes] suite tests in MTR sporadically fails

This is the first part of the fixes for MDEV-24097. This commit
contains the fixes for instability when testing Galera and when
restarting nodes quickly:

1) Protection against a "stuck" old SST process during the execution
   of the new SST (after restarting the node) is now implemented for
   mariabackup / xtrabackup, which should help to avoid almost all
   conflicts due to the use of the same ports - both during testing
   with mtr, so and when restarting nodes quickly in a production
   environment.
2) Added more protection to scripts against unexpected return of
   the rc != 0 (in the commands for deleting temporary files, etc).
3) Added protection against unexpected crashes during binlog transfer
   (in SST scripts for rsync).
4) Spaces and some special characters in binlog filenames shouldn't
   be a problem now (at the script level).
5) Daemon process termination tracking has been made more robust
   against crashes due to unexpected termination of the previous SST
   process while new scripts are running.
6) Reading ssl encryption parameters has been moved from specific
   SST scripts to a common wsrep_sst_common.sh script, which allows
   unified error handling, unified diagnostics and simplifies script
   revisions in the future.
7) Improved diagnostics of errors related to the use of openssl.
8) Corrections have been made for xtrabackup-v2 (both in tests and in
   the script code) that restore the work of xtrabackup with updated
   versions of innodb.
9) Fixed some tests for galera_3nodes, although the complete solution
   for the problem of starting three nodes at the same time on fast
   machines will be done in a separate commit.

No additional tests are required as this commit fixes problems with
existing tests.

mysql-test/suite/galera/disabled.def

@@ -28,5 +28,3 @@ query_cache: MDEV-15805 Test failure on galera.query_cache
 versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest: Result content mismatch
 galera_wsrep_provider_unset_set: wsrep_provider is read-only for security reasons
 pxc-421: wsrep_provider is read-only for security reasons
-galera_sst_xtrabackup-v2: Test fails due to innodb issues
-galera_sst_xtrabackup-v2_data_dir: Test fails due to innodb issues

mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result

@@ -1,4 +1,7 @@
+connection node_1;
+connection node_2;
 Performing State Transfer on a server that has been shut down cleanly and restarted
+connection node_1;
 CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
@@ -8,6 +11,7 @@ INSERT INTO t1 VALUES ('node1_committed_before');
 INSERT INTO t1 VALUES ('node1_committed_before');
 INSERT INTO t1 VALUES ('node1_committed_before');
 COMMIT;
+connection node_2;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node2_committed_before');
@@ -17,6 +21,7 @@ INSERT INTO t1 VALUES ('node2_committed_before');
 INSERT INTO t1 VALUES ('node2_committed_before');
 COMMIT;
 Shutting down server ...
+connection node_1;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node1_committed_during');
@@ -31,13 +36,15 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+connect node_1a_galera_st_shutdown_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+connection node_2;
 Starting server ...
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
@@ -47,6 +54,7 @@ INSERT INTO t1 VALUES ('node2_committed_after');
 INSERT INTO t1 VALUES ('node2_committed_after');
 INSERT INTO t1 VALUES ('node2_committed_after');
 COMMIT;
+connection node_1;
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
@@ -61,6 +69,7 @@ INSERT INTO t1 VALUES ('node1_committed_after');
 INSERT INTO t1 VALUES ('node1_committed_after');
 INSERT INTO t1 VALUES ('node1_committed_after');
 COMMIT;
+connection node_1a_galera_st_shutdown_slave;
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
@@ -75,6 +84,7 @@ COUNT(*) = 0
 1
 COMMIT;
 SET AUTOCOMMIT=ON;
+connection node_1;
 SELECT COUNT(*) = 35 FROM t1;
 COUNT(*) = 35
 1
@@ -86,6 +96,7 @@ COMMIT;
 SET AUTOCOMMIT=ON;
 Performing State Transfer on a server that starts from a clean var directory
 This is accomplished by shutting down node #2 and removing its var directory before restarting it
+connection node_1;
 CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
@@ -95,6 +106,7 @@ INSERT INTO t1 VALUES ('node1_committed_before');
 INSERT INTO t1 VALUES ('node1_committed_before');
 INSERT INTO t1 VALUES ('node1_committed_before');
 COMMIT;
+connection node_2;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node2_committed_before');
@@ -104,6 +116,7 @@ INSERT INTO t1 VALUES ('node2_committed_before');
 INSERT INTO t1 VALUES ('node2_committed_before');
 COMMIT;
 Shutting down server ...
+connection node_1;
 Cleaning var directory ...
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
@@ -119,13 +132,15 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+connect node_1a_galera_st_clean_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+connection node_2;
 Starting server ...
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
@@ -135,6 +150,7 @@ INSERT INTO t1 VALUES ('node2_committed_after');
 INSERT INTO t1 VALUES ('node2_committed_after');
 INSERT INTO t1 VALUES ('node2_committed_after');
 COMMIT;
+connection node_1;
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
@@ -149,6 +165,7 @@ INSERT INTO t1 VALUES ('node1_committed_after');
 INSERT INTO t1 VALUES ('node1_committed_after');
 INSERT INTO t1 VALUES ('node1_committed_after');
 COMMIT;
+connection node_1a_galera_st_clean_slave;
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
@@ -163,6 +180,7 @@ COUNT(*) = 0
 1
 COMMIT;
 SET AUTOCOMMIT=ON;
+connection node_1;
 SELECT COUNT(*) = 35 FROM t1;
 COUNT(*) = 35
 1
@@ -173,6 +191,7 @@ DROP TABLE t1;
 COMMIT;
 SET AUTOCOMMIT=ON;
 Performing State Transfer on a server that has been killed and restarted
+connection node_1;
 CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
@@ -182,6 +201,7 @@ INSERT INTO t1 VALUES ('node1_committed_before');
 INSERT INTO t1 VALUES ('node1_committed_before');
 INSERT INTO t1 VALUES ('node1_committed_before');
 COMMIT;
+connection node_2;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node2_committed_before');
@@ -191,6 +211,7 @@ INSERT INTO t1 VALUES ('node2_committed_before');
 INSERT INTO t1 VALUES ('node2_committed_before');
 COMMIT;
 Killing server ...
+connection node_1;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node1_committed_during');
@@ -205,13 +226,15 @@ INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
+connect node_1a_galera_st_kill_slave, 127.0.0.1, root, , test, $NODE_MYPORT_1;
 SET AUTOCOMMIT=OFF;
 START TRANSACTION;
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
+connection node_2;
 Performing --wsrep-recover ...
 Starting server ...
 Using --wsrep-start-position when starting mysqld ...
@@ -223,6 +246,7 @@ INSERT INTO t1 VALUES ('node2_committed_after');
 INSERT INTO t1 VALUES ('node2_committed_after');
 INSERT INTO t1 VALUES ('node2_committed_after');
 COMMIT;
+connection node_1;
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
 INSERT INTO t1 VALUES ('node1_to_be_committed_after');
@@ -237,6 +261,7 @@ INSERT INTO t1 VALUES ('node1_committed_after');
 INSERT INTO t1 VALUES ('node1_committed_after');
 INSERT INTO t1 VALUES ('node1_committed_after');
 COMMIT;
+connection node_1a_galera_st_kill_slave;
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
 INSERT INTO t1 VALUES ('node1_to_be_rollbacked_after');
@@ -251,6 +276,7 @@ COUNT(*) = 0
 1
 COMMIT;
 SET AUTOCOMMIT=ON;
+connection node_1;
 SELECT COUNT(*) = 35 FROM t1;
 COUNT(*) = 35
 1

mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf

@@ -4,6 +4,7 @@
 wsrep_sst_method=xtrabackup-v2
 wsrep_sst_auth="root:"
 innodb_safe_truncate=OFF
+# innodb_undo_logs=1
 
 [mysqld.1]
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf

@@ -4,6 +4,7 @@
 wsrep_sst_method=xtrabackup-v2
 wsrep_sst_auth=root:
 innodb_safe_truncate=OFF
+innodb_undo_logs=1
 
 [mysqld.1]
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_sst_xtrabackup-v2-options.cnf

@@ -5,6 +5,7 @@ wsrep_sst_method=xtrabackup-v2
 wsrep_sst_auth="root:"
 wsrep_debug=ON
 innodb_safe_truncate=OFF
+innodb_undo_logs=1
 
 [xtrabackup]
 backup-locks

mysql-test/suite/galera/t/galera_sst_xtrabackup-v2.cnf

@@ -5,6 +5,7 @@ wsrep_sst_method=xtrabackup-v2
 wsrep_sst_auth="root:"
 wsrep_debug=ON
 innodb_safe_truncate=OFF
+innodb_undo_logs=1
 
 [mysqld.1]
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_data_dir.cnf

@@ -5,6 +5,7 @@ wsrep_sst_method=xtrabackup-v2
 wsrep_sst_auth="root:"
 wsrep_debug=ON
 innodb_safe_truncate=OFF
+innodb_undo_logs=1
 
 [mysqld.1]
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_sst_xtrabackup-v2_encrypt_with_key.cnf

@@ -5,6 +5,7 @@ wsrep_sst_method=xtrabackup-v2
 wsrep_sst_auth="root:"
 wsrep_debug=ON
 innodb_safe_truncate=OFF
+innodb_undo_logs=1
 
 [SST]
 tkey=@ENV.MYSQL_TEST_DIR/std_data/cakey.pem

mysql-test/suite/galera_3nodes/r/galera_ist_gcache_rollover.result

@@ -6,10 +6,10 @@ CREATE TABLE t1 (f1 INTEGER PRIMARY KEY);
 INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 connection node_2;
 Unloading wsrep provider ...
-SET GLOBAL wsrep_provider = 'none';
+SET GLOBAL wsrep_cluster_address = '';
 connection node_3;
 Unloading wsrep provider ...
-SET GLOBAL wsrep_provider = 'none';
+SET GLOBAL wsrep_cluster_address = '';
 connection node_1;
 INSERT INTO t1 VALUES (11), (12), (13), (14), (15);
 INSERT INTO t1 VALUES (21), (22), (23), (24), (25);

mysql-test/suite/galera_3nodes/t/galera_ist_gcache_rollover.test

@@ -29,9 +29,11 @@ INSERT INTO t1 VALUES (01), (02), (03), (04), (05);
 
 # Disconnect nodes #2 and #3
 --connection node_2
+--let $wsrep_cluster_address_orig2 = `select @@wsrep_cluster_address`
 --source suite/galera/include/galera_stop_replication.inc
 
 --connection node_3
+--let $wsrep_cluster_address_orig3 = `select @@wsrep_cluster_address`
 --source suite/galera/include/galera_stop_replication.inc
 
 --connection node_1
@@ -51,17 +53,17 @@ INSERT INTO t1 VALUES (21), (22), (23), (24), (25);
 # ... and restart providers to force IST
 --connection node_2
 --disable_query_log
---eval SET GLOBAL wsrep_provider = '$wsrep_provider_orig';
---eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig';
+SET GLOBAL wsrep_cluster_address='';
+--eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig2';
 --enable_query_log
 
 --connection node_1
 INSERT INTO t1 VALUES (31), (32), (33), (34), (35);
 
 --connection node_3
 --disable_query_log
---eval SET GLOBAL wsrep_provider = '$wsrep_provider_orig';
---eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig';
+SET GLOBAL wsrep_cluster_address='';
+--eval SET GLOBAL wsrep_cluster_address = '$wsrep_cluster_address_orig3';
 --enable_query_log
 
 --connection node_1