MDEV-25719: stunnel uses "verifyChain" without subject checks

sysprg · sysprg · commit b01a9fd817d9 · 2021-05-21T03:13:37.000+02:00
Another batch of changes that should make the SST process
more reliable in all scenarios:

 1) Added hostname or CN verification when stunnel is used
    with certificate chain verification (verifyChain = yes);
 2) Added check for the absence of the stunnel utility for
    mtr tests;
 3) Deletion of working files before and after SST is done
    more accurately;
 4) rsync on joiner can be run even if the path to its
    configuration file contains spaces;
 5) More accurate directory creation (for data files and
    for logs);
 6) IST with mysqldump no longer turns off statement logging;
 7) Reset password for mysqldump when password is empty but
    username is specified;
 8) More reliable quoting when generating statements in
    wsrep_sst_mysqldump;
 9) Added explicit generation of 2048-bit Diffie-Hellman
    parameters for sockat &lt; 1.7.3, by analogy with xtrabackup;
10) Compression parameters for qpress are read from all
    suitable server groups in configuration file, as well as
    from the [sst] and [xtrabackup] groups;
11) Added a test that checks compression using qpress;
12) Checking for optional utilities is modified to work even
    if they implemented as built-in shell commands (unlikely
    on real systems, but more reliable).
diff --git a/mysql-test/suite.pm b/mysql-test/suite.pm
@@ -55,7 +55,7 @@ sub skip_combinations {
             unless ::have_mariabackup();
 
   $skip{'include/have_mariabackup.inc'} = 'Need socket statistics utility'
-            unless IS_WINDOWS || ::which("ss");
+            unless IS_WINDOWS || ! ::have_wsrep() || ::which("lsof") || ::which("sockstat") || ::which("ss");
 
   $skip{'include/have_mariabackup.inc'} = 'Need socat or nc'
             unless IS_WINDOWS || $ENV{MTR_GALERA_TFMT};
diff --git a/mysql-test/suite/galera/disabled.def b/mysql-test/suite/galera/disabled.def
@@ -21,10 +21,10 @@ galera_parallel_simple : MDEV-20318 galera.galera_parallel_simple fails
 galera_partition : MDEV-21806: galera.galera_partition MTR failed: failed to recover from DONOR state
 galera_shutdown_nonprim : MDEV-21493 galera.galera_shutdown_nonprim
 galera_var_node_address : MDEV-20485 Galera test failure
-galera_wan : MDEV-17259 Test failure on galera.galera_wan
+#galera_wan : MDEV-17259 Test failure on galera.galera_wan
 partition : MDEV-19958 Galera test failure on galera.partition
 query_cache: MDEV-15805 Test failure on galera.query_cache
-sql_log_bin : MDEV-21491 galera.sql_log_bin
+#sql_log_bin : MDEV-21491 galera.sql_log_bin
 versioning_trx_id: MDEV-18590: galera.versioning_trx_id: Test failure: mysqltest: Result content mismatch
 galera_wsrep_provider_unset_set: wsrep_provider is read-only for security reasons
 pxc-421: wsrep_provider is read-only for security reasons
diff --git a/mysql-test/suite/galera/include/have_qpress.inc b/mysql-test/suite/galera/include/have_qpress.inc
@@ -0,0 +1,4 @@
+#
+# suite.pm will make sure that all tests including this file
+# will be skipped as needed
+#
diff --git a/mysql-test/suite/galera/include/have_stunnel.inc b/mysql-test/suite/galera/include/have_stunnel.inc
@@ -0,0 +1,4 @@
+#
+# suite.pm will make sure that all tests including this file
+# will be skipped as needed
+#
diff --git a/mysql-test/suite/galera/r/galera_sst_mariabackup_qpress.result b/mysql-test/suite/galera/r/galera_sst_mariabackup_qpress.result
@@ -0,0 +1,4 @@
+SELECT 1;
+1
+1
+include/assert_grep.inc [Compressed qpress files found]
diff --git a/mysql-test/suite/galera/suite.pm b/mysql-test/suite/galera/suite.pm
@@ -69,4 +69,13 @@ push @::global_suppressions,
      qr|WSREP: Trying to continue unpaused monitor|,
    );
 
+sub skip_combinations {
+  my %skip = ();
+  $skip{'include/have_stunnel.inc'} = "Need 'stunnel' utility"
+            unless ::which("stunnel");
+  $skip{'include/have_qpress.inc'} = "Need 'qpress' utility"
+            unless ::which("qpress");
+  %skip;
+}
+
 bless { };
diff --git a/mysql-test/suite/galera/t/galera_ist_rsync.cnf b/mysql-test/suite/galera/t/galera_ist_rsync.cnf
@@ -10,4 +10,3 @@ wsrep_sync_wait=1
 [mysqld.2]
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
 wsrep_sync_wait=1
-
diff --git a/mysql-test/suite/galera/t/galera_log_bin.cnf b/mysql-test/suite/galera/t/galera_log_bin.cnf
@@ -7,4 +7,3 @@ log-slave-updates
 [mysqld.2]
 log-bin
 log-slave-updates
-
diff --git a/mysql-test/suite/galera/t/galera_log_bin.inc b/mysql-test/suite/galera/t/galera_log_bin.inc
@@ -39,4 +39,3 @@ DROP TABLE t2;
 --echo #cleanup
 --connection node_1
 RESET MASTER;
-
diff --git a/mysql-test/suite/galera/t/galera_log_bin_opt.cnf b/mysql-test/suite/galera/t/galera_log_bin_opt.cnf
@@ -12,4 +12,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore
 
 [sst]
 transferfmt=@ENV.MTR_GALERA_TFMT
-streamfmt=xbstream
+streamfmt=mbstream
diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf
@@ -0,0 +1,12 @@
+!include ../galera_2nodes.cnf
+
+[mysqld]
+wsrep_sst_method=mariabackup
+wsrep_sst_auth="root:"
+wsrep_debug=ON
+
+[sst]
+transferfmt=@ENV.MTR_GALERA_TFMT
+compress=quicklz
+compress-threads=2
+compress-chunk-size=32768
diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.test b/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.test
@@ -0,0 +1,24 @@
+#
+# This test checks that qpress compression works with mariabackup
+# Initial SST happens via mariabackup, so there is not much to do in the body
+# of the test
+#
+
+--source include/big_test.inc
+--source include/galera_cluster.inc
+--source include/have_innodb.inc
+--source include/have_mariabackup.inc
+--source include/have_qpress.inc
+
+SELECT 1;
+
+--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
+--source include/wait_condition.inc
+
+# Confirm that transfer was compressed using qpress algorithm
+--let $assert_text = Compressed qpress files found
+--let $assert_select = Compressed qpress files found
+--let $assert_count = 1
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.2.err
+--let $assert_only_after = CURRENT_TEST
+--source include/assert_grep.inc
diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_key.test
@@ -1,6 +1,7 @@
 --source include/big_test.inc
 --source include/galera_cluster.inc
 --source include/have_debug.inc
+--source include/have_stunnel.inc
 
 # Save original auto_increment_offset values.
 --let $node_1=node_1
diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test b/mysql-test/suite/galera/t/galera_sst_rsync_encrypt_with_server.test
@@ -1,6 +1,7 @@
 --source include/big_test.inc
 --source include/galera_cluster.inc
 --source include/have_debug.inc
+--source include/have_stunnel.inc
 
 # Save original auto_increment_offset values.
 --let $node_1=node_1
diff --git a/mysql-test/suite/galera/t/galera_wan_restart_sst.test b/mysql-test/suite/galera/t/galera_wan_restart_sst.test
@@ -4,7 +4,7 @@
 # We can not easily restart the first node, so instead we restart all the other nodes. MTR does not allow multiple nodes
 # to be down at the same time, so restarts are sequential.
 #
-# We can not test any of the actual WAN optimizations from inside MTR and no 
+# We can not test any of the actual WAN optimizations from inside MTR and no
 # status variables are provided. So we only check that simple replication works.
 #
 
@@ -120,7 +120,6 @@ INSERT INTO t1 VALUES (33);
 
 INSERT INTO t1 VALUES (341);
 
-
 #
 # Check all nodes
 #
diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh
@@ -1,5 +1,5 @@
-# Copyright (C) 2012-2015 Codership Oy
 # Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2012-2015 Codership Oy
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -822,14 +822,15 @@ wsrep_log_info()
 
 wsrep_cleanup_progress_file()
 {
-    [ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null || true
+    [ -n "$SST_PROGRESS_FILE" -a \
+      -f "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null || true
 }
 
 wsrep_check_program()
 {
     local prog="$1"
     local cmd=$(command -v "$prog")
-    if [ ! -x "$cmd" ]; then
+    if [ -z "$cmd" ]; then
         echo "'$prog' not found in PATH"
         return 2 # no such file or directory
     fi
@@ -865,9 +866,9 @@ get_openssl()
     fi
     # Let's look for openssl:
     OPENSSL_BINARY="$(command -v openssl)"
-    if [ ! -x "$OPENSSL_BINARY" ]; then
+    if [ -z "$OPENSSL_BINARY" ]; then
         OPENSSL_BINARY='/usr/bin/openssl'
-        if [ ! -x "$OPENSSL_BINARY" ]; then
+        if [ -z "$OPENSSL_BINARY" ]; then
             OPENSSL_BINARY=""
         fi
     fi
@@ -899,14 +900,14 @@ is_local_ip()
     [ "$1" = "$(hostname -d)" ] && return 0
 
     local ip_util="$(command -v ip)"
-    if [ -x "$ip_util" ]; then
+    if [ -n "$ip_util" ]; then
         # ip address show ouput format is " inet[6] <address>/<mask>":
         "$ip_util" address show \
              | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \
              | grep -F " $1/" >/dev/null && return 0
     else
         local ifconfig_util="$(command -v ifconfig)"
-        if [ -x "$ifconfig_util" ]; then
+        if [ -n "$ifconfig_util" ]; then
             # ifconfig output format is " inet[6] <address> ...":
             "$ifconfig_util" \
                  | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \
@@ -923,16 +924,79 @@ check_sockets_utils()
     sockstat_available=0
     ss_available=0
 
-    [ -x "$(command -v lsof)" ] && lsof_available=1
-    [ -x "$(command -v sockstat)" ] && sockstat_available=1
-    [ -x "$(command -v ss)" ] && ss_available=1
+    [ -n "$(command -v lsof)" ] && lsof_available=1
+    [ -n "$(command -v sockstat)" ] && sockstat_available=1
+    [ -n "$(command -v ss)" ] && ss_available=1
 
     if [ $lsof_available -eq 0 -a \
          $sockstat_available -eq 0 -a \
          $ss_available -eq 0 ]
     then
-        wsrep_log_error "Neither lsof tool, nor ss or sockstat was found in " \
-                        "the PATH! Make sure you have it installed."
+        wsrep_log_error "Neither lsof, nor sockstat or ss tool was found in " \
+                        "the PATH. Make sure you have it installed."
         exit 2 # ENOENT
     fi
 }
+
+#
+# If the ssl_dhparams variable is already set, uses that as a source
+# of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in
+# the datadir, and creates it there if it can't find the file.
+#
+check_for_dhparams()
+{
+    if [ -z "$ssl_dhparams" ]; then
+        ssl_dhparams="$DATA/dhparams.pem"
+        if [ ! -r "$ssl_dhparams" ]; then
+            get_openssl
+            if [ -n "$OPENSSL_BINARY" ]; then
+                wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams"
+                if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1
+                then
+                    wsrep_log_error "******** ERROR *****************************************"
+                    wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *"
+                    wsrep_log_error "********************************************************"
+                    ssl_dhparams=""
+                fi
+            else
+                # Rollback: if openssl is not installed, then use
+                # the default parameters:
+                ssl_dhparams=""
+            fi
+        fi
+    fi
+}
+
+#
+# Compares two version strings.
+# The first parameter is the version to be checked;
+# The second parameter is the minimum version required;
+# Returns 1 (failure) if $1 >= $2, 0 (success) otherwise.
+#
+check_for_version()
+{
+    y1=${1#*.}
+    [ "$y1" = "$1" ] && y1=""
+    z1=${y1#*.}
+    [ "$z1" = "$y1" ] && z1=""
+    x1=${1%%.*}
+    y1=${y1%%.*}
+    z1=${z1%%.*}
+    [ -z "$y1" ] && y1=0
+    [ -z "$z1" ] && z1=0
+    y2=${2#*.}
+    [ "$y2" = "$2" ] && y2=""
+    z2=${y2#*.}
+    [ "$z2" = "$y2" ] && z2=""
+    x2=${2%%.*}
+    y2=${y2%%.*}
+    z2=${z2%%.*}
+    [ -z "$y2" ] && y2=0
+    [ -z "$z2" ] && z2=0
+    [ $x1 -lt $x2 ] && return 1
+    [ $x1 -gt $x2 ] && return 0
+    [ $y1 -lt $y2 ] && return 1
+    [ $y1 -gt $y2 ] && return 0
+    [ $z1 -lt $z2 ] && return 1
+    return 0
+}
diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh

-Original file line number
+Diff line change
@@ @@ -0,0 +1,4 @@ @@
 +#
 +# suite.pm will make sure that all tests including this file
 +# will be skipped as needed
 +#