MDEV-25359 : Improve mariabackup SST script compliance with native Ma…

…riaDB SSL practices and configuration.

1. Pass joiner's authentication information to donor together with address
   in State Transfer Request. This allows joiner to authenticate donor on
   connection. Previously joiner would accept data from anywhere.

2. Deprecate custom SSL configuration variables tca, tcert and tkey in favor
   of more familiar ssl-ca, ssl-cert and ssl-key. For backward compatibility
   tca, tcert and tkey are still supported.

3. Allow falling back to server-wide SSL configuration in [mysqld] if no SSL
   configuration is found in [sst] section of the config file.

4. Introduce ssl-mode variable in [sst] section that takes standard values
   and has following effects:
    - old-style SSL configuration present in [sst]: no effect
      otherwise:
    - ssl-mode=DISABLED or absent: retains old, backward compatible behavior
      and ignores any other SSL configuration
    - ssl-mode=VERIFY*: verify joiner's certificate and CN on donor,
                        verify donor's secret on joiner
                        (passed to donor via State Transfer Request)
                        BACKWARD INCOMPATIBLE BEHAVIOR
    - anything else enables new SSL configuration convetions but does not
      require verification

    ssl-mode should be set to VERIFY only in a fully upgraded cluster.

    Examples:

    [mysqld]
    ssl-cert=/path/to/cert
    ssl-key=/path/to/key
    ssl-ca=/path/to/ca

    [sst]

     -- server-wide SSL configuration is ignored, SST does not use SSL

    [mysqld]
    ssl-cert=/path/to/cert
    ssl-key=/path/to/key
    ssl-ca=/path/to/ca

    [sst]
    ssl-mode=REQUIRED

     -- use server-wide SSL configuration for SST but don't attempt to
        verify the peer identity

    [sst]
    ssl-cert=/path/to/cert
    ssl-key=/path/to/key
    ssl-ca=/path/to/ca
    ssl-mode=VERIFY_CA

     -- use SST-specific SSL configuration for SST and require verification
        on both sides

Reviewed-by: Jan Lindström <jan.lindstrom@mariadb.com>

mysql-test/suite/galera/r/galera_sst_mariabackup_encrypt_with_key.result

@@ -3,3 +3,4 @@ connection node_1;
 SELECT 1;
 1
 1
+include/assert_grep.inc [Using openssl based encryption with socat]

mysql-test/suite/galera/r/galera_sst_mariabackup_encrypt_with_key_server.result

@@ -0,0 +1,6 @@
+connection node_2;
+connection node_1;
+SELECT 1;
+1
+1
+include/assert_grep.inc [Using openssl based encryption with socat]

mysql-test/suite/galera/t/galera_sst_mariabackup_encrypt_with_key.test

@@ -1,6 +1,8 @@
 #
-# This test checks that key and cert encryption options can be passed to mariabackup via the my.cnf file
-# Initial SST happens via mariabackup, so there is not much to do in the body of the test
+# This test checks that key and cert encryption options can be passed to
+# mariabackup via the my.cnf file
+# Initial SST happens via mariabackup, so there is not much to do in the body
+# of the test
 #
 
 --source include/big_test.inc
@@ -12,3 +14,11 @@ SELECT 1;
 
 --let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size';
 --source include/wait_condition.inc
+
+# Confirm that transfer was SSL-encrypted
+--let $assert_text = Using openssl based encryption with socat
+--let $assert_select = Using openssl based encryption with socat: with key and crt
+--let $assert_count = 1
+--let $assert_file = $MYSQLTEST_VARDIR/log/mysqld.1.err
+--let $assert_only_after = CURRENT_TEST
+--source include/assert_grep.inc

scripts/wsrep_sst_common.sh

@@ -25,6 +25,7 @@ WSREP_SST_OPT_DATA=""
 WSREP_SST_OPT_AUTH=${WSREP_SST_OPT_AUTH:-}
 WSREP_SST_OPT_USER=${WSREP_SST_OPT_USER:-}
 WSREP_SST_OPT_PSWD=${WSREP_SST_OPT_PSWD:-}
+WSREP_SST_OPT_REMOTE_AUTH=${WSREP_SST_OPT_REMOTE_AUTH:-}
 WSREP_SST_OPT_DEFAULT=""
 WSREP_SST_OPT_EXTRA_DEFAULT=""
 WSREP_SST_OPT_SUFFIX_DEFAULT=""
@@ -34,6 +35,7 @@ INNODB_DATA_HOME_DIR_ARG=""
 INNODB_LOG_GROUP_HOME_ARG=""
 INNODB_UNDO_DIR_ARG=""
 LOG_BIN_ARG=""
+readonly WSREP_SST_OPT_REMOTE_AUTH
 
 while [ $# -gt 0 ]; do
 case "$1" in
@@ -121,7 +123,8 @@ case "$1" in
         WSREP_SST_OPT_BYPASS=1
         ;;
     '--datadir')
-        readonly WSREP_SST_OPT_DATA="$2"
+        # strip trailing '/'
+        readonly WSREP_SST_OPT_DATA="${2%/}"
         shift
         ;;
     '--innodb-data-home-dir')
@@ -323,12 +326,22 @@ readonly WSREP_SST_OPT_AUTH
 # Splitting AUTH into potential user:password pair
 if ! wsrep_auth_not_set
 then
-    WSREP_SST_OPT_USER="${WSREP_SST_OPT_AUTH%%:*}"
+    WSREP_SST_OPT_USER="${WSREP_SST_OPT_AUTH%:*}"
     WSREP_SST_OPT_PSWD="${WSREP_SST_OPT_AUTH##*:}"
 fi
 readonly WSREP_SST_OPT_USER
 readonly WSREP_SST_OPT_PSWD
 
+if [ -n "$WSREP_SST_OPT_REMOTE_AUTH" ]
+then
+    # Split auth string at the last ':'
+    readonly WSREP_SST_OPT_REMOTE_USER="${WSREP_SST_OPT_REMOTE_AUTH%:*}"
+    readonly WSREP_SST_OPT_REMOTE_PSWD="${WSREP_SST_OPT_REMOTE_AUTH##*:}"
+else
+    readonly WSREP_SST_OPT_REMOTE_USER=
+    readonly WSREP_SST_OPT_REMOTE_PSWD=
+fi
+
 if [ -n "${WSREP_SST_OPT_DATA:-}" ]
 then
     SST_PROGRESS_FILE="$WSREP_SST_OPT_DATA/sst_in_progress"
@@ -386,6 +399,19 @@ wsrep_check_programs()
     done
 }
 
+# Generate a string equivalent to 16 random bytes
+wsrep_gen_secret()
+{
+    if [ -x /usr/bin/openssl ]
+    then
+        echo `/usr/bin/openssl rand -hex 16`
+    else
+        printf "%04x%04x%04x%04x%04x%04x%04x%04x" \
+                $RANDOM $RANDOM $RANDOM $RANDOM   \
+                $RANDOM $RANDOM $RANDOM $RANDOM
+    fi
+}
+
 #
 # user can specify mariabackup specific settings that will be used during sst
 # process like encryption, etc.....
@@ -397,14 +423,22 @@ wsrep_check_programs()
 parse_cnf()
 {
     local group=$1
-    local var=$2
+    local var=${2//_/-} # normalize variable name by replacing all '_' with '-'
     local reval=""
 
-    # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin)
+    # first normalize output variable names specified in cnf file:
+    # user can use _ or - (for example log-bin or log_bin) and/or prefix
+    # variable with --loose-
     # then search for needed variable
     # finally get the variable value (if variables has been specified multiple time use the last value only)
 
-    reval=$($MY_PRINT_DEFAULTS "${group}" | awk -v var="${var}" 'BEGIN { OFS=FS="=" } { gsub(/_/,"-",$1); if ( $1=="--"var) lastval=substr($0,length($1)+2) } END { print lastval}')
+    reval=$($MY_PRINT_DEFAULTS "${group}" | \
+            awk -v var="${var}" 'BEGIN { OFS=FS="=" } \
+                                       { sub(/^--loose/,"-",$0); \
+                                         gsub(/_/,"-",$1); \
+                                         if ( $1=="--"var) \
+                                             lastval=substr($0,length($1)+2) } \
+                                 END { print lastval}')
 
     # use default if we haven't found a value
     if [ -z "$reval" ]; then

scripts/wsrep_sst_mariabackup.sh

@@ -37,6 +37,7 @@ REMOTEIP=""
 tcert=""
 tpem=""
 tkey=""
+tmode="DISABLED"
 sockopt=""
 progress=""
 ttime=0
@@ -70,6 +71,8 @@ xtmpdir=""
 scomp=""
 sdecomp=""
 
+readonly SECRET_TAG="secret"
+
 # Required for backup locks
 # For backup locks it is 1 sent by joiner
 # 5.6.21 PXC and later can't donate to an older joiner
@@ -261,24 +264,30 @@ get_transfer()
                 exit 22
             fi
             stagemsg+="-OpenSSL-Encrypted-3"
-            if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
-                if [[ -z $tcert ]];then
+            if [[ -z $tcert ]];then
+                # no verification
+                if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
                     wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, verify=0"
                     tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},verify=0${sockopt} stdio"
                 else
-                    wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}"
-                    tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},cafile=${tcert}${sockopt} stdio"
-                fi
-            else
-                if [[ -z $tcert ]];then
                     wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, verify=0"
                     tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},verify=0${sockopt}"
+                fi
+            else
+                # CA verification
+                if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
+                    wsrep_log_info "Decrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}"
+                    tcmd="socat -u openssl-listen:${TSST_PORT},reuseaddr,cert=${tpem},key=${tkey},cafile=${tcert}${sockopt} stdio"
                 else
+                    if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
+                        CN_option=",commonname=$WSREP_SST_OPT_REMOTE_USER"
+                    else
+                        CN_option=""
+                    fi
                     wsrep_log_info "Encrypting with cert=${tpem}, key=${tkey}, cafile=${tcert}"
-                    tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},cafile=${tcert}${sockopt}"
+                    tcmd="socat -u stdio openssl-connect:${REMOTEIP}:${TSST_PORT},cert=${tpem},key=${tkey},cafile=${tcert}${CN_option}${sockopt}"
                 fi
             fi
-
         else 
             if [[ "$WSREP_SST_OPT_ROLE"  == "joiner" ]];then
                 tcmd="socat -u TCP-LISTEN:${TSST_PORT},reuseaddr${sockopt} stdio"
@@ -287,18 +296,24 @@ get_transfer()
             fi
         fi
     fi
-
 }
 
 parse_cnf()
 {
     local group=$1
     local var=$2
     # print the default settings for given group using my_print_default.
+    # remove possible 'loose' variable name prefix
     # normalize the variable names specified in cnf file (user can use _ or - for example log-bin or log_bin)
     # then grep for needed variable
     # finally get the variable value (if variables has been specified multiple time use the last value only)
-    reval=$($MY_PRINT_DEFAULTS $group | awk -F= '{if ($1 ~ /_/) { gsub(/_/,"-",$1); print $1"="$2 } else { print $0 }}' | grep -- "--$var=" | cut -d= -f2- | tail -1)
+    reval=$($MY_PRINT_DEFAULTS $group | \
+            awk -F= '{ sub(/^--loose/,"-",$0); \
+                       if ($1 ~ /_/) \
+                           { gsub(/_/,"-",$1); print $1"="$2 } \
+                       else \
+                           { print $0 }}' | \
+            grep -- "--$var=" | cut -d= -f2- | tail -1)
     if [[ -z $reval ]];then 
         [[ -n $3 ]] && reval=$3
     fi
@@ -348,6 +363,24 @@ adjust_progress()
     fi
 }
 
+check_server_ssl_config()
+{
+    local section=$1
+    tcert=$(parse_cnf $section ssl-ca "")
+    tpem=$(parse_cnf $section ssl-cert "")
+    tkey=$(parse_cnf $section ssl-key "")
+    if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ]
+    then
+         encrypt=3 # enable cert/key SSL encyption
+
+         # avoid CA verification if not set explicitly:
+         # nodes may happen to have different CA if self-generated
+         # zeroing up tcert does the trick
+         local mode=$(parse_cnf SST ssl-mode "")
+         [[ ${mode} = *VERIFY* ]] || tcert=""
+    fi
+}
+
 read_cnf()
 {
     sfmt=$(parse_cnf sst streamfmt "xbstream")
@@ -356,6 +389,26 @@ read_cnf()
     tpem=$(parse_cnf sst tcert "")
     tkey=$(parse_cnf sst tkey "")
     encrypt=$(parse_cnf sst encrypt 0)
+    tmode=$(parse_cnf sst ssl-mode "DISABLED" | tr [:lower:] [:upper:])
+
+    if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]
+    then # no old-style SSL config in [sst]
+        if [ "$tmode" != "DISABLED" ]
+        then # backward-incompatible behavior
+            check_server_ssl_config "sst"
+            if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]
+            then # no new-stype SSL config in [sst], try server-wide SSL config
+                check_server_ssl_config "mysqld.$WSREP_SST_OPT_SUFFIX_VALUE"
+                if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]
+                then
+                    check_server_ssl_config "mysqld"
+                fi
+            fi
+        fi
+    fi
+    wsrep_log_info "SSL configuration: CA='"$tcert"', CERT='"$tpem"'," \
+                   "KEY='"$tkey"', MODE='"$tmode"', encrypt="$encrypt
+
     sockopt=$(parse_cnf sst sockopt "")
     progress=$(parse_cnf sst progress "")
     ttime=$(parse_cnf sst time 0)
@@ -400,7 +453,7 @@ read_cnf()
 
     if [[ $encrypt -eq 1 ]]; then
         wsrep_log_error "Xtrabackup-based encryption is currently not" \
-            "supported with MariaBackup"
+                        "supported with MariaBackup"
         exit 2
     fi
 }
@@ -627,8 +680,8 @@ recv_joiner()
     popd 1>/dev/null 
 
     if [[ ${RC[0]} -eq 124 ]];then 
-        wsrep_log_error "Possible timeout in receiving first data from "
-	                "donor in gtid stage: exit codes: ${RC[@]}"
+        wsrep_log_error "Possible timeout in receiving first data from " \
+                        "donor in gtid stage: exit codes: ${RC[@]}"
         exit 32
     fi
 
@@ -640,12 +693,27 @@ recv_joiner()
         fi
     done
 
-    if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then
-        # this message should cause joiner to abort
-        wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
-        wsrep_log_info "Contents of datadir" 
-        wsrep_log_info "$(ls -l ${dir}/*)"
-        exit 32
+    if [[ $checkf -eq 1 ]]; then
+        if [[ ! -r "${MAGIC_FILE}" ]];then
+            # this message should cause joiner to abort
+            wsrep_log_error "receiving process ended without creating " \
+                            "'${MAGIC_FILE}'"
+            wsrep_log_info "Contents of datadir" 
+            wsrep_log_info "$(ls -l ${dir}/*)"
+            exit 32
+        fi
+
+        # check donor supplied secret
+        SECRET=$(grep "$SECRET_TAG " ${MAGIC_FILE} 2>/dev/null | cut -d ' ' -f 2)
+        if [[ $SECRET != $MY_SECRET ]]; then
+            wsrep_log_error "Donor does not know my secret!"
+            wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'"
+            exit 32
+        fi
+
+        # remove secret from magic file
+        grep -v "$SECRET_TAG " ${MAGIC_FILE} > ${MAGIC_FILE}.new
+        mv ${MAGIC_FILE}.new ${MAGIC_FILE}
     fi
 }
 
@@ -661,10 +729,9 @@ send_donor()
     set -e
     popd 1>/dev/null 
 
-
     for ecode in "${RC[@]}";do 
         if [[ $ecode -ne 0 ]];then 
-            wsrep_log_error "Error while getting data from donor node: " \
+            wsrep_log_error "Error while sending data to joiner node: " \
                             "exit codes: ${RC[@]}"
             exit 32
         fi
@@ -878,6 +945,11 @@ then
         # (separated by a space).
         echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}"
 
+        if [[ -n ${WSREP_SST_OPT_REMOTE_PSWD} ]]; then
+            # Let joiner know that we know its secret
+            echo "$SECRET_TAG ${WSREP_SST_OPT_REMOTE_PSWD}" >> ${MAGIC_FILE}
+        fi
+
         ttcmd="$tcmd"
 
         if [[ $encrypt -eq 1 ]];then
@@ -990,7 +1062,6 @@ then
 
     stagemsg="Joiner-Recv"
 
-
     sencrypted=1
     nthreads=1
 
@@ -1012,7 +1083,26 @@ then
         fi
     fi
 
-    wait_for_listen ${SST_PORT} ${ADDR} ${MODULE} &
+    if [[ "$tmode" = *"VERIFY"* ]]
+    then # backward-incompatible behavior
+        if [ -n "$tpem" ]
+        then
+            # find out my Common Name
+            wsrep_check_programs openssl
+            CN=$(openssl x509 -noout -subject -in $tpem | \
+                 tr "," "\n" | grep "CN =" | cut -d= -f2 | sed s/^\ // | \
+                 sed s/\ %//)
+        else
+            CN=""
+        fi
+        MY_SECRET=$(wsrep_gen_secret)
+        # Add authentication data to address
+        ADDR="$CN:$MY_SECRET@$ADDR"
+    else
+        MY_SECRET="" # for check down in recv_joiner()
+    fi # tmode == *VERIFY*
+
+    wait_for_listen ${SST_PORT} "${ADDR}" ${MODULE} &
 
     trap sig_joiner_cleanup HUP PIPE INT TERM
     trap cleanup_joiner EXIT