Fixes after review and correction of the problems caused by the fact …

…that during the SST innodb plugin is not yet initialized, as well as problems with running tests from the root user (not directly related to the MDEV-10754).

mysql-test/suite/galera/include/galera_wsrep_recover.inc

@@ -1,5 +1,14 @@
 --echo Performing --wsrep-recover ...
+#
+# When mysqld is run by a root user it will fail to start unless
+# we specify what user is ran it (using "--user root" option):
+#
+if ($MYSQL_TEST_ROOT == 1) {
+--exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --innodb --wsrep-recover --user root > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1
+}
+if ($MYSQL_TEST_ROOT != 1) {
 --exec $MYSQLD --defaults-group-suffix=.$galera_wsrep_recover_server_id --defaults-file=$MYSQLTEST_VARDIR/my.cnf --innodb --wsrep-recover > $MYSQL_TMP_DIR/galera_wsrep_recover.log 2>&1
+}
 
 --perl
 	use strict;

mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf

@@ -2,10 +2,10 @@
 
 [mysqld]
 wsrep_sst_method=rsync
-innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test
 
 [mysqld.1]
 wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore_sb=true'
 
 [mysqld.2]
+innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test_2
 wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'

mysql-test/suite/galera/t/galera_sst_rsync_data_dir.test

@@ -13,5 +13,4 @@
 --source include/auto_increment_offset_restore.inc
 
 # cleanup temporary database files:
---remove_files_wildcard $MYSQL_TMP_DIR/rsync_test *
---rmdir $MYSQL_TMP_DIR/rsync_test
+--remove_files_wildcard $MYSQL_TMP_DIR/rsync_test_2 *

scripts/wsrep_sst_common.sh

@@ -27,6 +27,7 @@ WSREP_SST_OPT_PSWD=${WSREP_SST_OPT_PSWD:-}
 WSREP_SST_OPT_DEFAULT=""
 WSREP_SST_OPT_EXTRA_DEFAULT=""
 WSREP_SST_OPT_SUFFIX_DEFAULT=""
+WSREP_SST_OPT_SUFFIX_VALUE=""
 
 while [ $# -gt 0 ]; do
 case "$1" in
@@ -76,6 +77,7 @@ case "$1" in
         ;;
     '--defaults-group-suffix')
         readonly WSREP_SST_OPT_SUFFIX_DEFAULT="$1=$2"
+        readonly WSREP_SST_OPT_SUFFIX_VALUE="$2"
         shift
         ;;
     '--host')

scripts/wsrep_sst_rsync.sh

@@ -123,8 +123,16 @@ fi
 
 INNODB_DATA_HOME_DIR=${INNODB_DATA_HOME_DIR:-""}
 # if INNODB_DATA_HOME_DIR env. variable is not set, try to get it from my.cnf
-if [ -z "INNODB_DATA_HOME_DIR" ]; then
-    INNODB_DATA_HOME_DIR=$(parse_cnf --mysqld innodb-data-home-dir '')
+if [ -z "$INNODB_DATA_HOME_DIR" ]; then
+    INNODB_DATA_HOME_DIR=$(parse_cnf mysqld$WSREP_SST_OPT_SUFFIX_VALUE innodb-data-home-dir '')
+fi
+
+if [ -n "$INNODB_DATA_HOME_DIR" ]; then
+    # handle both relative and absolute paths
+    INNODB_DATA_HOME_DIR=$(cd $WSREP_SST_OPT_DATA; mkdir -p "$INNODB_DATA_HOME_DIR"; cd $INNODB_DATA_HOME_DIR; pwd -P)
+else
+    # default to datadir
+    INNODB_DATA_HOME_DIR=$(cd $WSREP_SST_OPT_DATA; pwd -P)
 fi
 
 # Old filter - include everything except selected
@@ -135,8 +143,8 @@ fi
 
 # New filter - exclude everything except dirs (schemas) and innodb files
 FILTER=(-f '- /lost+found' -f '- /.fseventsd' -f '- /.Trashes'
-        -f '+ /wsrep_sst_binlog.tar' -f '+ $INNODB_DATA_HOME_DIR/ib_lru_dump'
-        -f '+ $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*')
+        -f '+ /wsrep_sst_binlog.tar' -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump'
+        -f '- $INNODB_DATA_HOME_DIR/ibdata*' -f '+ /*/' -f '- /*'"
 
 SSTKEY=$(parse_cnf sst tkey "")
 SSTCERT=$(parse_cnf sst tcert "")
@@ -242,6 +250,19 @@ EOF
             exit $RC
         fi
 
+        # Transfer InnoDB data files
+        rsync ${STUNNEL:+--rsh="$STUNNEL"} \
+              --owner --group --perms --links --specials \
+              --ignore-times --inplace --dirs --delete --quiet \
+              $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \
+              -f '- **' "$INNODB_DATA_HOME_DIR/" \
+              rsync://$WSREP_SST_OPT_ADDR-data_dir >&2 || RC=$?
+
+        if [ $RC -ne 0 ]; then
+            wsrep_log_error "rsync innodb_data_home_dir returned code $RC:"
+            exit 255 # unknown error
+        fi
+
         # second, we transfer InnoDB log files
         rsync ${STUNNEL:+--rsh="$STUNNEL"} \
               --owner --group --perms --links --specials \
@@ -340,8 +361,12 @@ $SILENT
     path = $WSREP_SST_OPT_DATA
 [$MODULE-log_dir]
     path = $WSREP_LOG_DIR
+[$MODULE-data_dir]
+    path = $INNODB_DATA_HOME_DIR
 EOF
 
+[ "$(whoami)" != root ] || sed -i '/read only = no/s/.*/&\nuid = root\ngid = root\n/' "$RSYNC_CONF"
+
 #    rm -rf "$DATA"/ib_logfile* # we don't want old logs around
 
     # listen at all interfaces (for firewalled setups)

sql/wsrep_mysqld.h

@@ -160,6 +160,7 @@ extern "C" time_t wsrep_thd_query_start(THD *thd);
 extern "C" query_id_t wsrep_thd_query_id(THD *thd);
 extern "C" query_id_t wsrep_thd_wsrep_last_query_id(THD *thd);
 extern "C" void wsrep_thd_set_wsrep_last_query_id(THD *thd, query_id_t id);
+extern "C" void wsrep_set_data_home_dir(const char *data_dir);
 
 extern void wsrep_close_client_connections(my_bool wait_to_end);
 extern int  wsrep_wait_committing_connections_close(int wait_time);

sql/wsrep_sst.cc

@@ -67,6 +67,13 @@ bool wsrep_sst_method_update (sys_var *self, THD* thd, enum_var_type type)
     return 0;
 }
 
+static const char* data_home_dir = NULL;
+
+extern "C"
+void wsrep_set_data_home_dir(const char *data_dir)
+{
+  data_home_dir= (data_dir && *data_dir) ? data_dir : NULL;
+}
 
 static void make_wsrep_defaults_file()
 {
@@ -549,6 +556,29 @@ static int sst_append_auth_env(wsp::env& env, const char* sst_auth)
   return -env.error();
 }
 
+#define DATA_HOME_DIR_ENV "INNODB_DATA_HOME_DIR"
+
+static int sst_append_data_dir(wsp::env& env, const char* data_dir)
+{
+  int const data_dir_size= strlen(DATA_HOME_DIR_ENV) + 1 /* = */
+    + (data_dir ? strlen(data_dir) : 0) + 1 /* \0 */;
+
+  wsp::string data_dir_str(data_dir_size); // for automatic cleanup on return
+  if (!data_dir_str()) return -ENOMEM;
+
+  int ret= snprintf(data_dir_str(), data_dir_size, "%s=%s",
+                    DATA_HOME_DIR_ENV, data_dir ? data_dir : "");
+
+  if (ret < 0 || ret >= data_dir_size)
+  {
+    WSREP_ERROR("sst_append_data_dir(): snprintf() failed: %d", ret);
+    return (ret < 0 ? ret : -EMSGSIZE);
+  }
+
+  env.append(data_dir_str());
+  return -env.error();
+}
+
 static ssize_t sst_prepare_other (const char*  method,
                                   const char*  sst_auth,
                                   const char*  addr_in,
@@ -610,6 +640,16 @@ static ssize_t sst_prepare_other (const char*  method,
     return ret;
   }
 
+  if (data_home_dir)
+  {
+    if ((ret= sst_append_data_dir(env, data_home_dir)))
+    {
+      WSREP_ERROR("sst_prepare_other(): appending data "
+                  "directory failed: %d", ret);
+      return ret;
+    }
+  }
+
   pthread_t tmp;
   sst_thread_arg arg(cmd_str(), env());
   mysql_mutex_lock (&arg.lock);
@@ -1301,6 +1341,16 @@ wsrep_cb_status_t wsrep_sst_donate_cb (void* app_ctx, void* recv_ctx,
     return WSREP_CB_FAILURE;
   }
 
+  if (data_home_dir)
+  {
+    if ((ret= sst_append_data_dir(env, data_home_dir)))
+    {
+      WSREP_ERROR("wsrep_sst_donate_cb(): appending data "
+                  "directory failed: %d", ret);
+      return WSREP_CB_FAILURE;
+    }
+  }
+
   if (!strcmp (WSREP_SST_MYSQLDUMP, method))
   {
     ret = sst_donate_mysqldump(data, &current_gtid->uuid, uuid_str,

storage/innobase/handler/ha_innodb.cc

@@ -3623,6 +3623,12 @@ innobase_init(
 	srv_data_home = (innobase_data_home_dir ? innobase_data_home_dir :
 			 default_path);
 
+#ifdef WITH_WSREP
+	/* If we use the wsrep API, then we need to tell the server
+	the path to the data files (for passing it to the SST scripts): */
+	wsrep_set_data_home_dir(innobase_data_home_dir);
+#endif /* WITH_WSREP */
+
 	/* Set default InnoDB data file size to 12 MB and let it be
 	auto-extending. Thus users can use InnoDB in >= 4.0 without having
 	to specify any startup options. */