bzr merge -r3985..3991 codership/5.5

.bzrignore

@@ -12,6 +12,7 @@
 *.dll
 *.dsp
 *.dylib
+*.diff
 *.exe
 *.exp
 *.gcda

scripts/wsrep_sst_common.sh

@@ -1,4 +1,4 @@
-# Copyright (C) 2012 Codership Oy
+# Copyright (C) 2012-2014 Codership Oy
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -125,3 +125,27 @@ wsrep_cleanup_progress_file()
 {
     [ -n "$SST_PROGRESS_FILE" ] && rm -f "$SST_PROGRESS_FILE" 2>/dev/null
 }
+
+wsrep_check_program()
+{
+    local prog=$1
+
+    if ! which $prog >/dev/null
+    then
+        echo "'$prog' not found in PATH"
+        return 2 # no such file or directory
+    fi
+}
+
+wsrep_check_programs()
+{
+    local ret=0
+
+    while [ $# -gt 0 ]
+    do
+        wsrep_check_program $1 || ret=$?
+        shift
+    done
+
+    return $ret
+}

scripts/wsrep_sst_rsync.sh

@@ -1,6 +1,6 @@
 #!/bin/bash -ue
 
-# Copyright (C) 2010 Codership Oy
+# Copyright (C) 2010-2014 Codership Oy
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,11 +23,13 @@ RSYNC_CONF=
 OS=$(uname)
 [ "$OS" == "Darwin" ] && export -n LD_LIBRARY_PATH
 
-. $(dirname $0)/wsrep_sst_common
-
 # Setting the path for lsof on CentOS
 export PATH="/usr/sbin:/sbin:$PATH"
 
+. $(dirname $0)/wsrep_sst_common
+
+wsrep_check_programs rsync
+
 cleanup_joiner()
 {
     wsrep_log_info "Joiner cleanup."
@@ -197,6 +199,8 @@ then
 
 elif [ "$WSREP_SST_OPT_ROLE" = "joiner" ]
 then
+    wsrep_check_programs lsof
+
     touch $SST_PROGRESS_FILE
     MYSQLD_PID=$WSREP_SST_OPT_PARENT
 

scripts/wsrep_sst_xtrabackup-v2.sh

@@ -63,6 +63,10 @@ STATDIR=""
 uextra=0
 disver=""
 
+tmpopts=""
+itmpdir=""
+xtmpdir=""
+
 scomp=""
 sdecomp=""
 
@@ -236,7 +240,7 @@ parse_cnf()
 get_footprint()
 {
     pushd $WSREP_SST_OPT_DATA 1>/dev/null
-    payload=$(du --block-size=1 -c  **/*.ibd **/*.MYI **/*.MYI ibdata1  | awk 'END { print $1 }')
+    payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }')
     if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 
         # QuickLZ has around 50% compression ratio
         # When compression/compaction used, the progress is only an approximate.
@@ -299,6 +303,7 @@ read_cnf()
     iopts=$(parse_cnf sst inno-backup-opts "")
     iapts=$(parse_cnf sst inno-apply-opts "")
     impts=$(parse_cnf sst inno-move-opts "")
+    stimeout=$(parse_cnf sst sst-initial-timeout 100)
 }
 
 get_stream()
@@ -370,28 +375,38 @@ cleanup_donor()
         wsrep_log_error "Cleanup after exit with status:$estatus"
     fi
 
-    if [[ -n $XTRABACKUP_PID ]];then 
+    if [[ -n ${XTRABACKUP_PID:-} ]];then 
         if check_pid $XTRABACKUP_PID
         then
             wsrep_log_error "xtrabackup process is still running. Killing... "
             kill_xtrabackup
         fi
 
-        rm -f $XTRABACKUP_PID 
     fi
-    rm -f ${DATA}/${IST_FILE}
+    rm -f ${DATA}/${IST_FILE} || true
 
     if [[ -n $progress && -p $progress ]];then 
         wsrep_log_info "Cleaning up fifo file $progress"
-        rm $progress
+        rm -f $progress || true
+    fi
+
+    wsrep_log_info "Cleaning up temporary directories"
+
+    if [[ -n $xtmpdir ]];then 
+       [[ -d $xtmpdir ]] &&  rm -rf $xtmpdir || true
+    fi
+
+    if [[ -n $itmpdir ]];then 
+       [[ -d $itmpdir ]] &&  rm -rf $itmpdir || true
     fi
 }
 
 kill_xtrabackup()
 {
     local PID=$(cat $XTRABACKUP_PID)
     [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
-    rm -f "$XTRABACKUP_PID"
+    wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID"
+    rm -f "$XTRABACKUP_PID" || true
 }
 
 setup_ports()
@@ -453,13 +468,30 @@ recv_joiner()
 {
     local dir=$1
     local msg=$2 
+    local tmt=$3
+    local ltcmd
 
     pushd ${dir} 1>/dev/null
     set +e
-    timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+
+    if [[ $tmt -gt 0 && -x `which timeout` ]];then 
+        if timeout --help | grep -q -- '-k';then 
+            ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
+        else 
+            ltcmd="timeout $tmt $tcmd"
+        fi
+        timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    else 
+        timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+    fi
+
     set -e
     popd 1>/dev/null 
 
+    if [[ ${RC[0]} -eq 124 ]];then 
+        wsrep_log_error "Possible timeout in receving first data from donor in gtid stage"
+        exit 32
+    fi
 
     for ecode in "${RC[@]}";do 
         if [[ $ecode -ne 0 ]];then 
@@ -501,7 +533,7 @@ send_donor()
 
 }
 
-if [[ ! -x `which innobackupex` ]];then 
+if [[ ! -x `which $INNOBACKUPEX_BIN` ]];then 
     wsrep_log_error "innobackupex not in path: $PATH"
     exit 2
 fi
@@ -526,7 +558,7 @@ fi
 INNOEXTRA=""
 INNOAPPLY="${INNOBACKUPEX_BIN} $disver $iapts --apply-log \$rebuildcmd \${DATA} &>\${DATA}/innobackup.prepare.log"
 INNOMOVE="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $impts  --move-back --force-non-empty-directories \${DATA} &>\${DATA}/innobackup.move.log"
-INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$INNOEXTRA --galera-info --stream=\$sfmt \${TMPDIR} 2>\${DATA}/innobackup.backup.log"
+INNOBACKUP="${INNOBACKUPEX_BIN} --defaults-file=${WSREP_SST_OPT_CONF} $disver $iopts \$tmpopts \$INNOEXTRA --galera-info --stream=\$sfmt \$itmpdir 2>\${DATA}/innobackup.backup.log"
 
 if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
 then
@@ -535,7 +567,14 @@ then
     if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
     then
 
-        TMPDIR="${TMPDIR:-/tmp}"
+        if [[ -z $(parse_cnf mysqld tmpdir "") && -z $(parse_cnf xtrabackup tmpdir "") ]];then 
+            xtmpdir=$(mktemp -d)
+            tmpopts=" --tmpdir=$xtmpdir "
+            wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory"
+        fi
+
+        itmpdir=$(mktemp -d)
+        wsrep_log_info "Using $itmpdir as innobackupex temporary directory"
 
         if [ "${AUTH[0]}" != "(null)" ]; then
            INNOEXTRA+=" --user=${AUTH[0]}"
@@ -613,8 +652,8 @@ then
           exit 22
         fi
 
-        # innobackupex implicitly writes PID to fixed location in ${TMPDIR}
-        XTRABACKUP_PID="${TMPDIR}/xtrabackup_pid"
+        # innobackupex implicitly writes PID to fixed location in $xtmpdir
+        XTRABACKUP_PID="$xtmpdir/xtrabackup_pid"
 
 
     else # BYPASS FOR IST
@@ -713,7 +752,7 @@ then
 
     STATDIR=$(mktemp -d)
     MAGIC_FILE="${STATDIR}/${INFO_FILE}"
-    recv_joiner $STATDIR  "${stagemsg}-gtid" 1 
+    recv_joiner $STATDIR  "${stagemsg}-gtid" $stimeout
 
     if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null
     then
@@ -763,7 +802,7 @@ then
 
 
         MAGIC_FILE="${DATA}/${INFO_FILE}"
-        recv_joiner $DATA "${stagemsg}-SST" 0 
+        recv_joiner $DATA "${stagemsg}-SST" 0
 
         get_proc
 
@@ -863,11 +902,10 @@ then
         wsrep_log_info "${IST_FILE} received from donor: Running IST"
     fi
 
-    if [[ ! -r ${MAGIC_FILE} ]];then
+    if [[ ! -r ${MAGIC_FILE} ]];then 
         wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
         exit 2
     fi
-
     cat "${MAGIC_FILE}" # output UUID:seqno
     wsrep_log_info "Total time on joiner: $totime seconds"
 fi

scripts/wsrep_sst_xtrabackup.sh

@@ -224,7 +224,7 @@ parse_cnf()
 get_footprint()
 {
     pushd $WSREP_SST_OPT_DATA 1>/dev/null
-    payload=$(du --block-size=1 -c  **/*.ibd **/*.MYI **/*.MYI ibdata1  | awk 'END { print $1 }')
+    payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | xargs -0 du --block-size=1 -c | awk 'END { print $1 }')
     if my_print_defaults -c $WSREP_SST_OPT_CONF xtrabackup | grep -q -- "--compress";then 
         # QuickLZ has around 50% compression ratio
         # When compression/compaction used, the progress is only an approximate.

sql/mysqld.cc

@@ -2990,9 +2990,14 @@ static bool cache_thread()
 bool one_thread_per_connection_end(THD *thd, bool put_in_cache)
 {
   DBUG_ENTER("one_thread_per_connection_end");
+#ifdef WITH_WSREP
+  const bool wsrep_applier(thd->wsrep_applier);
+#endif
+
   unlink_thd(thd);
   /* Mark that current_thd is not valid anymore */
   set_current_thd(0);
+
 #ifdef WITH_WSREP
   if (put_in_cache && cache_thread() && !thd->wsrep_applier)
 #else

sql/wsrep_mysqld.cc

@@ -531,9 +531,9 @@ int wsrep_init()
     wsrep_inited= 1;
     global_system_variables.wsrep_on = 0;
     wsrep_init_args args;
+    args.logger_cb = wsrep_log_cb;
     args.options = (wsrep_provider_options) ?
             wsrep_provider_options : "";
-    args.logger_cb= wsrep_log_cb;
     rcode = wsrep->init(wsrep, &args);
     if (rcode)
     {

sql/wsrep_thd.cc

@@ -28,11 +28,19 @@
 #include "rpl_rli.h"
 #include "rpl_mi.h"
 
-static long long wsrep_bf_aborts_counter = 0;
+#if (__LP64__)
+static volatile int64 wsrep_bf_aborts_counter(0);
+#define WSREP_ATOMIC_LOAD_LONG my_atomic_load64
+#define WSREP_ATOMIC_ADD_LONG  my_atomic_add64
+#else
+static volatile int32 wsrep_bf_aborts_counter(0);
+#define WSREP_ATOMIC_LOAD_LONG my_atomic_load32
+#define WSREP_ATOMIC_ADD_LONG  my_atomic_add32
+#endif
 
 int wsrep_show_bf_aborts (THD *thd, SHOW_VAR *var, char *buff)
 {
-    wsrep_local_bf_aborts = my_atomic_load64(&wsrep_bf_aborts_counter);
+    wsrep_local_bf_aborts = WSREP_ATOMIC_LOAD_LONG(&wsrep_bf_aborts_counter);
     var->type = SHOW_LONGLONG;
     var->value = (char*)&wsrep_local_bf_aborts;
     return 0;
@@ -44,7 +52,7 @@ void wsrep_client_rollback(THD *thd)
   WSREP_DEBUG("client rollback due to BF abort for (%ld), query: %s",
               thd->thread_id, thd->query());
 
-  my_atomic_add64(&wsrep_bf_aborts_counter, 1);
+  WSREP_ATOMIC_ADD_LONG(&wsrep_bf_aborts_counter, 1);
 
   thd->wsrep_conflict_state= ABORTING;
   mysql_mutex_unlock(&thd->LOCK_wsrep_thd);

support-files/mysql.spec.sh

@@ -304,7 +304,7 @@ Version:        %{mysql_version}
 Summary:        MySQL: a very fast and reliable SQL database server
 Group:          Applications/Databases
 %if %{defined with_wsrep}
-Requires:       %{distro_requires} rsync
+Requires:       %{distro_requires} rsync lsof
 %else
 Requires:       %{distro_requires}
 %endif

wsrep/wsrep_dummy.c

@@ -20,11 +20,13 @@
 
 #include <errno.h>
 #include <stdbool.h>
+#include <string.h>
 
 /*! Dummy backend context. */
 typedef struct wsrep_dummy
 {
     wsrep_log_cb_t log_fn;
+    char* options;
 } wsrep_dummy_t;
 
 /* Get pointer to wsrep_dummy context from wsrep_t pointer */
@@ -43,6 +45,10 @@ static void dummy_free(wsrep_t *w)
 {
     WSREP_DBUG_ENTER(w);
     free(w->ctx);
+    if (WSREP_DUMMY(w)->options) {
+        free(WSREP_DUMMY(w)->options);
+        WSREP_DUMMY(w)->options = NULL;
+    }
     w->ctx = NULL;
 }
 
@@ -51,6 +57,9 @@ static wsrep_status_t dummy_init (wsrep_t* w,
 {
     WSREP_DUMMY(w)->log_fn = args->logger_cb;
     WSREP_DBUG_ENTER(w);
+    if (args->options) {
+        WSREP_DUMMY(w)->options = strdup(args->options);
+    }
     return WSREP_OK;
 }
 
@@ -61,16 +70,23 @@ static uint64_t dummy_capabilities (wsrep_t* w __attribute__((unused)))
 
 static wsrep_status_t dummy_options_set(
     wsrep_t* w,
-    const char* conf __attribute__((unused)))
+    const char* conf)
 {
     WSREP_DBUG_ENTER(w);
+    if (WSREP_DUMMY(w)->options) {
+        free(WSREP_DUMMY(w)->options);
+        WSREP_DUMMY(w)->options = NULL;
+    }
+    if (conf) {
+        WSREP_DUMMY(w)->options = strdup(conf);
+    }
     return WSREP_OK;
 }
 
 static char* dummy_options_get (wsrep_t* w)
 {
     WSREP_DBUG_ENTER(w);
-    return NULL;
+    return WSREP_DUMMY(w)->options;
 }
 
 static wsrep_status_t dummy_connect(
@@ -385,6 +401,7 @@ int wsrep_dummy_loader(wsrep_t* w)
 
     // initialize private context
     WSREP_DUMMY(w)->log_fn = NULL;
+    WSREP_DUMMY(w)->options = NULL;
 
     return 0;
 }