Merge branch 'master' of github.com:pcbsd/pcbsd

src-sh/lpreserver/backend/functions.sh

@@ -13,6 +13,7 @@ PROGDIR="/usr/local/share/lpreserver"
 DBDIR="/var/db/lpreserver"
 if [ ! -d "$DBDIR" ] ; then mkdir -p ${DBDIR} ; fi
 CMDLOG="${DBDIR}/lp-lastcmdout"
+CMDLOG2="${DBDIR}/lp-lastcmdout2"
 REPCONF="${DBDIR}/replication"
 LOGDIR="/var/log"
 export DBDIR LOGDIR PROGDIR CMDLOG REPCONF
@@ -179,11 +180,146 @@ add_rep_task() {
   rem_rep_task "$LDATA"
   echo "$LDATA:$TIME:$HOST:$USER:$PORT:$RDATA" >> ${REPCONF}
 
+  if [ "$TIME" != "sync" ] ; then
+    cronscript="${PROGDIR}/backend/runrep.sh"
+    cLine="0    $TIME       *       *       *"
+    echo -e "$cLine\troot    ${cronscript} ${LDATA}" >> /etc/crontab
+  fi
 }
 
 rem_rep_task() {
   if [ ! -e "$REPCONF" ] ; then return ; fi
   cat ${REPCONF} | grep -v "^${1}:" > ${REPCONF}.tmp
   mv ${REPCONF}.tmp ${REPCONF}
+
+  # Make sure we remove any old replication entries for this dataset
+  cronscript="${PROGDIR}/backend/runrep.sh"
+  cat /etc/crontab | grep -v " $cronscript $1" > /etc/crontab.new
+  mv /etc/crontab.new /etc/crontab
+}
+
+list_rep_task() {
+  if [ ! -e "$REPCONF" ] ; then return ; fi
+
+  echo "Scheduled replications:"
+  echo "---------------------------------"
+
+  while read line
+  do
+     LDATA=`echo $line | cut -d ':' -f 1`
+     TIME=`echo $line | cut -d ':' -f 2`
+     HOST=`echo $line | cut -d ':' -f 3`
+     USER=`echo $line | cut -d ':' -f 4`
+     PORT=`echo $line | cut -d ':' -f 5`
+     RDATA=`echo $line | cut -d ':' -f 6`
+
+     echo "$LDATA -> $USER@$HOST[$PORT]:$RDATA Time: $TIME"
+
+  done < ${REPCONF}
 }
 
+check_rep_task() {
+  export DIDREP=0
+  if [ ! -e "$REPCONF" ] ; then return 0; fi
+
+  repLine=`cat ${REPCONF} | grep "^${1}:"`
+  if [ -z "$repLine" ] ; then return 0; fi
+
+  # We have a replication task for this dataset, lets check if we need to do it now
+  LDATA="$1"
+  TIME=`echo $repLine | cut -d ':' -f 2`
+
+  # Export the replication variables we will be using
+  export REPHOST=`echo $repLine | cut -d ':' -f 3`
+  export REPUSER=`echo $repLine | cut -d ':' -f 4`
+  export REPPORT=`echo $repLine | cut -d ':' -f 5`
+  export REPRDATA=`echo $repLine | cut -d ':' -f 6`
+
+  # If we are checking for a sync task, and the rep isn't marked as sync we can return
+  if [ "$2" = "sync" -a "$TIME" != "sync" ] ; then return 0; fi
+
+  # Is this a sync-task we do at the time of a snapshot?
+  if [ "$2" = "sync" -a "$TIME" = "sync" ] ; then
+     export DIDREP=1
+     echo_log "`date`: Starting replication SYNC task on ${DATASET}\n"
+     queue_msg "`date`: Starting replication SYNC task on ${DATASET}\n"
+     start_rep_task "$LDATA"
+     return $?
+  else
+     # Ready to do a scheduled replication
+     export DIDREP=1
+     echo_log "`date`: Starting replication SCHEDULED task on ${DATASET}\n"
+     queue_msg "`date`: Starting replication SCHEDULED task on ${DATASET}\n"
+     start_rep_task "$LDATA"
+     return $?
+  fi
+}
+
+start_rep_task() {
+  LDATA="$1"
+
+  # Check for the last snapshot marked as replicated already
+  lastSEND=`zfs get -r backup:lpreserver ${LDATA} | grep LATEST | awk '{$1=$1}1' OFS=" " | tail -1 | cut -d '@' -f 2 | cut -d ' ' -f 1`
+
+  # Lets get the last snapshot for this dataset
+  lastSNAP=`zfs list -t snapshot -d 1 -H ${LDATA} | tail -1 | awk '{$1=$1}1' OFS=" " | cut -d '@' -f 2 | cut -d ' ' -f 1`
+
+  if [ "$lastSEND" = "$lastSNAP" ] ; then
+     queue_msg "`date`: Last snapshot $lastSNAP is already marked as replicated!"
+     return 1
+  fi
+
+  # Starting replication, first lets check if we can do an incremental send
+  if [ -n "$lastSEND" ] ; then
+     zFLAGS="-Rv -I $lastSEND $LDATA@$lastSNAP"
+  else
+     zFLAGS="-Rv $LDATA@$lastSNAP"
+  fi
+
+  zSEND="zfs send $zFLAGS"
+  zRCV="ssh -p ${REPPORT} ${REPUSER}@${REPHOST} zfs receive -dvuF ${REPRDATA}"
+
+  queue_msg "Using ZFS send command:\n$zSEND | $zRCV\n\n"
+
+  # Start up our process
+  $zSEND 2>${CMDLOG} | $zRCV >${CMDLOG2} 2>${CMDLOG2}
+  zStatus=$?
+  queue_msg "ZFS SEND LOG:\n--------------\n`cat ${CMDLOG}`\n\n"
+  queue_msg "ZFS RCV LOG:\n--------------\n`cat ${CMDLOG2}`\n\n"
+
+  if [ $zStatus -eq 0 ] ; then
+     # SUCCESS!
+     # Lets mark our new latest snapshot and unmark the last one
+     zfs set backup:lpreserver=' ' ${LDATA}@$lastSEND
+     zfs set backup:lpreserver=LATEST ${LDATA}@$lastSNAP
+     echo_log "`date`: Finished replication task on ${DATASET}\n"
+  else
+     # FAILED :-(
+     # Lets save the output for us to look at later
+     FLOG=${LOGDIR}/lpreserver_failed.log
+     echo "Failed with command:\n$zSEND | $zRCV\n" > ${FLOG}
+     echo "\nSending log:\n" >> ${FLOG}
+     cat ${CMDLOG} >> ${FLOG}
+     echo "\nRecv log:\n" >> ${FLOG}
+     cat ${CMDLOG2} >> ${FLOG}
+     echo_log "`date`: FAILED replication task on ${DATASET}: LOGFILE: $FLOG\n"
+  fi
+
+  return $zStatus
+}
+
+get_data_props() {
+
+  # Lets start by building a list of props to keep
+  pTag=`echo $DATASET | md5`
+
+  if [ "$RECURMODE" = "ON" ] ; then
+     zfs get -r all $DATASET | grep ' local$' | awk '{$1=$1}1' OFS=" " > /tmp/.propList.$$
+  else
+     zfs get all $DATASET | grep ' local$' | awk '{$1=$1}1' OFS=" " > /tmp/.propList.$$
+  fi
+
+  cat /tmp/.propList.$$
+  rm /tmp/.propList.$$
+
+}

src-sh/lpreserver/backend/runrep.sh

@@ -10,23 +10,28 @@ PROGDIR="/usr/local/share/lpreserver"
 . ${PROGDIR}/backend/functions.sh
 
 DATASET="${1}"
+TIME="${2}"
 
 if [ -z "${DATASET}" ]; then
   exit_err "No dataset specified!"
 fi
 
-# Lets start by building a list of props to keep
-pTag=`echo $DATASET | md5`
+check_rep_task "$DATASET" "$TIME"
+status=$?
 
-if [ "$RECURMODE" = "ON" ] ; then
-   zfs get -r all $DATASET | grep ' local$' | awk '{$1=$1}1' OFS=" " > /tmp/.propList.$$
+# No replication was needed / done
+if [ $DIDREP -eq 0 ] ; then exit 0 ; fi
+
+if [ $status -eq 0 ] ; then
+  title="Success"
 else
-   zfs get all $DATASET | grep ' local$' | awk '{$1=$1}1' OFS=" " > /tmp/.propList.$$
+  title="FAILED"
 fi
 
-cat /tmp/.propList.$$
-rm /tmp/.propList.$$
-
-if [ "$EMAILMODE" = "ALL" ] ; then
-   #email_msg "Automated Snapshot" "`echo_queue_msg`"
-fi
+case $EMAILMODE in
+    ALL) email_msg "Automated Replication - $title" "`echo_queue_msg`" ;;
+    *) if [ $status -ne 0 ] ; then
+          email_msg "Automated Replication - $title" "`echo_queue_msg`"
+       fi
+       ;;
+esac

src-sh/lpreserver/backend/runsnap.sh

@@ -62,3 +62,6 @@ done
 if [ "$EMAILMODE" = "ALL" ] ; then
    email_msg "Automated Snapshot" "`echo_queue_msg`"
 fi
+
+# Check if we need to run a replication task for this dataset
+${PROGDIR}/backend/runrep.sh ${DATASET} sync

src-sh/lpreserver/lpreserver

@@ -374,7 +374,7 @@ revertsnap) require_root
 	    shift
 	    case ${1} in
 		add) add_rep_task "$2" "$3" "$4" "$5" "$6" "$7" ;;
-		list) ;;
+		list) list_rep_task ;;
 		remove) cat ${REPCONF} | grep -q "^${2}:" 
 			if [ $? -eq 0 ] ; then
 			   rem_rep_task "$2"