Merge pull request #2831 from pcahyna/rsync-url-fix-refactor

Refactor rsync URL support, fixes rsync OUTPUT_URL: The code to parse rsync:// URLs was BACKUP_URL specific. If one specified BACKUP=RSYNC and an OUTPUT_URL different from BACKUP_URL, the OUTPUT_URL was ignored and the output files went to BACKUP_URL. Fix by introducing generic functions for rsync URL parsing and use them for both BACKUP_URL and OUTPUT_URL, as appropriate. Replace all uses of global RSYNC_* variables derived from BACKUP_URL by those functions. There also was inconsistent special handling for OUTPUT=PXE which is now removed: An rsync OUTPUT_URL with OUTPUT=PXE now creates the RSYNC_PREFIX directory at the destination and the URL is interpreted as in all other cases. See #2831 and #2781
rear · Jul 12, 2022 · e0d3c6e · e0d3c6e
2 parents 659f885 + e7556a1
commit e0d3c6e
Show file tree

Hide file tree

Showing 18 changed files with 334 additions and 140 deletions.
diff --git a/usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh b/usr/share/rear/backup/NETFS/default/200_check_rsync_relative_option.sh
diff --git a/usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh b/usr/share/rear/backup/NETFS/default/210_check_rsync_relative_option.sh
@@ -0,0 +1 @@
+../../RSYNC/default/210_check_rsync_relative_option.sh
diff --git a/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh b/usr/share/rear/backup/RSYNC/GNU/Linux/610_start_selinux.sh
@@ -6,29 +6,29 @@ local backup_prog_rc
 	touch "${TMP_DIR}/selinux.autorelabel"
 	cat $TMP_DIR/selinux.mode > $SELINUX_ENFORCE
 	Log "Restored original SELinux mode"
-	case $RSYNC_PROTO in
+	case $(rsync_proto "$BACKUP_URL") in
 
 	(ssh)
 		# for some reason rsync changes the mode of backup after each run to 666
                 # FIXME: Add an explanatory comment why "2>/dev/null" is useful here
                 # or remove it according to https://github.com/rear/rear/issues/1395
-		ssh $RSYNC_USER@$RSYNC_HOST "chmod $v 755 ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" 2>/dev/null
+		ssh $(rsync_remote_ssh "$BACKUP_URL") "chmod $v 755 $(rsync_path_full "$BACKUP_URL")/backup" 2>/dev/null
 		$BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" \
-		 "$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" 2>/dev/null
+		 "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" 2>/dev/null
 		backup_prog_rc=$?
 		if [ $backup_prog_rc -ne 0 ]; then
-			LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]"
-			#StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup"
+			LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]"
+			#StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup"
 		fi
 		;;
 
 	(rsync)
 		$BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" "${BACKUP_RSYNC_OPTIONS[@]}" \
-		 "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel"
+		 "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel"
 		backup_prog_rc=$?
 		if [ $backup_prog_rc -ne 0 ]; then
-			LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]"
-			#StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup"
+			LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]"
+			#StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup"
 		fi
 		;;
 

diff --git a/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh b/usr/share/rear/backup/RSYNC/GNU/Linux/620_force_autorelabel.sh
@@ -4,29 +4,29 @@ local backup_prog_rc
 
 	> "${TMP_DIR}/selinux.autorelabel"
 
-	case $RSYNC_PROTO in
+	case $(rsync_proto "$BACKUP_URL") in
 
 	(ssh)
 		# for some reason rsync changes the mode of backup after each run to 666
                 # FIXME: Add an explanatory comment why "2>/dev/null" is useful here
                 # or remove it according to https://github.com/rear/rear/issues/1395
-		ssh $RSYNC_USER@$RSYNC_HOST "chmod $v 755 ${RSYNC_PATH}/${RSYNC_PREFIX}/backup" 2>/dev/null
+		ssh $(rsync_remote_ssh "$BACKUP_URL") "chmod $v 755 $(rsync_path_full "$BACKUP_URL")/backup" 2>/dev/null
 		$BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" \
-		 "$RSYNC_USER@$RSYNC_HOST:${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel" 2>/dev/null
+		 "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel" 2>/dev/null
 		backup_prog_rc=$?
 		if [ $backup_prog_rc -ne 0 ]; then
-			LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]"
-			#StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup"
+			LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]"
+			#StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup"
 		fi
 		;;
 
 	(rsync)
 		$BACKUP_PROG -a "${TMP_DIR}/selinux.autorelabel" "${BACKUP_RSYNC_OPTIONS[@]}" \
-		 "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup/.autorelabel"
+		 "$(rsync_remote_full "$BACKUP_URL")/backup/.autorelabel"
 		backup_prog_rc=$?
 		if [ $backup_prog_rc -ne 0 ]; then
-			LogPrint "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup [${rsync_err_msg[$backup_prog_rc]}]"
-			#StopIfError "Failed to create .autorelabel on ${RSYNC_PATH}/${RSYNC_PREFIX}/backup"
+			LogPrint "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup [${rsync_err_msg[$backup_prog_rc]}]"
+			#StopIfError "Failed to create .autorelabel on $(rsync_path_full "$BACKUP_URL")/backup"
 		fi
 		;;
 

diff --git a/usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh b/usr/share/rear/backup/RSYNC/default/200_make_prefix_dir.sh
@@ -0,0 +1,28 @@
+# Create RSYNC_PREFIX/backup on remote rsync server
+# RSYNC_PREFIX=$HOSTNAME as set in default.conf
+
+local proto host
+
+proto="$(rsync_proto "$BACKUP_URL")"
+host="$(rsync_host "$BACKUP_URL")"
+
+mkdir -p $v -m0750 "${TMP_DIR}/rsync/${RSYNC_PREFIX}" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}'"
+mkdir -p $v -m0755 "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup" >&2 || Error "Could not mkdir '${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup'"
+
+case $proto in
+
+	(ssh)
+		$BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "$(rsync_remote "$BACKUP_URL")" >/dev/null 2>&1 \
+                    || Error "Could not create '$(rsync_path_full "$BACKUP_URL")' on remote ${host}"
+		;;
+
+	(rsync)
+		$BACKUP_PROG -a $v -r "${TMP_DIR}/rsync/${RSYNC_PREFIX}" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote "$BACKUP_URL")/" >/dev/null \
+                    || Error "Could not create '$(rsync_path_full "$BACKUP_URL")' on remote ${host}"
+		;;
+
+esac
+
+# We don't need it anymore, from now we operate on the remote copy
+rmdir $v "${TMP_DIR}/rsync/${RSYNC_PREFIX}/backup"
+rmdir $v "${TMP_DIR}/rsync/${RSYNC_PREFIX}"
diff --git a/...efault/200_check_rsync_relative_option.sh → ...efault/210_check_rsync_relative_option.sh b/...efault/200_check_rsync_relative_option.sh → ...efault/210_check_rsync_relative_option.sh
@@ -1,4 +1,4 @@
-# 200_check_rsync_relative_option.sh
+# 210_check_rsync_relative_option.sh
 # See issue #871 for details
 
 # check for the --relative option in BACKUP_RSYNC_OPTIONS array

diff --git a/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh b/usr/share/rear/backup/RSYNC/default/450_calculate_req_space.sh
@@ -1,6 +1,12 @@
 # here we will calculate the space required to hold the backup archive on the remote rsync system
 # This file is part of Relax-and-Recover, licensed under the GNU General
 # Public License. Refer to the included COPYING for full text of license.
+local proto host path
+
+proto="$(rsync_proto "$BACKUP_URL")"
+host="$(rsync_host "$BACKUP_URL")"
+path="$(rsync_path "$BACKUP_URL")"
+
 _local_size=0
 _remote_size=0
 while read -r ; do
@@ -14,17 +20,17 @@ done < $TMP_DIR/backup-include.txt
 LogPrint "Estimated size of local file systems is $(( _local_size / 1024 )) MB"
 
 # Commenting out next block according decision of issue #2760
-#case $RSYNC_PROTO in
+#case $proto in
 #    (ssh)
-#        LogPrint "Calculating size of $RSYNC_HOST:$RSYNC_PATH"
-#        ssh -l $RSYNC_USER $RSYNC_HOST "df -P $RSYNC_PATH" >$TMP_DIR/rs_size
-#        StopIfError "Failed to determine size of $RSYNC_PATH"
+#        LogPrint "Calculating size of ${host}:${path}"
+#        ssh $(rsync_remote_ssh "$BACKUP_URL") "df -P ${path}" >$TMP_DIR/rs_size
+#        StopIfError "Failed to determine size of ${path}"
 #        _div=1 # 1024-blocks
 #        grep -q "512-blocks" $TMP_DIR/rs_size && _div=2 # HPUX: divide with 2 to get kB size
 #        _remote_size=$( tail -n 1 $TMP_DIR/rs_size | awk '{print $2}' )
 #        _remote_size=$(( _remote_size / _div ))
 #        [[ $_remote_size -gt $_local_size ]]
-#        StopIfError "Not enough disk space available on $RSYNC_HOST:$RSYNC_PATH ($_remote_size < $_local_size)"
+#        StopIfError "Not enough disk space available on ${host}:${path} ($_remote_size < $_local_size)"
 #        ;;
 #    (rsync)
 #        # TODO: how can we calculate the free size on remote system via rsync protocol??

diff --git a/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh b/usr/share/rear/backup/RSYNC/default/500_make_rsync_backup.sh
@@ -5,6 +5,11 @@
 local backup_prog_rc
 local backup_log_message
 
+local host path
+
+host="$(rsync_host "$BACKUP_URL")"
+path="$(rsync_path "$BACKUP_URL")"
+
 Log "Include list:"
 while read -r ; do
 	Log "  $REPLY"
@@ -14,26 +19,27 @@ while read -r ; do
 	Log " $REPLY"
 done < $TMP_DIR/backup-exclude.txt
 
-LogPrint "Creating $BACKUP_PROG backup on '${RSYNC_HOST}:${RSYNC_PATH}'"
+LogPrint "Creating $BACKUP_PROG backup on '${host}:${path}'"
 
 ProgressStart "Running backup operation"
 (
 	case "$(basename $BACKUP_PROG)" in
 
 		(rsync)
+			# We are in a subshell, so this change will not propagate to later scripts
 			BACKUP_RSYNC_OPTIONS+=( --one-file-system --delete --exclude-from=$TMP_DIR/backup-exclude.txt --delete-excluded )
 
-			case $RSYNC_PROTO in
+			case $(rsync_proto "$BACKUP_URL") in
 
 				(ssh)
-					Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup"
+					Log $BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) "$(rsync_remote_full "$BACKUP_URL")/backup"
 					$BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) \
-					"${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/backup"
+					"$(rsync_remote_full "$BACKUP_URL")/backup"
 					;;
 
 				(rsync)
 					$BACKUP_PROG "${BACKUP_RSYNC_OPTIONS[@]}" $(cat $TMP_DIR/backup-include.txt) \
-					"${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/backup"
+					"$(rsync_remote_full "$BACKUP_URL")/backup"
 					;;
 
 			esac
@@ -57,11 +63,11 @@ get_size() {
 }
 
 check_remote_df() {
-	echo $(ssh ${RSYNC_USER}@${RSYNC_HOST} df -P ${RSYNC_PATH} 2>/dev/null | tail -1 | awk '{print $5}' | sed -e 's/%//')
+	echo $(ssh $(rsync_remote_ssh "$BACKUP_URL") df -P ${path} 2>/dev/null | tail -1 | awk '{print $5}' | sed -e 's/%//')
 }
 
 check_remote_du() {
-	x=$(ssh ${RSYNC_USER}@${RSYNC_HOST} du -sb ${RSYNC_PATH}/${RSYNC_PREFIX}/backup 2>/dev/null | awk '{print $1}')
+	x=$(ssh $(rsync_remote_ssh "$BACKUP_URL") du -sb $(rsync_path_full "$BACKUP_URL")/backup 2>/dev/null | awk '{print $1}')
 	[[ -z "${x}" ]] && x=0
 	echo $x
 }
@@ -81,7 +87,7 @@ case "$(basename $BACKUP_PROG)" in
 			case $i in
 
 			300)
-			[[ $(check_remote_df) -eq 100 ]] && Error "Disk is full on system ${RSYNC_HOST}"
+			[[ $(check_remote_df) -eq 100 ]] && Error "Disk is full on system ${host}"
 			;;
 
 			15|30|45|60|75|90|105|120|135|150|165|180|195|210|225|240|255|270|285)

diff --git a/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh b/usr/share/rear/backup/RSYNC/default/700_copy_backup_log.sh
@@ -1,26 +1,27 @@
 
 # copy the backup.log & rear.log file to remote destination with timestamp added
-local timestamp
+local timestamp proto
 
 timestamp=$( date +%Y%m%d.%H%M )
+proto="$(rsync_proto "$BACKUP_URL")"
 
 # compress the log file first
 gzip "$TMP_DIR/$BACKUP_PROG_ARCHIVE.log" || Error "Failed to 'gzip $TMP_DIR/$BACKUP_PROG_ARCHIVE.log'"
 
-case $RSYNC_PROTO in
+case $proto in
     (ssh)
         # FIXME: Add an explanatory comment why "2>/dev/null" is useful here
         # or remove it according to https://github.com/rear/rear/issues/1395
         $BACKUP_PROG -a "${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log.gz" \
-        "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" 2>/dev/null
+        "$(rsync_remote_full "$BACKUP_URL")/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz" 2>/dev/null
 
-        $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PATH}/${RSYNC_PREFIX}/rear-${timestamp}.log" 2>/dev/null
+        $BACKUP_PROG -a "$RUNTIME_LOGFILE" "$(rsync_remote_full "$BACKUP_URL")/rear-${timestamp}.log" 2>/dev/null
         ;;
     (rsync)
         $BACKUP_PROG -a "${TMP_DIR}/${BACKUP_PROG_ARCHIVE}.log.gz" "${BACKUP_RSYNC_OPTIONS[@]}" \
-        "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz"
+        "$(rsync_remote_full "$BACKUP_URL")/${BACKUP_PROG_ARCHIVE}-${timestamp}.log.gz"
 
-        $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${BACKUP_RSYNC_OPTIONS[@]}" "${RSYNC_PROTO}://${RSYNC_USER}@${RSYNC_HOST}:${RSYNC_PORT}/${RSYNC_PATH}/${RSYNC_PREFIX}//rear-${timestamp}.log"
+        $BACKUP_PROG -a "$RUNTIME_LOGFILE" "${BACKUP_RSYNC_OPTIONS[@]}" "$(rsync_remote_full "$BACKUP_URL")//rear-${timestamp}.log"
         ;;
 esac
 
diff --git a/usr/share/rear/lib/global-functions.sh b/usr/share/rear/lib/global-functions.sh
@@ -259,7 +259,7 @@ function url_scheme() {
     # the scheme is the leading part up to '://'
     local scheme=${url%%://*}
     # rsync scheme does not have to start with rsync:// it can also be scp style
-    # see the comments in usr/share/rear/prep/RSYNC/default/100_check_rsync.sh
+    # see the comments in usr/share/rear/lib/rsync-functions.sh
     echo $scheme | grep -q ":" && echo rsync || echo $scheme
 }