Pass backup status and command output to scripts

.github/workflows/build.yml

@@ -12,7 +12,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v4.2.2

.github/workflows/push-test.yml

@@ -7,7 +7,7 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     steps:
       - name: Checkout
         uses: actions/checkout@v3

.github/workflows/verify.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   verify:
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-latest
     env:
       SRC_DIR: /tmp/source
       DEST_DIR: /tmp/dest

README.md

@@ -187,6 +187,8 @@ Potential use-cases include sending notifications, or replicating a restic repos
 
 The backup waits for the server to respond to a rcon "save-on" command before running the scripts. After, the `PRE_SAVE_ALL_SCRIPT` is run, followed by rcon "save-off" and "save-all" commands. The, the `PRE_BACKUP_SCRIPT` is run, followed by the backup process. Then, the `PRE_SAVE_ON_SCRIPT` is run, followed by a rcon "save-on" command. Finally, the `POST_BACKUP_SCRIPT` is run.
 
+`PRE_SAVE_ON_SCRIPT` and `POST_BACKUP_SCRIPT` are both passed the exit code of the backup as the first argument, and the path to a log of the backup tool's output as the second argument. This may be used to take different actions depending on whether or not the backup failed.
+
 Alternatively `PRE_SAVE_ALL_SCRIPT_FILE` `PRE_BACKUP_SCRIPT_FILE`, `PRE_SAVE_ON_SCRIPT_FILE`, and `POST_BACKUP_SCRIPT_FILE` may be set to the path of a script that has been mounted into the container. The file must be executable.
 
 Note that `*_FILE` variables will be overridden by their non-FILE versions if both are set.

scripts/opt/backup-loop.sh

@@ -231,6 +231,7 @@ load_rcon_password() {
 # Should define following functions inside them
 # init() -> called before entering loop. Verify arguments, prepare for operations etc.
 # backup() -> create backup. It's guaranteed that all data is already flushed to disk.
+#   Passed a path to a log file that the backup tool's stdout and stderr should be written to.
 # prune() -> prune old backups. PRUNE_BACKUPS_DAYS is guaranteed to be positive.
 
 
@@ -280,21 +281,23 @@ tar() {
     esac
   }
   backup() {
+    if [[ ! $1 ]]; then
+      log INTERNALERROR "Backup log path not passed to tar.backup! Aborting"
+      exit 1
+    fi
+
     ts=$(date +"%Y%m%d-%H%M%S")
     outFile="${DEST_DIR}/${BACKUP_NAME}-${ts}.${backup_extension}"
     log INFO "Backing up content in ${SRC_DIR} to ${outFile}"
-    command tar "${excludes[@]}" "${tar_parameters[@]}" -cf "${outFile}" -C "${SRC_DIR}" "${includes_patterns[@]}" || exitCode=$?
-    if [ ${exitCode:-0} -eq 0 ]; then
-      true
-    elif [ ${exitCode:-0} -eq 1 ]; then
+    exitCode=0
+    command tar "${excludes[@]}" "${tar_parameters[@]}" -cf "${outFile}" -C "${SRC_DIR}" "${includes_patterns[@]}" 2>&1 | tee "$1" || exitCode=$?
+    if [[ $exitCode -eq 1 ]]; then
       log WARN "Dat files changed as we read it"
-    elif [ ${exitCode:-0} -gt 1 ]; then
-      log ERROR "tar exited with code ${exitCode}! Aborting"
-      exit 1
     fi
     if [ "${LINK_LATEST^^}" == "TRUE" ]; then
       ln -sf "${BACKUP_NAME}-${ts}.${backup_extension}" "${DEST_DIR}/latest.${backup_extension}"
     fi
+    return $exitCode
   }
   prune() {
 
@@ -313,6 +316,7 @@ tar() {
   call_if_function_exists "${@}"
 }
 
+# shellcheck disable=SC2317
 rsync() {
   _find_old_backups() {
     find "${DEST_DIR}" -maxdepth 1 -type d -mtime "+${PRUNE_BACKUPS_DAYS}" "${@}"
@@ -328,6 +332,11 @@ rsync() {
     mkdir -p "${DEST_DIR}"
   }
   backup() {
+    if [[ ! $1 ]]; then
+      log INTERNALERROR "Backup log path not passed to rsync.backup! Aborting"
+      exit 1
+    fi
+
     ts=$(date +"%Y%m%d-%H%M%S")
     outFile="${DEST_DIR}/${BACKUP_NAME}-${ts}"
     if [ -d "${DEST_DIR}/latest" ]; then
@@ -342,19 +351,18 @@ rsync() {
     fi
     log INFO "Backing up content in ${SRC_DIR} to ${outFile}"
     mkdir -p $outFile
-    command rsync -a "${link_dest[@]}" "${excludes[@]}" "${SRC_DIR}/" "${outFile}/"  || exitCode=$?
-    if [ ${exitCode:-0} -eq 0 ]; then
+    exitCode=0
+    command rsync -a "${link_dest[@]}" "${excludes[@]}" "${SRC_DIR}/" "${outFile}/" 2>&1 | tee "$1" || exitCode=$?
+    if [[ $exitCode -eq 0 ]]; then
       touch "${outFile}"
       true
-    elif [ ${exitCode:-0} -eq 1 ]; then
+    elif [[ $exitCode -eq 1 ]]; then
       log WARN "Dat files changed as we read it"
-    elif [ ${exitCode:-0} -gt 1 ]; then
-      log ERROR "rsync exited with code ${exitCode}! Aborting"
-      exit 1
     fi
     if [ "${LINK_LATEST^^}" == "TRUE" ]; then
       ln -sfT "${BACKUP_NAME}-${ts}" "${DEST_DIR}/latest"
     fi
+    return $exitCode
   }
   prune() {
 
@@ -380,6 +388,7 @@ rsync() {
 }
 
 
+# shellcheck disable=SC2317
 restic() {
   readarray -td, includes_patterns < <(printf '%s' "${INCLUDES:-${SRC_DIR}}")
 
@@ -461,6 +470,11 @@ restic() {
     readonly restic_tags_filter
   }
   backup() {
+    if [[ ! $1 ]]; then
+      log INTERNALERROR "Backup log path not passed to restic.backup! Aborting"
+      exit 1
+    fi
+
     log INFO "Backing up content in ${SRC_DIR} as host ${RESTIC_HOSTNAME}"
     args=(
       --host "${RESTIC_HOSTNAME}"
@@ -469,7 +483,7 @@ restic() {
       args+=(-vv)
     fi
     (cd "$SRC_DIR" &&
-          command restic backup "${args[@]}" "${restic_tags_arguments[@]}" "${excludes[@]}" "${includes_patterns[@]}" | log INFO
+          command restic backup "${args[@]}" "${restic_tags_arguments[@]}" "${excludes[@]}" "${includes_patterns[@]}" 2>&1 | tee "$1" | log INFO
     )
   }
   prune() {
@@ -484,6 +498,7 @@ restic() {
 }
 
 
+# shellcheck disable=SC2317
 rclone() {
   readarray -td, includes_patterns < <(printf '%s' "${INCLUDES:-.}")
 
@@ -519,6 +534,11 @@ rclone() {
     esac
   }
   backup() {
+    if [[ ! $1 ]]; then
+      log INTERNALERROR "Backup log path not passed to rsync.backup! Aborting"
+      exit 1
+    fi
+
     ts=$(date +"%Y%m%d-%H%M%S")
     outFile="${DEST_DIR}/${BACKUP_NAME}-${ts}.${backup_extension}"
     log INFO "Backing up content in ${SRC_DIR} to ${outFile}"
@@ -532,10 +552,10 @@ rclone() {
       exit 1
     fi
 
-    if ! command rclone copy "${outFile}" "${RCLONE_REMOTE}:${RCLONE_DEST_DIR}"; then
-      log ERROR "rclone copy operation failed -- will retry next time"
-    fi
+    exitCode=0
+    command rclone copy "${outFile}" "${RCLONE_REMOTE}:${RCLONE_DEST_DIR}" 2>&1 | tee "$1" || exitCode=$?
     rm "${outFile}"
+    return $exitCode
   }
   prune() {
     if [ -n "$(_find_old_backups)" ]; then
@@ -621,6 +641,8 @@ fi
 first_run=TRUE
 
 while true; do
+  backup_log="$(mktemp)"
+
   if [[ $first_run == TRUE && ${ONE_SHOT^^} = FALSE && ${BACKUP_ON_STARTUP^^} = FALSE ]]; then
     log INFO "Skipping backup on startup"
     first_run=false
@@ -647,10 +669,15 @@ while true; do
         "$PRE_BACKUP_SCRIPT_FILE"
       fi
 
-      "${BACKUP_METHOD}" backup
+      backup_status=0
+      "${BACKUP_METHOD}" backup "$backup_log" || backup_status=$?
+
+      if [[ $backup_status -ne 0 ]]; then
+        log ERROR "Backup failed with exit code $backup_status"
+      fi
 
       if [[ $PRE_SAVE_ON_SCRIPT_FILE ]]; then
-        "$PRE_SAVE_ON_SCRIPT_FILE"
+        "$PRE_SAVE_ON_SCRIPT_FILE" $backup_status "$backup_log"
       fi
 
       retry ${RCON_RETRIES} ${RCON_RETRY_INTERVAL} rcon-cli save-on
@@ -659,7 +686,7 @@ while true; do
       trap EXIT
 
       if [[ $POST_BACKUP_SCRIPT_FILE ]]; then
-        "$POST_BACKUP_SCRIPT_FILE"
+        "$POST_BACKUP_SCRIPT_FILE" $backup_status "$backup_log"
       fi
 
     else
@@ -673,13 +700,20 @@ while true; do
       "$PRE_BACKUP_SCRIPT_FILE"
     fi
 
-    "${BACKUP_METHOD}" backup
+    backup_status=0
+    "${BACKUP_METHOD}" backup "$backup_log" || backup_status=$?
+
+    if [[ $backup_status -ne 0 ]]; then
+      log WARN "Backup failed with exit code $backup_status"
+    fi
 
     if [[ $POST_BACKUP_SCRIPT_FILE ]]; then
-      "$POST_BACKUP_SCRIPT_FILE"
+      "$POST_BACKUP_SCRIPT_FILE" $backup_status "$backup_log"
     fi
   fi
 
+  rm "$backup_log"
+
   if (( PRUNE_BACKUPS_DAYS > 0 )); then
     "${BACKUP_METHOD}" prune
   fi