ARTESCA-9658 retry for ETCd backup

scality · Oct 4, 2023 · a9e1b44 · a9e1b44
1 parent a5688a7
commit a9e1b44
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 4 deletions.
diff --git a/scripts/backup.sh.in b/scripts/backup.sh.in
@@ -181,7 +181,7 @@ replicate_archives() {
 
 run "Backing up MetalK8s configurations" backup_metalk8s_conf
 run "Backing up CAs certificates and keys" backup_cas
-run "Backing up etcd data" backup_etcd
+run_with_retry 12 5 "Backing up etcd data" backup_etcd
 run "Creating backup archive '$BACKUP_ARCHIVE'" create_archive
 
 if (( REPLICATION )); then

diff --git a/scripts/common.sh.in b/scripts/common.sh.in
@@ -153,6 +153,11 @@ stop_salt_minion_service() {
     ${SYSTEMCTL} stop salt-minion.service 2>/dev/null || true
 }
 
+die() {
+    echo 1>&2 "$@"
+    return 1
+}
+
 run_quiet() {
     local name=$1
     shift 1
@@ -213,9 +218,24 @@ run() {
     fi
 }
 
-die() {
-    echo 1>&2 "$@"
-    return 1
+run_with_retry() {
+    local retries=$1
+    local delay=$2
+    shift 2
+
+    local -i i=0
+    while true; do
+        if [[ $(( ++i )) -gt $retries ]]; then
+            die "Failed to run '$*' after ${retries} retries."
+        fi
+
+        echo -n "> ${i}/${retries} "
+        if run "${@}"; then
+            break
+        fi
+
+        sleep "${delay}"
+    done
 }
 
 check_package_manager_yum() {