feat: retry-system

docker-compose.yml

@@ -9,7 +9,7 @@ services:
       - .:/app
       - cargo-registry:/usr/local/cargo/registry
       - cargo-git:/usr/local/cargo/git
-#      - ./databases.json:/config/config.json
+      - ./databases.json:/config/config.json
       #- ./databases.toml:/config/config.toml
       - /var/run/docker.sock:/var/run/docker.sock
 #      - cargo-target:/app/target
@@ -21,9 +21,11 @@ services:
       LOG: debug
       TZ: "Europe/Paris"
 #      TMPDIR: /scratch
-      EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiZjlkZjhiNWYtM2I0MC00NWM3LWI3N2UtYzY4NzQ1YmU2NjMwIiwibWFzdGVyS2V5QjY0IjoiQlhWM1hvbEM2NTZTVjdkTmdjV1BHUWxrKytycExJNmxHRGk3Q1BCNWllbz0ifQ=="
+      EDGE_KEY: "eyJzZXJ2ZXJVcmwiOiJodHRwOi8vbG9jYWxob3N0Ojg4ODciLCJhZ2VudElkIjoiY2UxNjRiZDItZGZkMy00YzY4LThlZGItNmQ3OTczODAzZWEyIiwibWFzdGVyS2V5QjY0IjoiMUh0djdtWCtYVkJxL0IzUEV2WDlZZjlQeUdVZW5oRHlXemo5THRqNW90WT0ifQ=="
       #CHUNK_SIZE_MB: "1"
       #POOLING: 1
+      #RETRY_ATTEMPTS: 3
+      #RETRY_BACKOFF_MS: 1000
       #DATABASES_CONFIG_FILE: "config.toml"
     extra_hosts:
       - "localhost:host-gateway"

helm/templates/env-configmap.yaml

@@ -7,4 +7,6 @@ data:
   TZ: {{ .Values.env.TZ | quote }}
   POLLING: {{ .Values.env.POLLING | quote }}
   APP_ENV: {{ .Values.env.APP_ENV | quote }}
-  LOG: {{ .Values.env.LOG | quote }}
+  LOG: {{ .Values.env.LOG | quote }}
+  RETRY_ATTEMPTS: {{ .Values.env.RETRY_ATTEMPTS | quote }}
+  RETRY_BACKOFF_MS: {{ .Values.env.RETRY_BACKOFF_MS | quote }}

helm/values.yaml

@@ -11,6 +11,8 @@ env:
   POLLING: "5"
   APP_ENV: "production"
   LOG: "info"
+  RETRY_ATTEMPTS: "3"
+  RETRY_BACKOFF_MS: "1000"
 
 resources:
   limits:

src/services/backup/models.rs

@@ -1,6 +1,7 @@
 #![allow(dead_code)]
 
 use crate::services::config::DbType;
+use std::fmt::{self, Display, Formatter};
 use std::path::PathBuf;
 
 #[derive(Debug, Clone)]
@@ -20,3 +21,9 @@ pub struct UploadResult {
     pub remote_file_path: Option<String>,
     pub total_size: Option<u64>,
 }
+
+impl Display for UploadResult {
+    fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
+        write!(f, "{}", self.error.as_deref().unwrap_or("unknown error"))
+    }
+}

src/services/backup/runner.rs

@@ -4,6 +4,7 @@ use super::service::BackupService;
 
 use crate::domain::factory::DatabaseFactory;
 use crate::services::config::DatabaseConfig;
+use crate::utils::retry::{RetryPolicy, retry};
 
 use anyhow::Result;
 use std::path::Path;
@@ -39,7 +40,31 @@ impl BackupService {
             });
         }
 
-        match db.backup(tmp_path, Arc::clone(&logger)).await {
+        let policy = RetryPolicy::default();
+
+        let db_ref = &db;
+        let logger_ref = &logger;
+
+        let outcome = retry("Database backup", &logger, &policy, move |attempt| {
+            let dir = tmp_path.join(format!("attempt-{attempt}"));
+
+            async move {
+                if let Err(e) = tokio::fs::create_dir_all(&dir).await {
+                    return Err(anyhow::Error::from(e));
+                }
+
+                match db_ref.backup(&dir, Arc::clone(logger_ref)).await {
+                    Ok(f) => Ok(f),
+                    Err(e) => {
+                        let _ = tokio::fs::remove_dir_all(&dir).await;
+                        Err(e)
+                    }
+                }
+            }
+        })
+        .await;
+
+        match outcome {
             Ok(file) => Ok(BackupResult {
                 generated_id,
                 db_type,

src/services/backup/uploader.rs

@@ -4,6 +4,7 @@ use super::service::BackupService;
 use crate::services::api::models::agent::status::DatabaseStorage;
 use crate::services::storage;
 use crate::utils::common::BackupMethod;
+use crate::utils::retry::{RetryPolicy, retry};
 use anyhow::{Result, bail};
 use futures::future::join_all;
 use std::sync::Arc;
@@ -97,16 +98,44 @@ impl BackupService {
                 /*
                  STORAGE UPLOAD
                 */
-                let upload_result = provider
-                    .upload(
-                        ctx_clone.clone(),
-                        result_clone,
-                        method,
-                        &storage,
-                        Some(encrypt),
-                        &backup_storage_id,
+                let policy = RetryPolicy::default();
+
+                let attempt_result = if result_clone.backup_file.is_none() {
+                    logger_clone.log("error", format!("Missing backup file for storage {}", storage_id));
+
+                    Err(UploadResult {
+                        storage_id: storage_id.clone(),
+                        success: false,
+                        error: Some("Missing backup file path".into()),
+                        remote_file_path: None,
+                        total_size: None,
+                    })
+                } else {
+                    retry(
+                        &format!("Upload to storage {storage_id}"),
+                        &logger_clone,
+                        &policy,
+                        |_| async {
+                            let r = provider
+                                .upload(
+                                    ctx_clone.clone(),
+                                    result_clone.clone(),
+                                    method,
+                                    &storage,
+                                    Some(encrypt),
+                                    &backup_storage_id,
+                                )
+                                .await;
+
+                            if r.success { Ok(r) } else { Err(r) }
+                        },
                     )
-                    .await;
+                    .await
+                };
+
+                let upload_result = match attempt_result {
+                    Ok(r) | Err(r) => r,
+                };
 
                 let status = if upload_result.success { "success" } else { "failed" };
 
@@ -117,8 +146,6 @@ impl BackupService {
                         upload_result.error.as_deref().unwrap_or("unknown error")
                     ));
 
-                    // `backup_upload_init` opened a per-storage record; close it as "failed"
-                    // so the server is notified of the failure (no path/size on this path).
                     if let Err(err) = ctx_clone
                         .api
                         .backup_upload_status(

src/services/restore/downloader.rs

@@ -8,6 +8,7 @@ use std::sync::Arc;
 use std::time::Instant;
 use tokio::io::AsyncWriteExt;
 use crate::services::backup::logger::JobLogger;
+use crate::utils::retry::{RetryPolicy, retry};
 
 fn human_size(bytes: u64) -> String {
     if bytes >= 1024 * 1024 {
@@ -26,6 +27,34 @@ impl RestoreService {
         tmp_path: &Path,
         logger: Arc<JobLogger>,
         expected_size: Option<String>,
+    ) -> Result<PathBuf> {
+        let policy = RetryPolicy::default();
+
+        let logger_ref = &logger;
+
+        let outcome = retry("Backup download", &logger, &policy, move |_| {
+            let expected = expected_size.clone();
+
+            async move {
+                self.download_once(file_url, tmp_path, Arc::clone(logger_ref), expected)
+                    .await
+            }
+        })
+        .await;
+
+        if let Err(e) = &outcome {
+            logger.log("error", format!("Download failed: {e}"));
+        }
+
+        outcome
+    }
+
+    pub async fn download_once(
+        &self,
+        file_url: &str,
+        tmp_path: &Path,
+        logger: Arc<JobLogger>,
+        expected_size: Option<String>,
     ) -> Result<PathBuf> {
         logger.log("info", "Start downloading backup archive".to_string());
 

src/settings.rs

@@ -16,6 +16,8 @@ pub struct Settings {
     pub timezone: String,
     pub log: String,
     pub chunk_size: usize, // bytes
+    pub retry_attempts: u32,
+    pub retry_backoff_ms: u64,
 }
 
 impl Settings {
@@ -49,6 +51,24 @@ impl Settings {
 
         let chunk_size = chunk_size_mb * 1024 * 1024;
 
+        let retry_attempts = env::var("RETRY_ATTEMPTS")
+            .unwrap_or_else(|_| "3".to_string())
+            .parse::<u32>()
+            .expect("RETRY_ATTEMPTS must be a valid positive integer");
+
+        if retry_attempts < 3 || retry_attempts > 5 {
+            panic!("RETRY_ATTEMPTS must be between 3 and 5");
+        }
+
+        let retry_backoff_ms = env::var("RETRY_BACKOFF_MS")
+            .unwrap_or_else(|_| "1000".to_string())
+            .parse::<u64>()
+            .expect("RETRY_BACKOFF_MS must be a valid positive integer");
+
+        if retry_backoff_ms < 100 || retry_backoff_ms > 30_000 {
+            panic!("RETRY_BACKOFF_MS must be between 100 and 30000 milliseconds");
+        }
+
         let tz = env::var("TZ").unwrap_or_else(|_| "UTC".to_string());
 
         Self {
@@ -64,7 +84,9 @@ impl Settings {
             pooling: pooling_seconds,
             timezone: tz,
             log: env::var("LOG").unwrap_or_else(|_| "info".into()),
-            chunk_size
+            chunk_size,
+            retry_attempts,
+            retry_backoff_ms,
         }
     }
 }

src/tests/services/backup_runner_tests.rs

@@ -0,0 +1,68 @@
+use crate::services::backup::BackupService;
+use crate::services::backup::logger::JobLogger;
+use crate::services::config::{DatabaseConfig, DbType};
+use crate::tests::init_tracing_for_test;
+
+use std::collections::HashMap;
+use std::sync::Arc;
+use tempfile::TempDir;
+
+fn sqlite_config(path: &str) -> DatabaseConfig {
+    DatabaseConfig {
+        name: "retry-test".to_string(),
+        database: String::new(),
+        db_type: DbType::Sqlite,
+        username: String::new(),
+        password: String::new(),
+        port: 0,
+        host: String::new(),
+        generated_id: "retry-test-gen".to_string(),
+        path: path.to_string(),
+        max_packet_size: String::new(),
+        volume_name: String::new(),
+        container_name: None,
+        options: HashMap::new(),
+    }
+}
+
+#[tokio::test]
+async fn a_failing_backup_is_retried_and_leaves_no_attempt_directory() {
+    init_tracing_for_test();
+
+    let temp_dir = TempDir::new().unwrap();
+    let tmp_path = temp_dir.path();
+    let logger = Arc::new(JobLogger::new());
+
+    let cfg = sqlite_config("/nonexistent/definitely-not-here.sqlite");
+
+    let result = BackupService::run(cfg, tmp_path, Arc::clone(&logger))
+        .await
+        .unwrap();
+
+    assert_eq!(result.status, "failed");
+    assert!(result.backup_file.is_none());
+
+    let entries = Arc::try_unwrap(logger).unwrap().into_entries();
+    assert_eq!(
+        entries.iter().filter(|e| e.level == "warn").count(),
+        2,
+        "expected one warn per non-final failed attempt"
+    );
+    assert!(
+        entries
+            .iter()
+            .any(|e| e.level == "error" && e.message.starts_with("Backup failed:")),
+        "expected a single terminal error from the runner"
+    );
+
+    let leftovers: Vec<_> = std::fs::read_dir(tmp_path)
+        .unwrap()
+        .filter_map(|e| e.ok())
+        .filter(|e| e.file_name().to_string_lossy().starts_with("attempt-"))
+        .collect();
+    assert!(
+        leftovers.is_empty(),
+        "failed attempt directories must be cleaned up, found {:?}",
+        leftovers.iter().map(|e| e.file_name()).collect::<Vec<_>>()
+    );
+}