prevent backup from silently failing

engines/restic.go

@@ -57,29 +57,35 @@ func (r *ResticEngine) Backup() (err error) {
 	err = util.Retry(3, r.init)
 	if err != nil {
 		err = fmt.Errorf("failed to create a secure repository: %v", err)
+		r.sendBackupStatus(1, v.Name)
 		return
 	}
 
 	err = util.Retry(3, r.resticBackup)
 	if err != nil {
 		err = fmt.Errorf("failed to backup the volume: %v", err)
+		r.sendBackupStatus(1, v.Name)
 		return
 	}
 
 	err = util.Retry(3, r.forget)
 	if err != nil {
 		err = fmt.Errorf("failed to forget the oldest snapshots: %v", err)
+		r.sendBackupStatus(1, v.Name)
 		return
 	}
 
 	if c.IsCheckScheduled(v) {
 		err = util.Retry(3, r.verify)
 		if err != nil {
 			err = fmt.Errorf("failed to verify backup: %v", err)
+			r.sendBackupStatus(1, v.Name)
 			return err
 		}
 	}
 
+	r.sendBackupStatus(0, v.Name)
+
 	return
 }
 
@@ -268,6 +274,22 @@ func (r *ResticEngine) snapshots() (snapshots []Snapshot, err error) {
 	return
 }
 
+// sendBackupStatus creates a metric which represents the backup status. 0 == OK / 1 == KO
+func (r *ResticEngine) sendBackupStatus(status int, volume string) {
+	metric := r.Volume.MetricsHandler.NewMetric("bivac_backupExitCode", "gauge")
+	err := metric.UpdateEvent(
+		&metrics.Event{
+			Labels: map[string]string{
+				"volume": volume,
+			},
+			Value: strconv.Itoa(status),
+		},
+	)
+	if err != nil {
+		log.Errorf("failed to send metric: %v", err)
+	}
+}
+
 // launchRestic starts a restic container with the given command
 func (r *ResticEngine) launchRestic(cmd []string, volumes []*volume.Volume) (state int, stdout string, err error) {
 	config := r.Orchestrator.GetHandler().Config