Add alert for snapshotter failure in etcd-backup-restore

pkg/operation/botanist/component/etcd/monitoring.go

@@ -63,6 +63,7 @@ const (
 	monitoringMetricBackupRestoreValidationDurationSecondsBucket      = "etcdbr_validation_duration_seconds_bucket"
 	monitoringMetricBackupRestoreValidationDurationSecondsCount       = "etcdbr_validation_duration_seconds_count"
 	monitoringMetricBackupRestoreValidationDurationSecondsSum         = "etcdbr_validation_duration_seconds_sum"
+	monitoringMetricBackupRestoreSnapshotterFailure                   = "etcdbr_snapshotter_failure"
 
 	monitoringMetricProcessMaxFds              = "process_max_fds"
 	monitoringMetricProcessOpenFds             = "process_open_fds"
@@ -189,6 +190,19 @@ const (
     annotations:
       description: Etcd data restoration was triggered, but has failed.
       summary: Etcd data restoration failure.
+
+  # etcd backup failure alert
+  - alert: KubeEtcdBackupRestore{{ .Role }}Down
+    expr: (sum(up{job="` + monitoringPrometheusJobEtcdNamePrefix + `-{{ .role }}"}) - sum(up{job="` + monitoringPrometheusJobBackupRestoreNamePrefix + `-{{ .role }}"}) > 0) or (rate(` + monitoringMetricBackupRestoreSnapshotterFailure + `{job="` + monitoringPrometheusJobBackupRestoreNamePrefix + `-{{ .role }}"}[5m]) > 0)
+    for: 10m
+    labels:
+      service: etcd
+      severity: critical
+      type: seed
+      visibility: operator
+    annotations:
+      description: Etcd backup restore {{ .role }} process down or snapshotter failed with error. Backups will not be triggered unless backup restore is brought back up. This is unsafe behaviour and may cause data loss.
+      summary: Etcd backup restore {{ .role }} process down or snapshotter failed with error
   {{- end }}
 `
 )
@@ -234,6 +248,7 @@ var (
 		monitoringMetricBackupRestoreValidationDurationSecondsBucket,
 		monitoringMetricBackupRestoreValidationDurationSecondsCount,
 		monitoringMetricBackupRestoreValidationDurationSecondsSum,
+		monitoringMetricBackupRestoreSnapshotterFailure,
 		monitoringMetricProcessResidentMemoryBytes,
 		monitoringMetricProcessCPUSecondsTotal,
 	}

pkg/operation/botanist/component/etcd/monitoring_test.go

@@ -135,7 +135,7 @@ metric_relabel_configs:
   action: labeldrop
 - source_labels: [ __name__ ]
   action: keep
-  regex: ^(etcdbr_defragmentation_duration_seconds_bucket|etcdbr_defragmentation_duration_seconds_count|etcdbr_defragmentation_duration_seconds_sum|etcdbr_network_received_bytes|etcdbr_network_transmitted_bytes|etcdbr_restoration_duration_seconds_bucket|etcdbr_restoration_duration_seconds_count|etcdbr_restoration_duration_seconds_sum|etcdbr_snapshot_duration_seconds_bucket|etcdbr_snapshot_duration_seconds_count|etcdbr_snapshot_duration_seconds_sum|etcdbr_snapshot_gc_total|etcdbr_snapshot_latest_revision|etcdbr_snapshot_latest_timestamp|etcdbr_snapshot_required|etcdbr_validation_duration_seconds_bucket|etcdbr_validation_duration_seconds_count|etcdbr_validation_duration_seconds_sum|process_resident_memory_bytes|process_cpu_seconds_total)$`
+  regex: ^(etcdbr_defragmentation_duration_seconds_bucket|etcdbr_defragmentation_duration_seconds_count|etcdbr_defragmentation_duration_seconds_sum|etcdbr_network_received_bytes|etcdbr_network_transmitted_bytes|etcdbr_restoration_duration_seconds_bucket|etcdbr_restoration_duration_seconds_count|etcdbr_restoration_duration_seconds_sum|etcdbr_snapshot_duration_seconds_bucket|etcdbr_snapshot_duration_seconds_count|etcdbr_snapshot_duration_seconds_sum|etcdbr_snapshot_gc_total|etcdbr_snapshot_latest_revision|etcdbr_snapshot_latest_timestamp|etcdbr_snapshot_required|etcdbr_validation_duration_seconds_bucket|etcdbr_validation_duration_seconds_count|etcdbr_validation_duration_seconds_sum|etcdbr_snapshotter_failure|process_resident_memory_bytes|process_cpu_seconds_total)$`
 
 	alertingRulesNormal = `groups:
 - name: kube-etcd3-` + testRole + `.rules
@@ -289,6 +289,19 @@ metric_relabel_configs:
     annotations:
       description: Etcd data restoration was triggered, but has failed.
       summary: Etcd data restoration failure.
+
+  # etcd backup failure alert
+  - alert: KubeEtcdBackupRestore` + testROLE + `Down
+    expr: (sum(up{job="kube-etcd3-` + testRole + `"}) - sum(up{job="kube-etcd3-backup-restore-` + testRole + `"}) > 0) or (rate(etcdbr_snapshotter_failure{job="kube-etcd3-backup-restore-` + testRole + `"}[5m]) > 0)
+    for: 10m
+    labels:
+      service: etcd
+      severity: critical
+      type: seed
+      visibility: operator
+    annotations:
+      description: Etcd backup restore ` + testRole + ` process down or snapshotter failed with error. Backups will not be triggered unless backup restore is brought back up. This is unsafe behaviour and may cause data loss.
+      summary: Etcd backup restore ` + testRole + ` process down or snapshotter failed with error
 `
 
 	expectedAlertingRulesNormalWithoutBackup    = alertingRulesNormal + alertingRulesDefault

pkg/operation/botanist/component/etcd/testdata/monitoring_alertingrules_normal_with_backup.yaml

@@ -7,8 +7,9 @@ tests:
 - interval: 30s
   input_series:
   # KubeEtcdTestDown
+  # KubeEtcdBackupRestoreTestDown
   - series: 'up{job="kube-etcd3-test"}'
-    values: '0+0x30'
+    values: '0+0x30 1+0x40'
   # KubeEtcd3TestNoLeader
   - series: 'etcd_server_has_leader{job="kube-etcd3-test"}'
     values: '0+0x30'
@@ -32,6 +33,11 @@ tests:
   # KubeEtcdRestorationFailed
   - series: 'etcdbr_restoration_duration_seconds_count{job="kube-etcd3-backup-restore-test",succeeded="false"}'
     values: '0+0x7 1 2 2'
+  # KubeEtcdBackupRestoreTestDown
+  - series: 'up{job="kube-etcd3-backup-restore-test"}'
+    values: '0+0x60 1+0x10'
+  - series: 'etcdbr_snapshotter_failure{job="kube-etcd3-backup-restore-test"}'
+    values: '1+1x30 1+0x40'
   alert_rule_test:
   - eval_time: 15m
     alertname: KubeEtcdTestDown
@@ -131,3 +137,29 @@ tests:
       exp_annotations:
         description: Etcd data restoration was triggered, but has failed.
         summary: Etcd data restoration failure.
+  - eval_time: 16m
+    alertname: KubeEtcdBackupRestoreTestDown
+    exp_alerts:
+    - exp_labels:
+        job: kube-etcd3-backup-restore-test
+        service: etcd
+        severity: critical
+        type: seed
+        visibility: operator
+      exp_annotations:
+        description: Etcd backup restore test process down or snapshotter failed with error. Backups will not be triggered unless backup restore is brought back up. This is unsafe behaviour and may cause data loss.
+        summary: Etcd backup restore test process down or snapshotter failed with error
+  - eval_time: 30m
+    alertname: KubeEtcdBackupRestoreTestDown
+    exp_alerts:
+    - exp_labels:
+        service: etcd
+        severity: critical
+        type: seed
+        visibility: operator
+      exp_annotations:
+        description: Etcd backup restore test process down or snapshotter failed with error. Backups will not be triggered unless backup restore is brought back up. This is unsafe behaviour and may cause data loss.
+        summary: Etcd backup restore test process down or snapshotter failed with error
+  - eval_time: 35m
+    alertname: KubeEtcdBackupRestoreTestDown
+    exp_alerts: