From 0ff79aade38b20e8bc9a118c6072e52a58f3f1eb Mon Sep 17 00:00:00 2001
From: Yonas Habteab <yonas.habteab@icinga.com>
Date: Fri, 26 Apr 2024 12:17:49 +0200
Subject: [PATCH] Ignore state change events while in downtime/flapping state

---
 internal/icinga2/api_responses.go |  1 +
 internal/icinga2/client_api.go    | 25 ++++++++++++++++++++++++-
 2 files changed, 25 insertions(+), 1 deletion(-)

diff --git a/internal/icinga2/api_responses.go b/internal/icinga2/api_responses.go
index afd0cf831..2debafd02 100644
--- a/internal/icinga2/api_responses.go
+++ b/internal/icinga2/api_responses.go
@@ -132,6 +132,7 @@ type HostServiceRuntimeAttributes struct {
 	Groups                    []string    `json:"groups"`
 	State                     int         `json:"state"`
 	StateType                 int         `json:"state_type"`
+	IsFlapping                bool        `json:"flapping"`
 	LastCheckResult           CheckResult `json:"last_check_result"`
 	LastStateChange           UnixFloat   `json:"last_state_change"`
 	DowntimeDepth             int         `json:"downtime_depth"`
diff --git a/internal/icinga2/client_api.go b/internal/icinga2/client_api.go
index a7a057b07..24d6fcace 100644
--- a/internal/icinga2/client_api.go
+++ b/internal/icinga2/client_api.go
@@ -230,6 +230,16 @@ func (client *Client) checkMissedChanges(ctx context.Context, objType string, ca
 			return fmt.Errorf("querying API delivered a wrong object type %q", objQueriesResult.Type)
 		}
 
+		// Downtime suppresses all kind of events on that checkable, so skip it if it's in downtime.
+		if objQueriesResult.Attrs.DowntimeDepth > 0 {
+			continue
+		}
+
+		// If the checkable is not in downtime but in flapping state, all other states/events become irrelevant as well.
+		if objQueriesResult.Attrs.IsFlapping {
+			continue
+		}
+
 		// Only process HARD states
 		if objQueriesResult.Attrs.StateType == StateTypeSoft {
 			client.Logger.Debugf("Skipping SOFT event, %#v", objQueriesResult.Attrs)
@@ -428,7 +438,20 @@ func (client *Client) listenEventStream() error {
 		)
 		switch respT := resp.(type) {
 		case *StateChange:
-			// Only process HARD states
+			// Ignore any state changes while in downtime
+			if respT.DowntimeDepth > 0 {
+				stateType := "HARD"
+				if respT.StateType == StateTypeSoft {
+					stateType = "SOFT"
+				}
+
+				client.Logger.Debugf("Skipping %q state change while in downtime, %#v", stateType, respT)
+				continue
+			}
+
+			// Only process HARD states.
+			// The checkable might also be in the flapping state, but Icinga 2 doesn't expose "is_flapping" to the
+			// event streams for state change. So, the only thing we can do here is to ignore soft state changes.
 			if respT.StateType == StateTypeSoft {
 				client.Logger.Debugf("Skipping SOFT State Change, %#v", respT)
 				continue