diff --git a/cmd/channel/email/main.go b/cmd/channel/email/main.go index cc13a9a7..d826c1f7 100644 --- a/cmd/channel/email/main.go +++ b/cmd/channel/email/main.go @@ -54,7 +54,7 @@ func (ch *Email) SendNotification(req *plugin.NotificationRequest) error { return enmime.Builder(). ToAddrs(to). From(ch.SenderName, ch.SenderMail). - Subject(fmt.Sprintf("[#%d] %s %s is %s", req.Incident.Id, req.Event.Type, req.Object.Name, req.Incident.Severity)). + Subject(plugin.FormatSubject(req)). Header("Message-Id", fmt.Sprintf("<%s-%s>", uuid.New().String(), ch.SenderMail)). Text(msg.Bytes()). Send(ch) diff --git a/cmd/channel/rocketchat/main.go b/cmd/channel/rocketchat/main.go index 3c951bed..143c5b9e 100644 --- a/cmd/channel/rocketchat/main.go +++ b/cmd/channel/rocketchat/main.go @@ -23,7 +23,7 @@ func main() { func (ch *RocketChat) SendNotification(req *plugin.NotificationRequest) error { var output bytes.Buffer - _, _ = fmt.Fprintf(&output, "[#%d] %s %s is %s\n\n", req.Incident.Id, req.Event.Type, req.Object.Name, req.Incident.Severity) + _, _ = fmt.Fprint(&output, plugin.FormatSubject(req)+"\n\n") plugin.FormatMessage(&output, req) diff --git a/internal/event/event.go b/internal/event/event.go index 1abf70af..a3d60ad2 100644 --- a/internal/event/event.go +++ b/internal/event/event.go @@ -34,11 +34,52 @@ type Event struct { } const ( - TypeState = "state" - TypeAcknowledgement = "acknowledgement" - TypeInternal = "internal" + TypeState = "state" + TypeAcknowledgementSet = "acknowledgement-set" + TypeAcknowledgementCleared = "acknowledgement-cleared" + TypeInternal = "internal" + TypeDowntimeRemoved = "downtime-removed" + TypeDowntimeStart = "downtime-start" + TypeDowntimeEnd = "downtime-end" + TypeCustom = "custom" + TypeFlappingStart = "flapping-start" + TypeFlappingEnd = "flapping-end" ) +// Validate validates the current event state. +// Returns an error if it detects a misconfigured field. +func (e *Event) Validate() error { + if len(e.Tags) == 0 { + return fmt.Errorf("invalid event: tags must not be empty") + } + + if e.SourceId == 0 { + return fmt.Errorf("invalid event: source ID must not be empty") + } + + if e.Severity != SeverityNone && e.Type != TypeState { + return fmt.Errorf("invalid event: if 'severity' is set, 'type' must be set to %q", TypeState) + } + + switch e.Type { + case "": + return fmt.Errorf("invalid event: 'type' must not be empty") + case TypeState, + TypeAcknowledgementSet, + TypeAcknowledgementCleared, + TypeInternal, + TypeDowntimeRemoved, + TypeDowntimeStart, + TypeDowntimeEnd, + TypeCustom, + TypeFlappingStart, + TypeFlappingEnd: + return nil + default: + return fmt.Errorf("invalid event: unsupported event type %q", e.Type) + } +} + func (e *Event) String() string { return fmt.Sprintf("[time=%s type=%q severity=%s]", e.Time, e.Type, e.Severity.String()) } diff --git a/internal/icinga2/api_responses.go b/internal/icinga2/api_responses.go index 0a0b2624..cbb9f0c9 100644 --- a/internal/icinga2/api_responses.go +++ b/internal/icinga2/api_responses.go @@ -91,6 +91,21 @@ type Downtime struct { Service string `json:"service_name"` Author string `json:"author"` Comment string `json:"comment"` + + // RemoveTime is used to indicate whether a downtime was ended automatically or cancelled prematurely by a user. + // It is set to zero time for the former case, otherwise to the timestamp at which time has been cancelled. + RemoveTime UnixFloat `json:"remove_time"` + + // IsFixed is used to differentiate between fixed and flexible downtimes. + // Fixed downtimes always emits a start and triggered event and cause two notifications being sent + // for the very (same) event. Flexible downtimes, on the other hand, only emits a trigger event, and + // don't produce duplicates for the same event. + IsFixed bool `json:"fixed"` +} + +// WasCancelled returns true when the current downtime was cancelled prematurely by a user. +func (d *Downtime) WasCancelled() bool { + return d.RemoveTime.Time().After(time.UnixMilli(0)) } // HostServiceRuntimeAttributes are common attributes of both Host and Service objects. @@ -141,6 +156,7 @@ const ( typeDowntimeRemoved = "DowntimeRemoved" typeDowntimeStarted = "DowntimeStarted" typeDowntimeTriggered = "DowntimeTriggered" + typeFlapping = "Flapping" ) // StateChange represents the Icinga 2 API Event Stream StateChange response for host/service state changes. @@ -244,6 +260,19 @@ type DowntimeTriggered struct { Downtime Downtime `json:"downtime"` } +// Flapping represents the Icinga 2 API Event Stream Flapping response for flapping host/services. +// +// NOTE: +// - An empty Service field indicates a host being in flapping state. +// +// https://icinga.com/docs/icinga-2/latest/doc/12-icinga2-api/#event-stream-type-flapping +type Flapping struct { + Timestamp UnixFloat `json:"timestamp"` + Host string `json:"host"` + Service string `json:"service"` + IsFlapping bool `json:"is_flapping"` +} + // UnmarshalEventStreamResponse unmarshal a JSON response line from the Icinga 2 API Event Stream. // // The function expects an Icinga 2 API Event Stream Response in its JSON form and tries to unmarshal it into one of the @@ -281,6 +310,8 @@ func UnmarshalEventStreamResponse(bytes []byte) (any, error) { resp = new(DowntimeStarted) case typeDowntimeTriggered: resp = new(DowntimeTriggered) + case typeFlapping: + resp = new(Flapping) default: return nil, fmt.Errorf("unsupported type %q", responseType) } diff --git a/internal/icinga2/api_responses_test.go b/internal/icinga2/api_responses_test.go index 04a35c7e..ef817275 100644 --- a/internal/icinga2/api_responses_test.go +++ b/internal/icinga2/api_responses_test.go @@ -128,9 +128,46 @@ func TestObjectQueriesResult_UnmarshalJSON(t *testing.T) { Name: "dummy-11!af73f9d9-2ed8-45f8-b541-cce3f3fe0f6c", Type: "Downtime", Attrs: Downtime{ - Host: "dummy-11", - Author: "icingaadmin", - Comment: "turn down for what", + Host: "dummy-11", + Author: "icingaadmin", + Comment: "turn down for what", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, + }, + }, + }, + { + // $ curl -k -s -u root:icinga 'https://localhost:5665/v1/objects/downtimes' | jq -c '[.results[] | select(.attrs.fixed == false)][1]' + name: "flexible-downtime-host", + jsonData: `{"attrs":{"__name":"dummy-7!691d508b-c93f-4565-819c-3e46ffef1555","active":true,"author":"icingaadmin","authoritative_zone":"","comment":"Flexible","config_owner":"","config_owner_hash":"","duration":7200,"end_time":1714043658,"entry_time":1714040073.241627,"fixed":false,"ha_mode":0,"host_name":"dummy-7","legacy_id":4,"name":"691d508b-c93f-4565-819c-3e46ffef1555","original_attributes":null,"package":"_api","parent":"","paused":false,"remove_time":0,"scheduled_by":"","service_name":"","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/e5ec468f-6d29-4055-9cd4-495dbbef16e3/conf.d/downtimes/dummy-7!691d508b-c93f-4565-819c-3e46ffef1555.conf"},"start_time":1714040058,"templates":["691d508b-c93f-4565-819c-3e46ffef1555"],"trigger_time":1714040073.241627,"triggered_by":"","triggers":[],"type":"Downtime","version":1714040073.241642,"was_cancelled":false,"zone":"master"},"joins":{},"meta":{},"name":"dummy-7!691d508b-c93f-4565-819c-3e46ffef1555","type":"Downtime"}`, + resp: &ObjectQueriesResult[Downtime]{}, + expected: &ObjectQueriesResult[Downtime]{ + Name: "dummy-7!691d508b-c93f-4565-819c-3e46ffef1555", + Type: "Downtime", + Attrs: Downtime{ + Host: "dummy-7", + Author: "icingaadmin", + Comment: "Flexible", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: false, + }, + }, + }, + { + // $ curl -k -s -u root:icinga 'https://localhost:5665/v1/objects/downtimes' | jq -c '[.results[] | select(.attrs.fixed == false)][0]' + name: "flexible-downtime-service", + jsonData: `{"attrs":{"__name":"docker-master!disk /!97078a44-8902-495a-9f2a-c1f6802bc63d","active":true,"author":"icingaadmin","authoritative_zone":"","comment":"Flexible","config_owner":"","config_owner_hash":"","duration":7200,"end_time":1714042731,"entry_time":1714039143.459298,"fixed":false,"ha_mode":0,"host_name":"docker-master","legacy_id":3,"name":"97078a44-8902-495a-9f2a-c1f6802bc63d","original_attributes":null,"package":"_api","parent":"","paused":false,"remove_time":0,"scheduled_by":"","service_name":"disk /","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/e5ec468f-6d29-4055-9cd4-495dbbef16e3/conf.d/downtimes/docker-master!disk %2F!97078a44-8902-495a-9f2a-c1f6802bc63d.conf"},"start_time":1714039131,"templates":["97078a44-8902-495a-9f2a-c1f6802bc63d"],"trigger_time":1714039143.459298,"triggered_by":"","triggers":[],"type":"Downtime","version":1714039143.459324,"was_cancelled":false,"zone":""},"joins":{},"meta":{},"name":"docker-master!disk /!97078a44-8902-495a-9f2a-c1f6802bc63d","type":"Downtime"}`, + resp: &ObjectQueriesResult[Downtime]{}, + expected: &ObjectQueriesResult[Downtime]{ + Name: "docker-master!disk /!97078a44-8902-495a-9f2a-c1f6802bc63d", + Type: "Downtime", + Attrs: Downtime{ + Host: "docker-master", + Service: "disk /", + Author: "icingaadmin", + Comment: "Flexible", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: false, }, }, }, @@ -143,10 +180,12 @@ func TestObjectQueriesResult_UnmarshalJSON(t *testing.T) { Name: "docker-master!load!c27b27c2-e0ab-45ff-8b9b-e95f29851eb0", Type: "Downtime", Attrs: Downtime{ - Host: "docker-master", - Service: "load", - Author: "icingaadmin", - Comment: "Scheduled downtime for backup", + Host: "docker-master", + Service: "load", + Author: "icingaadmin", + Comment: "Scheduled downtime for backup", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, }, }, }, @@ -173,7 +212,7 @@ func TestObjectQueriesResult_UnmarshalJSON(t *testing.T) { LastStateChange: UnixFloat(time.UnixMicro(1697099900637215)), DowntimeDepth: 0, Acknowledgement: AcknowledgementNone, - AcknowledgementLastChange: UnixFloat(time.UnixMicro(0)), + AcknowledgementLastChange: UnixFloat(time.UnixMilli(0)), }, }, }, @@ -229,7 +268,7 @@ func TestObjectQueriesResult_UnmarshalJSON(t *testing.T) { LastStateChange: UnixFloat(time.UnixMicro(1697704135756310)), DowntimeDepth: 0, Acknowledgement: AcknowledgementNone, - AcknowledgementLastChange: UnixFloat(time.UnixMicro(0)), + AcknowledgementLastChange: UnixFloat(time.UnixMilli(0)), }, }, }, @@ -416,9 +455,11 @@ func TestApiResponseUnmarshal(t *testing.T) { expected: &DowntimeAdded{ Timestamp: UnixFloat(time.UnixMicro(1697207050511293)), Downtime: Downtime{ - Host: "dummy-157", - Author: "icingaadmin", - Comment: "updates", + Host: "dummy-157", + Author: "icingaadmin", + Comment: "updates", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, }, }, }, @@ -428,10 +469,12 @@ func TestApiResponseUnmarshal(t *testing.T) { expected: &DowntimeAdded{ Timestamp: UnixFloat(time.UnixMicro(1697207141217425)), Downtime: Downtime{ - Host: "docker-master", - Service: "http", - Author: "icingaadmin", - Comment: "broken until Monday", + Host: "docker-master", + Service: "http", + Author: "icingaadmin", + Comment: "broken until Monday", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, }, }, }, @@ -441,9 +484,11 @@ func TestApiResponseUnmarshal(t *testing.T) { expected: &DowntimeStarted{ Timestamp: UnixFloat(time.UnixMicro(1697207050511378)), Downtime: Downtime{ - Host: "dummy-157", - Author: "icingaadmin", - Comment: "updates", + Host: "dummy-157", + Author: "icingaadmin", + Comment: "updates", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, }, }, }, @@ -453,10 +498,12 @@ func TestApiResponseUnmarshal(t *testing.T) { expected: &DowntimeStarted{ Timestamp: UnixFloat(time.UnixMicro(1697207141217507)), Downtime: Downtime{ - Host: "docker-master", - Service: "http", - Author: "icingaadmin", - Comment: "broken until Monday", + Host: "docker-master", + Service: "http", + Author: "icingaadmin", + Comment: "broken until Monday", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, }, }, }, @@ -466,9 +513,25 @@ func TestApiResponseUnmarshal(t *testing.T) { expected: &DowntimeTriggered{ Timestamp: UnixFloat(time.UnixMicro(1697207050511608)), Downtime: Downtime{ - Host: "dummy-157", - Author: "icingaadmin", - Comment: "updates", + Host: "dummy-157", + Author: "icingaadmin", + Comment: "updates", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, + }, + }, + }, + { + name: "flexible-downtimetriggered-host", + jsonData: `{"downtime":{"__name":"dummy-7!691d508b-c93f-4565-819c-3e46ffef1555","author":"icingaadmin","authoritative_zone":"","comment":"Flexible","config_owner":"","config_owner_hash":"","duration":7200,"end_time":1714043658,"entry_time":1714040073.241627,"fixed":false,"host_name":"dummy-7","legacy_id":4,"name":"691d508b-c93f-4565-819c-3e46ffef1555","package":"_api","parent":"","remove_time":0,"scheduled_by":"","service_name":"","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/e5ec468f-6d29-4055-9cd4-495dbbef16e3/conf.d/downtimes/dummy-7!691d508b-c93f-4565-819c-3e46ffef1555.conf"},"start_time":1714040058,"templates":["691d508b-c93f-4565-819c-3e46ffef1555"],"trigger_time":0,"triggered_by":"","triggers":[],"type":"Downtime","version":1714040073.241642,"zone":"master"},"timestamp":1714040073.242575,"type":"DowntimeAdded"}`, + expected: &DowntimeTriggered{ + Timestamp: UnixFloat(time.UnixMicro(1714040073242575)), + Downtime: Downtime{ + Host: "dummy-7", + Author: "icingaadmin", + Comment: "Flexible", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: false, }, }, }, @@ -478,10 +541,41 @@ func TestApiResponseUnmarshal(t *testing.T) { expected: &DowntimeTriggered{ Timestamp: UnixFloat(time.UnixMicro(1697207141217726)), Downtime: Downtime{ - Host: "docker-master", - Service: "http", - Author: "icingaadmin", - Comment: "broken until Monday", + Host: "docker-master", + Service: "http", + Author: "icingaadmin", + Comment: "broken until Monday", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, + }, + }, + }, + { + name: "flexible-downtimetriggered-service", + jsonData: `{"downtime":{"__name":"docker-master!disk /!97078a44-8902-495a-9f2a-c1f6802bc63d","author":"icingaadmin","authoritative_zone":"","comment":"Flexible","config_owner":"","config_owner_hash":"","duration":7200,"end_time":1714042731,"entry_time":1714039143.459298,"fixed":false,"host_name":"docker-master","legacy_id":3,"name":"97078a44-8902-495a-9f2a-c1f6802bc63d","package":"_api","parent":"","remove_time":0,"scheduled_by":"","service_name":"disk /","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/e5ec468f-6d29-4055-9cd4-495dbbef16e3/conf.d/downtimes/docker-master!disk %2F!97078a44-8902-495a-9f2a-c1f6802bc63d.conf"},"start_time":1714039131,"templates":["97078a44-8902-495a-9f2a-c1f6802bc63d"],"trigger_time":1714039143.459298,"triggered_by":"","triggers":[],"type":"Downtime","version":1714039143.459324,"zone":""},"timestamp":1714039143.460918,"type":"DowntimeTriggered"}`, + expected: &DowntimeTriggered{ + Timestamp: UnixFloat(time.UnixMicro(1714039143460918)), + Downtime: Downtime{ + Host: "docker-master", + Service: "disk /", + Author: "icingaadmin", + Comment: "Flexible", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: false, + }, + }, + }, + { + name: "downtimeended-host", + jsonData: `{"downtime":{"__name":"dummy-157!e5d4d4ac-615a-4995-ab8f-09d9cd9503b1","author":"icingaadmin","authoritative_zone":"","comment":"updates","config_owner":"","config_owner_hash":"","duration":0,"end_time":1697210639,"entry_time":1697207050.509957,"fixed":true,"host_name":"dummy-157","legacy_id":3,"name":"e5d4d4ac-615a-4995-ab8f-09d9cd9503b1","package":"_api","parent":"","remove_time":0.0,"scheduled_by":"","service_name":"","source_location":{"first_column":0,"first_line":1,"last_column":69,"last_line":1,"path":"/var/lib/icinga2/api/packages/_api/997346d3-374d-443f-b734-80789fd59b31/conf.d/downtimes/dummy-157!e5d4d4ac-615a-4995-ab8f-09d9cd9503b1.conf"},"start_time":1697207039,"templates":["e5d4d4ac-615a-4995-ab8f-09d9cd9503b1"],"trigger_time":1697207050.509957,"triggered_by":"","triggers":[],"type":"Downtime","version":1697207050.509971,"zone":"master"},"timestamp":1697207096.187866,"type":"DowntimeRemoved"}`, + expected: &DowntimeRemoved{ + Timestamp: UnixFloat(time.UnixMicro(1697207096187866)), + Downtime: Downtime{ + Host: "dummy-157", + Author: "icingaadmin", + Comment: "updates", + RemoveTime: UnixFloat(time.UnixMilli(0)), + IsFixed: true, }, }, }, @@ -491,9 +585,11 @@ func TestApiResponseUnmarshal(t *testing.T) { expected: &DowntimeRemoved{ Timestamp: UnixFloat(time.UnixMicro(1697207096187866)), Downtime: Downtime{ - Host: "dummy-157", - Author: "icingaadmin", - Comment: "updates", + Host: "dummy-157", + Author: "icingaadmin", + Comment: "updates", + RemoveTime: UnixFloat(time.UnixMicro(1697207096187718)), + IsFixed: true, }, }, }, @@ -503,10 +599,12 @@ func TestApiResponseUnmarshal(t *testing.T) { expected: &DowntimeRemoved{ Timestamp: UnixFloat(time.UnixMicro(1697207144746333)), Downtime: Downtime{ - Host: "docker-master", - Service: "http", - Author: "icingaadmin", - Comment: "broken until Monday", + Host: "docker-master", + Service: "http", + Author: "icingaadmin", + Comment: "broken until Monday", + RemoveTime: UnixFloat(time.UnixMicro(1697207144746117)), + IsFixed: true, }, }, }, diff --git a/internal/icinga2/client.go b/internal/icinga2/client.go index 62dc0d3a..b18dbf1a 100644 --- a/internal/icinga2/client.go +++ b/internal/icinga2/client.go @@ -176,19 +176,62 @@ func (client *Client) buildHostServiceEvent(ctx context.Context, result CheckRes } // buildAcknowledgementEvent from the given fields. -func (client *Client) buildAcknowledgementEvent(ctx context.Context, host, service, author, comment string) (*event.Event, error) { +func (client *Client) buildAcknowledgementEvent( + ctx context.Context, host, service, author, comment string, clearEvent bool, +) (*event.Event, error) { ev, err := client.buildCommonEvent(ctx, host, service) if err != nil { return nil, err } - ev.Type = event.TypeAcknowledgement + if clearEvent { + ev.Type = event.TypeAcknowledgementCleared + } else { + ev.Type = event.TypeAcknowledgementSet + } + ev.Username = author ev.Message = comment return ev, nil } +// buildDowntimeEvent from the given fields. +func (client *Client) buildDowntimeEvent(ctx context.Context, d Downtime, startEvent bool) (*event.Event, error) { + ev, err := client.buildCommonEvent(ctx, d.Host, d.Service) + if err != nil { + return nil, err + } + + if startEvent { + ev.Type = event.TypeDowntimeStart + } else if !d.WasCancelled() { + ev.Type = event.TypeDowntimeEnd + } else { + ev.Type = event.TypeDowntimeRemoved + } + + ev.Username = d.Author + ev.Message = d.Comment + + return ev, nil +} + +// buildFlappingEvent from the given fields. +func (client *Client) buildFlappingEvent(ctx context.Context, host, service string, isFlapping bool) (*event.Event, error) { + ev, err := client.buildCommonEvent(ctx, host, service) + if err != nil { + return nil, err + } + + ev.Type = event.TypeFlappingStart + if !isFlapping { + ev.Type = event.TypeFlappingEnd + } + + return ev, nil +} + // startCatchupWorkers launches goroutines for catching up the Icinga 2 API state. // // Each event will be sent to the returned channel. When all launched workers have finished - either because all are diff --git a/internal/icinga2/client_api.go b/internal/icinga2/client_api.go index 33fc67e2..9f7ad4ee 100644 --- a/internal/icinga2/client_api.go +++ b/internal/icinga2/client_api.go @@ -267,7 +267,7 @@ func (client *Client) checkMissedChanges(ctx context.Context, objType string, ca ev, err = client.buildAcknowledgementEvent( ctx, hostName, serviceName, - ackComment.Author, ackComment.Text) + ackComment.Author, ackComment.Text, false) if err != nil { return fmt.Errorf("failed to construct Event from Acknowledgement response, %w", err) } @@ -390,13 +390,14 @@ func (client *Client) listenEventStream() error { eventStream, err := client.connectEventStream([]string{ typeStateChange, typeAcknowledgementSet, - // typeAcknowledgementCleared, + typeAcknowledgementCleared, // typeCommentAdded, // typeCommentRemoved, // typeDowntimeAdded, - // typeDowntimeRemoved, - // typeDowntimeStarted, - // typeDowntimeTriggered, + typeDowntimeRemoved, + typeDowntimeStarted, + typeDowntimeTriggered, + typeFlapping, }) if err != nil { return err @@ -435,18 +436,41 @@ func (client *Client) listenEventStream() error { ev, err = client.buildHostServiceEvent(client.Ctx, respT.CheckResult, respT.State, respT.Host, respT.Service) evTime = respT.Timestamp.Time() - case *AcknowledgementSet: - ev, err = client.buildAcknowledgementEvent(client.Ctx, respT.Host, respT.Service, respT.Author, respT.Comment) + ev, err = client.buildAcknowledgementEvent(client.Ctx, respT.Host, respT.Service, respT.Author, respT.Comment, false) + evTime = respT.Timestamp.Time() + case *AcknowledgementCleared: + ev, err = client.buildAcknowledgementEvent(client.Ctx, respT.Host, respT.Service, "", "", true) evTime = respT.Timestamp.Time() - - // case *AcknowledgementCleared: // case *CommentAdded: // case *CommentRemoved: // case *DowntimeAdded: - // case *DowntimeRemoved: - // case *DowntimeStarted: - // case *DowntimeTriggered: + case *DowntimeRemoved: + ev, err = client.buildDowntimeEvent(client.Ctx, respT.Downtime, false) + evTime = respT.Timestamp.Time() + case *DowntimeStarted: + if !respT.Downtime.IsFixed { + // This may never happen, but Icinga 2 does the same thing, and we need to ignore the start + // event for flexible downtime, as there will definitely be a triggered event for it. + client.Logger.Debugf("Skipping flexible downtime start event, %#v", respT) + continue + } + + ev, err = client.buildDowntimeEvent(client.Ctx, respT.Downtime, true) + evTime = respT.Timestamp.Time() + case *DowntimeTriggered: + if respT.Downtime.IsFixed { + // Fixed downtimes generate two events (start, triggered), the latter applies here and must + // be ignored, since we're going to process its start event to avoid duplicated notifications. + client.Logger.Debugf("Skipping fixed downtime triggered event, %#v", respT) + continue + } + + ev, err = client.buildDowntimeEvent(client.Ctx, respT.Downtime, true) + evTime = respT.Timestamp.Time() + case *Flapping: + ev, err = client.buildFlappingEvent(client.Ctx, respT.Host, respT.Service, respT.IsFlapping) + evTime = respT.Timestamp.Time() default: err = fmt.Errorf("unsupported type %T", resp) } diff --git a/internal/incident/incident.go b/internal/incident/incident.go index cff07774..070e3dd2 100644 --- a/internal/incident/incident.go +++ b/internal/incident/incident.go @@ -146,42 +146,34 @@ func (i *Incident) ProcessEvent(ctx context.Context, ev *event.Event, created bo return errors.New("can't insert incident event to the database") } - if ev.Type == event.TypeAcknowledgement { - if err = i.processAcknowledgementEvent(ctx, tx, ev); err != nil { - return err + switch ev.Type { + case event.TypeState: + if !created { + if err := i.processSeverityChangedEvent(ctx, tx, ev); err != nil { + return err + } } - if err = tx.Commit(); err != nil { - i.logger.Errorw("Can't commit db transaction", zap.Error(err)) - - return errors.New("can't commit db transaction") + // Check if any (additional) rules match this object. Filters of rules that already have a state don't have + // to be checked again, these rules already matched and stay effective for the ongoing incident. + err = i.evaluateRules(ctx, tx, ev.ID) + if err != nil { + return err } - return nil - } - - if !created { - err := i.processSeverityChangedEvent(ctx, tx, ev) + // Re-evaluate escalations based on the newly evaluated rules. + escalations, err := i.evaluateEscalations(ev.Time) if err != nil { return err } - } - - // Check if any (additional) rules match this object. Filters of rules that already have a state don't have - // to be checked again, these rules already matched and stay effective for the ongoing incident. - err = i.evaluateRules(ctx, tx, ev.ID) - if err != nil { - return err - } - - // Re-evaluate escalations based on the newly evaluated rules. - escalations, err := i.evaluateEscalations(ev.Time) - if err != nil { - return err - } - if err := i.triggerEscalations(ctx, tx, ev, escalations); err != nil { - return err + if err := i.triggerEscalations(ctx, tx, ev, escalations); err != nil { + return err + } + case event.TypeAcknowledgementSet: + if err := i.processAcknowledgementEvent(ctx, tx, ev); err != nil { + return err + } } notifications, err := i.addPendingNotifications(ctx, tx, ev, i.getRecipientsChannel(ev.Time)) @@ -566,7 +558,8 @@ func (i *Incident) notifyContact(contact *recipient.Contact, ev *event.Event, ch return fmt.Errorf("could not find config for channel ID: %d", chID) } - i.logger.Infow(fmt.Sprintf("Notify contact %q via %q of type %q", contact.FullName, ch.Name, ch.Type), zap.Int64("channel_id", chID)) + i.logger.Infow(fmt.Sprintf("Notify contact %q via %q of type %q", contact.FullName, ch.Name, ch.Type), + zap.Int64("channel_id", chID), zap.String("event_type", ev.Type)) err := ch.Notify(contact, i, ev, daemon.Config().Icingaweb2URL) if err != nil { @@ -574,9 +567,8 @@ func (i *Incident) notifyContact(contact *recipient.Contact, ev *event.Event, ch return err } - i.logger.Infow( - "Successfully sent a notification via channel plugin", zap.String("type", ch.Type), zap.String("contact", contact.FullName), - ) + i.logger.Infow("Successfully sent a notification via channel plugin", zap.String("type", ch.Type), + zap.String("contact", contact.FullName), zap.String("event_type", ev.Type)) return nil } diff --git a/internal/incident/incidents.go b/internal/incident/incidents.go index 1583c78f..d6191d94 100644 --- a/internal/incident/incidents.go +++ b/internal/incident/incidents.go @@ -231,9 +231,10 @@ func ProcessEvent( if currentIncident == nil { switch { - case ev.Type == event.TypeAcknowledgement: - return fmt.Errorf("%q does not have an active incident, ignoring acknowledgement event from source %d", - obj.DisplayName(), ev.SourceId) + // ignore non-state event without incident + case ev.Severity == event.SeverityNone: + return fmt.Errorf("%q does not have an active incident, ignoring %q event from source %d", + obj.DisplayName(), ev.Type, ev.SourceId) case ev.Severity != event.SeverityOK: panic(fmt.Sprintf("cannot process event %v with a non-OK state %v without a known incident", ev, ev.Severity)) default: diff --git a/internal/listener/listener.go b/internal/listener/listener.go index ad626fbb..73c0955e 100644 --- a/internal/listener/listener.go +++ b/internal/listener/listener.go @@ -116,35 +116,15 @@ func (l *Listener) ProcessEvent(w http.ResponseWriter, req *http.Request) { return } - if len(ev.Tags) == 0 { - abort(http.StatusBadRequest, &ev, "ignoring invalid event: tags cannot be empty") - return - } - ev.Time = time.Now() ev.SourceId = source.ID - - if ev.Severity == event.SeverityNone && ev.Type == "" { - abort(http.StatusBadRequest, &ev, "ignoring invalid event: must set 'type' or 'severity'") - return + if ev.Type == "" { + ev.Type = event.TypeState } - if ev.Severity != event.SeverityNone { - if ev.Type == "" { - ev.Type = event.TypeState - } else if ev.Type != event.TypeState { - abort(http.StatusBadRequest, &ev, - "ignoring invalid event: if 'severity' is set, 'type' must not be set or set to %q", event.TypeState) - return - } - } - - if ev.Severity == event.SeverityNone { - if ev.Type != event.TypeAcknowledgement { - // It's neither a state nor an acknowledgement event. - abort(http.StatusBadRequest, &ev, "received not a state/acknowledgement event, ignoring") - return - } + if err := ev.Validate(); err != nil { + abort(http.StatusBadRequest, &ev, err.Error()) + return } l.logger.Infow("Processing event", zap.String("event", ev.String())) @@ -154,6 +134,8 @@ func (l *Listener) ProcessEvent(w http.ResponseWriter, req *http.Request) { return } + l.logger.Infow("Successfully processed event", zap.String("event", ev.String())) + w.WriteHeader(http.StatusOK) _, _ = fmt.Fprintln(w, "event processed successfully") _, _ = fmt.Fprintln(w) diff --git a/pkg/plugin/plugin.go b/pkg/plugin/plugin.go index b2cb7ab0..9d3d1923 100644 --- a/pkg/plugin/plugin.go +++ b/pkg/plugin/plugin.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/icinga/icinga-notifications/internal/event" "github.com/icinga/icinga-notifications/internal/utils" "github.com/icinga/icinga-notifications/pkg/rpc" "github.com/icinga/icingadb/pkg/types" @@ -194,11 +195,19 @@ func RunPlugin(plugin Plugin) { // FormatMessage formats a notification message and adds to the given io.Writer func FormatMessage(writer io.Writer, req *NotificationRequest) { - _, _ = fmt.Fprintf(writer, "Info: %s\n\n", req.Event.Message) + if req.Event.Message != "" { + msgTitle := "Comment" + if req.Event.Type == event.TypeState { + msgTitle = "Output" + } + + _, _ = fmt.Fprintf(writer, "%s: %s\n\n", msgTitle, req.Event.Message) + } + _, _ = fmt.Fprintf(writer, "When: %s\n\n", req.Event.Time.Format("2006-01-02 15:04:05 MST")) if req.Event.Username != "" { - _, _ = fmt.Fprintf(writer, "Commented by %s\n\n", req.Event.Username) + _, _ = fmt.Fprintf(writer, "Author: %s\n\n", req.Event.Username) } _, _ = fmt.Fprintf(writer, "Object: %s\n\n", req.Object.Url) _, _ = writer.Write([]byte("Tags:\n")) @@ -217,3 +226,15 @@ func FormatMessage(writer io.Writer, req *NotificationRequest) { _, _ = fmt.Fprintf(writer, "\nIncident: %s", req.Incident.Url) } + +// FormatSubject returns the formatted subject string based on the event type. +func FormatSubject(req *NotificationRequest) string { + switch req.Event.Type { + case event.TypeState: + return fmt.Sprintf("[#%d] %s %s is %s", req.Incident.Id, req.Event.Type, req.Object.Name, req.Incident.Severity) + case event.TypeAcknowledgementCleared, event.TypeDowntimeRemoved: + return fmt.Sprintf("[#%d] %s from %s", req.Incident.Id, req.Event.Type, req.Object.Name) + default: + return fmt.Sprintf("[#%d] %s on %s", req.Incident.Id, req.Event.Type, req.Object.Name) + } +}