feat(alerting): extract logic state updates and notifications

ref #6444
grafana · Nov 7, 2016 · f0b591b · f0b591b
1 parent b887911
commit f0b591b
Show file tree

Hide file tree

Showing 4 changed files with 160 additions and 23 deletions.
diff --git a/pkg/services/alerting/eval_context.go b/pkg/services/alerting/eval_context.go
@@ -26,6 +26,7 @@ type EvalContext struct {
 	ImagePublicUrl  string
 	ImageOnDiskPath string
 	NoDataFound     bool
+	PrevAlertState  m.AlertStateType
 
 	Ctx context.Context
 }
@@ -63,6 +64,36 @@ func (c *EvalContext) GetStateModel() *StateDescription {
 	}
 }
 
+func (c *EvalContext) ShouldUpdateAlertState() bool {
+	return c.Rule.State != c.PrevAlertState
+}
+
+func (c *EvalContext) ShouldSendNotification() bool {
+	if (c.PrevAlertState == m.AlertStatePending) && (c.Rule.State == m.AlertStateOK) {
+		return false
+	}
+
+	alertState := c.Rule.State
+
+	if c.NoDataFound {
+		if c.Rule.NoDataState == m.NoDataKeepState {
+			return false
+		}
+
+		alertState = c.Rule.NoDataState.ToAlertState()
+	}
+
+	if c.Error != nil {
+		if c.Rule.ExecutionErrorState == m.NoDataKeepState {
+			return false
+		}
+
+		alertState = c.Rule.ExecutionErrorState.ToAlertState()
+	}
+
+	return alertState != c.PrevAlertState
+}
+
 func (a *EvalContext) GetDurationMs() float64 {
 	return float64(a.EndTime.Nanosecond()-a.StartTime.Nanosecond()) / float64(1000000)
 }

diff --git a/pkg/services/alerting/eval_context_test.go b/pkg/services/alerting/eval_context_test.go
@@ -0,0 +1,110 @@
+package alerting
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/grafana/grafana/pkg/models"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestAlertingEvalContext(t *testing.T) {
+	Convey("Eval context", t, func() {
+		ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
+		err := fmt.Errorf("Dummie error!")
+
+		Convey("Should update alert state", func() {
+
+			Convey("ok -> alerting", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.State = models.AlertStateAlerting
+
+				So(ctx.ShouldUpdateAlertState(), ShouldBeTrue)
+			})
+
+			Convey("ok -> ok", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.State = models.AlertStateOK
+
+				So(ctx.ShouldUpdateAlertState(), ShouldBeFalse)
+			})
+		})
+
+		Convey("Should send notifications", func() {
+			Convey("pending -> ok", func() {
+				ctx.PrevAlertState = models.AlertStatePending
+				ctx.Rule.State = models.AlertStateOK
+
+				So(ctx.ShouldSendNotification(), ShouldBeFalse)
+			})
+
+			Convey("ok -> alerting", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.State = models.AlertStateAlerting
+
+				So(ctx.ShouldSendNotification(), ShouldBeTrue)
+			})
+
+			Convey("alerting -> ok", func() {
+				ctx.PrevAlertState = models.AlertStateAlerting
+				ctx.Rule.State = models.AlertStateOK
+
+				So(ctx.ShouldSendNotification(), ShouldBeTrue)
+			})
+
+			Convey("ok -> no_data(alerting)", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.NoDataState = models.NoDataSetAlerting
+				ctx.Rule.State = models.AlertStateAlerting
+
+				So(ctx.ShouldSendNotification(), ShouldBeTrue)
+			})
+
+			Convey("ok -> no_data(ok)", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.NoDataState = models.NoDataSetOK
+				ctx.NoDataFound = true
+				ctx.Rule.State = models.AlertStateNoData
+
+				So(ctx.ShouldSendNotification(), ShouldBeFalse)
+			})
+
+			Convey("ok -> no_data(keep_last)", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.NoDataState = models.NoDataKeepState
+				ctx.Rule.State = models.AlertStateNoData
+				ctx.NoDataFound = true
+
+				So(ctx.ShouldSendNotification(), ShouldBeFalse)
+			})
+
+			Convey("ok -> execution_error(alerting)", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.State = models.AlertStateExecError
+				ctx.Rule.ExecutionErrorState = models.NoDataSetAlerting
+				ctx.Error = err
+
+				So(ctx.ShouldSendNotification(), ShouldBeTrue)
+			})
+
+			Convey("ok -> execution_error(ok)", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.State = models.AlertStateExecError
+				ctx.Rule.ExecutionErrorState = models.NoDataSetOK
+				ctx.Error = err
+
+				So(ctx.ShouldSendNotification(), ShouldBeFalse)
+			})
+
+			Convey("ok -> execution_error(keep_last)", func() {
+				ctx.PrevAlertState = models.AlertStateOK
+				ctx.Rule.State = models.AlertStateExecError
+				ctx.Rule.ExecutionErrorState = models.NoDataKeepState
+				ctx.Error = err
+
+				So(ctx.ShouldSendNotification(), ShouldBeFalse)
+			})
+		})
+	})
+}
diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go
@@ -28,7 +28,7 @@ func NewResultHandler() *DefaultResultHandler {
 }
 
 func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
-	oldState := evalContext.Rule.State
+	evalContext.PrevAlertState = evalContext.Rule.State
 
 	executionError := ""
 	annotationData := simplejson.New()
@@ -51,8 +51,8 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 	}
 
 	countStateResult(evalContext.Rule.State)
-	if handler.shouldUpdateAlertState(evalContext, oldState) {
-		handler.log.Info("New state change", "alertId", evalContext.Rule.Id, "newState", evalContext.Rule.State, "oldState", oldState)
+	if evalContext.ShouldUpdateAlertState() {
+		handler.log.Info("New state change", "alertId", evalContext.Rule.Id, "newState", evalContext.Rule.State, "prev state", evalContext.PrevAlertState)
 
 		cmd := &m.SetAlertStateCommand{
 			AlertId:  evalContext.Rule.Id,
@@ -76,7 +76,7 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 			Title:       evalContext.Rule.Name,
 			Text:        evalContext.GetStateModel().Text,
 			NewState:    string(evalContext.Rule.State),
-			PrevState:   string(oldState),
+			PrevState:   string(evalContext.PrevAlertState),
 			Epoch:       time.Now().Unix(),
 			Data:        annotationData,
 		}
@@ -86,21 +86,16 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
 			handler.log.Error("Failed to save annotation for new alert state", "error", err)
 		}
 
-		if (oldState == m.AlertStatePending) && (evalContext.Rule.State == m.AlertStateOK) {
-			handler.log.Info("Notfication not sent", "oldState", oldState, "newState", evalContext.Rule.State)
-		} else {
+		if evalContext.ShouldSendNotification() {
 			handler.notifier.Notify(evalContext)
+		} else {
+			handler.log.Info("Notfication not sent", "prev state", evalContext.PrevAlertState, "new state", evalContext.Rule.State)
 		}
-
 	}
 
 	return nil
 }
 
-func (handler *DefaultResultHandler) shouldUpdateAlertState(evalContext *EvalContext, oldState m.AlertStateType) bool {
-	return evalContext.Rule.State != oldState
-}
-
 func countStateResult(state m.AlertStateType) {
 	switch state {
 	case m.AlertStatePending:

diff --git a/pkg/services/alerting/rule.go b/pkg/services/alerting/rule.go
@@ -11,17 +11,18 @@ import (
 )
 
 type Rule struct {
-	Id            int64
-	OrgId         int64
-	DashboardId   int64
-	PanelId       int64
-	Frequency     int64
-	Name          string
-	Message       string
-	NoDataState   m.NoDataOption
-	State         m.AlertStateType
-	Conditions    []Condition
-	Notifications []int64
+	Id                  int64
+	OrgId               int64
+	DashboardId         int64
+	PanelId             int64
+	Frequency           int64
+	Name                string
+	Message             string
+	NoDataState         m.NoDataOption
+	ExecutionErrorState m.NoDataOption
+	State               m.AlertStateType
+	Conditions          []Condition
+	Notifications       []int64
 }
 
 type ValidationError struct {