Skip to content

Commit

Permalink
feat(alerting): reduce states. Make exeuction result configurable.
Browse files Browse the repository at this point in the history 
ref #6444
  • Loading branch information
@bergquist
bergquist committed Nov 7, 2016
1 parent f0b591b commit 489f087
Show file tree
Hide file tree
Showing 10 changed files with 181 additions and 158 deletions.
4 changes: 1 addition & 3 deletions pkg/metrics/metrics.go
View file
Expand Up @@ -40,8 +40,7 @@ var (
M_Alerting_Result_State_Ok Counter
M_Alerting_Result_State_Paused Counter
M_Alerting_Result_State_NoData Counter
M_Alerting_Result_State_ExecError Counter
M_Alerting_Result_State_Pending Counter
M_Alerting_Result_State_Pending Counter
M_Alerting_Active_Alerts Counter
M_Alerting_Notification_Sent_Slack Counter
M_Alerting_Notification_Sent_Email Counter
Expand Down Expand Up @@ -102,7 +101,6 @@ func initMetricVars(settings *MetricSettings) {
M_Alerting_Result_State_Ok = RegCounter("alerting.result", "state", "ok")
M_Alerting_Result_State_Paused = RegCounter("alerting.result", "state", "paused")
M_Alerting_Result_State_NoData = RegCounter("alerting.result", "state", "no_data")
M_Alerting_Result_State_ExecError = RegCounter("alerting.result", "state", "exec_error")
M_Alerting_Result_State_Pending = RegCounter("alerting.result", "state", "pending")

M_Alerting_Active_Alerts = RegCounter("alerting.active_alerts")
Expand Down
30 changes: 21 additions & 9 deletions pkg/models/alert.go
View file
Expand Up @@ -9,35 +9,47 @@ import (
type AlertStateType string
type AlertSeverityType string
type NoDataOption string
type ExecutionErrorOption string

const (
AlertStateNoData AlertStateType = "no_data"
AlertStateExecError AlertStateType = "execution_error"
AlertStatePaused AlertStateType = "paused"
AlertStateAlerting AlertStateType = "alerting"
AlertStateOK AlertStateType = "ok"
AlertStatePending AlertStateType = "pending"
AlertStateNoData AlertStateType = "no_data"
AlertStatePaused AlertStateType = "paused"
AlertStateAlerting AlertStateType = "alerting"
AlertStateOK AlertStateType = "ok"
AlertStatePending AlertStateType = "pending"
)

const (
NoDataSetNoData NoDataOption = "no_data"
NoDataSetAlerting NoDataOption = "alerting"
NoDataSetOK NoDataOption = "ok"
NoDataKeepState NoDataOption = "keep_state"
)

const (
ExecutionErrorSetAlerting ExecutionErrorOption = "alerting"
ExecutionErrorKeepState ExecutionErrorOption = "keep_state"
)

func (s AlertStateType) IsValid() bool {
return s == AlertStateOK || s == AlertStateNoData || s == AlertStateExecError || s == AlertStatePaused || s == AlertStatePending
return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
}

func (s NoDataOption) IsValid() bool {
return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataSetOK || s == NoDataKeepState
return s == NoDataSetNoData || s == NoDataSetAlerting || s == NoDataKeepState
}

func (s NoDataOption) ToAlertState() AlertStateType {
return AlertStateType(s)
}

func (s ExecutionErrorOption) IsValid() bool {
return s == ExecutionErrorSetAlerting || s == ExecutionErrorKeepState
}

func (s ExecutionErrorOption) ToAlertState() AlertStateType {
return AlertStateType(s)
}

type Alert struct {
Id int64
Version int64
Expand Down
47 changes: 12 additions & 35 deletions pkg/services/alerting/eval_context.go
View file
Expand Up @@ -31,6 +31,17 @@ type EvalContext struct {
Ctx context.Context
}

func NewEvalContext(alertCtx context.Context, rule *Rule) *EvalContext {
return &EvalContext{
Ctx: alertCtx,
StartTime: time.Now(),
Rule: rule,
Logs: make([]*ResultLogEntry, 0),
EvalMatches: make([]*EvalMatch, 0),
log: log.New("alerting.evalContext"),
}
}

type StateDescription struct {
Color string
Text string
Expand All @@ -49,11 +60,6 @@ func (c *EvalContext) GetStateModel() *StateDescription {
Color: "#888888",
Text: "No Data",
}
case m.AlertStateExecError:
return &StateDescription{
Color: "#000",
Text: "Execution Error",
}
case m.AlertStateAlerting:
return &StateDescription{
Color: "#D63232",
Expand All @@ -73,25 +79,7 @@ func (c *EvalContext) ShouldSendNotification() bool {
return false
}

alertState := c.Rule.State

if c.NoDataFound {
if c.Rule.NoDataState == m.NoDataKeepState {
return false
}

alertState = c.Rule.NoDataState.ToAlertState()
}

if c.Error != nil {
if c.Rule.ExecutionErrorState == m.NoDataKeepState {
return false
}

alertState = c.Rule.ExecutionErrorState.ToAlertState()
}

return alertState != c.PrevAlertState
return true
}

func (a *EvalContext) GetDurationMs() float64 {
Expand Down Expand Up @@ -128,14 +116,3 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
return ruleUrl, nil
}
}

func NewEvalContext(alertCtx context.Context, rule *Rule) *EvalContext {
return &EvalContext{
Ctx: alertCtx,
StartTime: time.Now(),
Rule: rule,
Logs: make([]*ResultLogEntry, 0),
EvalMatches: make([]*EvalMatch, 0),
log: log.New("alerting.evalContext"),
}
}
62 changes: 0 additions & 62 deletions pkg/services/alerting/eval_context_test.go
View file
Expand Up @@ -2,7 +2,6 @@ package alerting

import (
"context"
"fmt"
"testing"

"github.com/grafana/grafana/pkg/models"
Expand All @@ -12,7 +11,6 @@ import (
func TestAlertingEvalContext(t *testing.T) {
Convey("Eval context", t, func() {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
err := fmt.Errorf("Dummie error!")

Convey("Should update alert state", func() {

Expand Down Expand Up @@ -45,66 +43,6 @@ func TestAlertingEvalContext(t *testing.T) {

So(ctx.ShouldSendNotification(), ShouldBeTrue)
})

Convey("alerting -> ok", func() {
ctx.PrevAlertState = models.AlertStateAlerting
ctx.Rule.State = models.AlertStateOK

So(ctx.ShouldSendNotification(), ShouldBeTrue)
})

Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.Rule.State = models.AlertStateAlerting

So(ctx.ShouldSendNotification(), ShouldBeTrue)
})

Convey("ok -> no_data(ok)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetOK
ctx.NoDataFound = true
ctx.Rule.State = models.AlertStateNoData

So(ctx.ShouldSendNotification(), ShouldBeFalse)
})

Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.Rule.State = models.AlertStateNoData
ctx.NoDataFound = true

So(ctx.ShouldSendNotification(), ShouldBeFalse)
})

Convey("ok -> execution_error(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataSetAlerting
ctx.Error = err

So(ctx.ShouldSendNotification(), ShouldBeTrue)
})

Convey("ok -> execution_error(ok)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataSetOK
ctx.Error = err

So(ctx.ShouldSendNotification(), ShouldBeFalse)
})

Convey("ok -> execution_error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.State = models.AlertStateExecError
ctx.Rule.ExecutionErrorState = models.NoDataKeepState
ctx.Error = err

So(ctx.ShouldSendNotification(), ShouldBeFalse)
})
})
})
}
53 changes: 37 additions & 16 deletions pkg/services/alerting/result_handler.go
View file
Expand Up @@ -27,27 +27,52 @@ func NewResultHandler() *DefaultResultHandler {
}
}

func (handler *DefaultResultHandler) GetStateFromEvaluation(evalContext *EvalContext) m.AlertStateType {
if evalContext.Error != nil {
handler.log.Error("Alert Rule Result Error",
"ruleId", evalContext.Rule.Id,
"name", evalContext.Rule.Name,
"error", evalContext.Error,
"changing state to", evalContext.Rule.ExecutionErrorState.ToAlertState())

if evalContext.Rule.ExecutionErrorState == m.ExecutionErrorKeepState {
return evalContext.PrevAlertState
} else {
return evalContext.Rule.ExecutionErrorState.ToAlertState()
}
} else if evalContext.Firing {
return m.AlertStateAlerting
} else if evalContext.NoDataFound {
handler.log.Info("Alert Rule returned no data",
"ruleId", evalContext.Rule.Id,
"name", evalContext.Rule.Name,
"changing state to", evalContext.Rule.NoDataState.ToAlertState())

if evalContext.Rule.NoDataState == m.NoDataKeepState {
return evalContext.PrevAlertState
} else {
return evalContext.Rule.NoDataState.ToAlertState()
}
}

return m.AlertStateOK
}

func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
evalContext.PrevAlertState = evalContext.Rule.State

executionError := ""
annotationData := simplejson.New()

evalContext.Rule.State = handler.GetStateFromEvaluation(evalContext)

if evalContext.Error != nil {
handler.log.Error("Alert Rule Result Error", "ruleId", evalContext.Rule.Id, "error", evalContext.Error)
evalContext.Rule.State = m.AlertStateExecError
executionError = evalContext.Error.Error()
annotationData.Set("errorMessage", executionError)
} else if evalContext.Firing {
evalContext.Rule.State = m.AlertStateAlerting
}

if evalContext.Firing {
annotationData = simplejson.NewFromAny(evalContext.EvalMatches)
} else {
if evalContext.NoDataFound {
if evalContext.Rule.NoDataState != m.NoDataKeepState {
evalContext.Rule.State = evalContext.Rule.NoDataState.ToAlertState()
}
} else {
evalContext.Rule.State = m.AlertStateOK
}
}

countStateResult(evalContext.Rule.State)
Expand Down Expand Up @@ -88,8 +113,6 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {

if evalContext.ShouldSendNotification() {
handler.notifier.Notify(evalContext)
} else {
handler.log.Info("Notfication not sent", "prev state", evalContext.PrevAlertState, "new state", evalContext.Rule.State)
}
}

Expand All @@ -108,7 +131,5 @@ func countStateResult(state m.AlertStateType) {
metrics.M_Alerting_Result_State_Paused.Inc(1)
case m.AlertStateNoData:
metrics.M_Alerting_Result_State_NoData.Inc(1)
case m.AlertStateExecError:
metrics.M_Alerting_Result_State_ExecError.Inc(1)
}
}

0 comments on commit 489f087

Please sign in to comment.