Skip to content

Commit

Permalink
fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@ifireice
ifireice committed Mar 27, 2019
1 parent 88be703 commit 712836c
Show file tree
Hide file tree
Showing 6 changed files with 300 additions and 279 deletions.
66 changes: 35 additions & 31 deletions checker/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ func (triggerChecker *TriggerChecker) compareTriggerStates(currentCheck moira.Ch

currentCheck.SuppressedState = lastStateSuppressedValue

needSend, message := needSendEvent(currentStateValue, lastStateValue, timestamp, triggerChecker.lastCheck.GetEventTimestamp(), lastStateSuppressed, lastStateSuppressedValue, triggerChecker.lastCheck.MaintenanceWho)
needSend, message := needSendEvent(currentStateValue, lastStateValue, timestamp, triggerChecker.lastCheck.GetEventTimestamp(), lastStateSuppressed, lastStateSuppressedValue, triggerChecker.lastCheck.MaintenanceInfo)
if !needSend {
return currentCheck, nil
}
Expand Down Expand Up @@ -88,7 +88,7 @@ func (triggerChecker *TriggerChecker) compareMetricStates(metric string, current

currentState.SuppressedState = lastState.SuppressedState

needSend, message := needSendEvent(currentState.State, lastState.State, currentState.Timestamp, lastState.GetEventTimestamp(), lastState.Suppressed, lastState.SuppressedState, currentState.MaintenanceWho)
needSend, message := needSendEvent(currentState.State, lastState.State, currentState.Timestamp, lastState.GetEventTimestamp(), lastState.Suppressed, lastState.SuppressedState, currentState.MaintenanceInfo)
if !needSend {
return currentState, nil
}
Expand Down Expand Up @@ -142,40 +142,13 @@ func (triggerChecker *TriggerChecker) isTriggerSuppressed(event *moira.Notificat
return false
}

func needSendEvent(currentStateValue moira.State, lastStateValue moira.State, currentStateTimestamp int64, lastStateEventTimestamp int64, isLastCheckSuppressed bool, lastStateSuppressedValue moira.State, maintenanceWho moira.MaintenanceWho) (needSend bool, message *string) {
func needSendEvent(currentStateValue moira.State, lastStateValue moira.State, currentStateTimestamp int64, lastStateEventTimestamp int64, isLastCheckSuppressed bool, lastStateSuppressedValue moira.State, maintenanceInfo moira.MaintenanceInfo) (needSend bool, message *string) {
if !isLastCheckSuppressed && currentStateValue != lastStateValue {
return true, nil
}

if isLastCheckSuppressed && currentStateValue != lastStateSuppressedValue {
messageBuffer := bytes.NewBuffer([]byte(""))
messageBuffer.WriteString("This metric changed its state during maintenance interval.")

if maintenanceWho.StartMaintenanceUser != nil || maintenanceWho.StartMaintenanceTime != nil {
messageBuffer.WriteString(" Maintenance was set")
if maintenanceWho.StartMaintenanceUser != nil {
messageBuffer.WriteString(" by user ")
messageBuffer.WriteString(*maintenanceWho.StartMaintenanceUser)
}
if maintenanceWho.StartMaintenanceTime != nil {
messageBuffer.WriteString(" at ")
messageBuffer.WriteString(time.Unix(*maintenanceWho.StartMaintenanceTime, 0).Format(format))
}
messageBuffer.WriteString(".")
}
if maintenanceWho.StopMaintenanceUser != nil || maintenanceWho.StopMaintenanceTime != nil {
messageBuffer.WriteString(" Maintenance removed")
if maintenanceWho.StopMaintenanceUser != nil {
messageBuffer.WriteString(" by user ")
messageBuffer.WriteString(*maintenanceWho.StopMaintenanceUser)
}
if maintenanceWho.StopMaintenanceTime != nil {
messageBuffer.WriteString(" at ")
messageBuffer.WriteString(time.Unix(*maintenanceWho.StopMaintenanceTime, 0).Format(format))
}
messageBuffer.WriteString(".")
}
message := messageBuffer.String()
message := getMaintenceCreateMessage(maintenanceInfo)
return true, &message
}
remindInterval, ok := badStateReminder[currentStateValue]
Expand All @@ -189,3 +162,34 @@ func needSendEvent(currentStateValue moira.State, lastStateValue moira.State, cu
func needRemindAgain(currentStateTimestamp, lastStateEventTimestamp, remindInterval int64) bool {
return currentStateTimestamp-lastStateEventTimestamp >= remindInterval
}

func getMaintenceCreateMessage (info moira.MaintenanceInfo) string {
messageBuffer := bytes.NewBuffer([]byte(""))
messageBuffer.WriteString("This metric changed its state during maintenance interval.")

if info.StartUser != nil || info.StartTime != nil {
messageBuffer.WriteString(" Maintenance was set")
if info.StartUser != nil {
messageBuffer.WriteString(" by user ")
messageBuffer.WriteString(*info.StartUser)
}
if info.StartTime != nil {
messageBuffer.WriteString(" at ")
messageBuffer.WriteString(time.Unix(*info.StartTime, 0).Format(format))
}
messageBuffer.WriteString(".")
}
if info.StopUser != nil || info.StopTime != nil {
messageBuffer.WriteString(" Maintenance removed")
if info.StopUser != nil {
messageBuffer.WriteString(" by user ")
messageBuffer.WriteString(*info.StopUser)
}
if info.StopTime != nil {
messageBuffer.WriteString(" at ")
messageBuffer.WriteString(time.Unix(*info.StopTime, 0).Format(format))
}
messageBuffer.WriteString(".")
}
return messageBuffer.String()
}
65 changes: 25 additions & 40 deletions checker/event_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -510,89 +510,72 @@ func TestTriggerMaintenance(t *testing.T) {
}

func TestNeedSendEvent(t *testing.T){

startMaintenanceUser := "testStartMtUser"
startMaintenanceTime := int64(123)
stopMaintenanceUser := "testStopMtUser"
stopMaintenanceTime := int64(1230)

Convey("Test needSendEvents for trigger", t, func(){

Convey("Start Maintenance not start user and time", func(){
actual := fmt.Sprintf("This metric changed its state during maintenance interval.")
needSend, message := needSendEvent(currentCheckTest.State, lastCheckTest.State, currentCheckTest.Timestamp, lastCheckTest.GetEventTimestamp(),lastCheckTest.Suppressed, lastCheckTest.SuppressedState, lastCheckTest.MaintenanceWho)
needSend, message := needSendEvent(currentCheckTest.State, lastCheckTest.State, currentCheckTest.Timestamp, lastCheckTest.GetEventTimestamp(),lastCheckTest.Suppressed, lastCheckTest.SuppressedState, lastCheckTest.MaintenanceInfo)
So(needSend, ShouldBeTrue)
So(*message, ShouldResemble, actual)
})
Convey("Start Maintenance", func(){
actual := fmt.Sprintf("This metric changed its state during maintenance interval. Maintenance was set by user %v at %v.", startMaintenanceUser, time.Unix(startMaintenanceTime, 0).Format("15:04 02.01.2006"))
lastCheckTest.MaintenanceWho.StartMaintenanceUser = &startMaintenanceUser
lastCheckTest.MaintenanceWho.StartMaintenanceTime = &startMaintenanceTime
needSend, message := needSendEvent(currentCheckTest.State, lastCheckTest.State, currentCheckTest.Timestamp, lastCheckTest.GetEventTimestamp(),lastCheckTest.Suppressed, lastCheckTest.SuppressedState, lastCheckTest.MaintenanceWho)
lastCheckTest.MaintenanceInfo.Set(&startMaintenanceUser, &startMaintenanceTime, nil, nil)
needSend, message := needSendEvent(currentCheckTest.State, lastCheckTest.State, currentCheckTest.Timestamp, lastCheckTest.GetEventTimestamp(),lastCheckTest.Suppressed, lastCheckTest.SuppressedState, lastCheckTest.MaintenanceInfo)
So(needSend, ShouldBeTrue)
So(*message, ShouldResemble, actual)
})
Convey("Stop Maintenance", func(){
actual := fmt.Sprintf("This metric changed its state during maintenance interval. Maintenance was set by user %v at %v. Maintenance removed by user %v at %v.", startMaintenanceUser, time.Unix(startMaintenanceTime, 0).Format("15:04 02.01.2006"), stopMaintenanceUser, time.Unix(stopMaintenanceTime, 0).Format("15:04 02.01.2006") )
lastCheckTest.MaintenanceWho.StartMaintenanceUser = &startMaintenanceUser
lastCheckTest.MaintenanceWho.StartMaintenanceTime = &startMaintenanceTime
lastCheckTest.MaintenanceWho.StopMaintenanceUser = &stopMaintenanceUser
lastCheckTest.MaintenanceWho.StopMaintenanceTime = &stopMaintenanceTime
needSend, message := needSendEvent(currentCheckTest.State, lastCheckTest.State, currentCheckTest.Timestamp, lastCheckTest.GetEventTimestamp(),lastCheckTest.Suppressed, lastCheckTest.SuppressedState, lastCheckTest.MaintenanceWho)
actual := fmt.Sprintf("This metric changed its state during maintenance interval. Maintenance was set by user %v at %v. Maintenance removed by user %v at %v.", startMaintenanceUser, time.Unix(startMaintenanceTime, 0).Format("15:04 02.01.2006"), stopMaintenanceUser, time.Unix(stopMaintenanceTime, 0).Format("15:04 02.01.2006"))
lastCheckTest.MaintenanceInfo.Set(&startMaintenanceUser, &startMaintenanceTime, &stopMaintenanceUser, &stopMaintenanceTime)
needSend, message := needSendEvent(currentCheckTest.State, lastCheckTest.State, currentCheckTest.Timestamp, lastCheckTest.GetEventTimestamp(),lastCheckTest.Suppressed, lastCheckTest.SuppressedState, lastCheckTest.MaintenanceInfo)
So(needSend, ShouldBeTrue)
So(*message, ShouldResemble, actual)
})
Convey("Stop Maintenance not start user and time", func(){
actual := fmt.Sprintf("This metric changed its state during maintenance interval. Maintenance removed by user %v at %v.", stopMaintenanceUser, time.Unix(stopMaintenanceTime, 0).Format("15:04 02.01.2006") )
lastCheckTest.MaintenanceWho.StartMaintenanceUser = nil
lastCheckTest.MaintenanceWho.StartMaintenanceTime = nil
lastCheckTest.MaintenanceWho.StopMaintenanceUser = &stopMaintenanceUser
lastCheckTest.MaintenanceWho.StopMaintenanceTime = &stopMaintenanceTime
needSend, message := needSendEvent(currentCheckTest.State, lastCheckTest.State, currentCheckTest.Timestamp, lastCheckTest.GetEventTimestamp(),lastCheckTest.Suppressed, lastCheckTest.SuppressedState, lastCheckTest.MaintenanceWho)
lastCheckTest.MaintenanceInfo.Set(nil, nil, &stopMaintenanceUser, &stopMaintenanceTime)
needSend, message := needSendEvent(currentCheckTest.State, lastCheckTest.State, currentCheckTest.Timestamp, lastCheckTest.GetEventTimestamp(),lastCheckTest.Suppressed, lastCheckTest.SuppressedState, lastCheckTest.MaintenanceInfo)
So(needSend, ShouldBeTrue)
So(*message, ShouldResemble, actual)
})
})

Convey("Test needSendEvents for metric", t, func(){

Convey("Start Maintenance not start user and time", func(){
actual := fmt.Sprintf("This metric changed its state during maintenance interval.")
needSend, message := needSendEvent(currenttMetricTest.State, lastMetricTest.State, currenttMetricTest.Timestamp, lastMetricTest.GetEventTimestamp(), lastMetricTest.Suppressed, lastMetricTest.SuppressedState, currenttMetricTest.MaintenanceWho)
needSend, message := needSendEvent(currentMetricTest.State, lastMetricsTest.State, currentMetricTest.Timestamp, lastMetricsTest.GetEventTimestamp(), lastMetricsTest.Suppressed, lastMetricsTest.SuppressedState, currentMetricTest.MaintenanceInfo)
So(needSend, ShouldBeTrue)
So(*message, ShouldResemble, actual)
})
Convey("Start Maintenance", func(){
actual := fmt.Sprintf("This metric changed its state during maintenance interval. Maintenance was set by user %v at %v.", startMaintenanceUser, time.Unix(startMaintenanceTime, 0).Format("15:04 02.01.2006"))
currenttMetricTest.MaintenanceWho.StartMaintenanceUser = &startMaintenanceUser
currenttMetricTest.MaintenanceWho.StartMaintenanceTime = &startMaintenanceTime
needSend, message := needSendEvent(currenttMetricTest.State, lastMetricTest.State, currenttMetricTest.Timestamp, lastMetricTest.GetEventTimestamp(), lastMetricTest.Suppressed, lastMetricTest.SuppressedState, currenttMetricTest.MaintenanceWho)
currentMetricTest.MaintenanceInfo.Set(&startMaintenanceUser, &startMaintenanceTime, nil, nil)
needSend, message := needSendEvent(currentMetricTest.State, lastMetricsTest.State, currentMetricTest.Timestamp, lastMetricsTest.GetEventTimestamp(), lastMetricsTest.Suppressed, lastMetricsTest.SuppressedState, currentMetricTest.MaintenanceInfo)
So(needSend, ShouldBeTrue)
So(*message, ShouldResemble, actual)
})
Convey("Stop Maintenance", func(){
actual := fmt.Sprintf("This metric changed its state during maintenance interval. Maintenance removed by user %v at %v.", stopMaintenanceUser, time.Unix(stopMaintenanceTime, 0).Format("15:04 02.01.2006") )
currenttMetricTest.MaintenanceWho.StartMaintenanceUser = nil
currenttMetricTest.MaintenanceWho.StartMaintenanceTime = nil
currenttMetricTest.MaintenanceWho.StopMaintenanceUser = &stopMaintenanceUser
currenttMetricTest.MaintenanceWho.StopMaintenanceTime = &stopMaintenanceTime
needSend, message := needSendEvent(currenttMetricTest.State, lastMetricTest.State, currenttMetricTest.Timestamp, lastMetricTest.GetEventTimestamp(), lastMetricTest.Suppressed, lastMetricTest.SuppressedState, currenttMetricTest.MaintenanceWho)
currentMetricTest.MaintenanceInfo.Set(nil, nil, &stopMaintenanceUser, &stopMaintenanceTime)
needSend, message := needSendEvent(currentMetricTest.State, lastMetricsTest.State, currentMetricTest.Timestamp, lastMetricsTest.GetEventTimestamp(), lastMetricsTest.Suppressed, lastMetricsTest.SuppressedState, currentMetricTest.MaintenanceInfo)
So(needSend, ShouldBeTrue)
So(*message, ShouldResemble, actual)
})
Convey("Stop Maintenance not start user and time", func(){
actual := fmt.Sprintf("This metric changed its state during maintenance interval. Maintenance was set by user %v at %v. Maintenance removed by user %v at %v.", startMaintenanceUser, time.Unix(startMaintenanceTime, 0).Format("15:04 02.01.2006"), stopMaintenanceUser, time.Unix(stopMaintenanceTime, 0).Format("15:04 02.01.2006") )
currenttMetricTest.MaintenanceWho.StartMaintenanceUser = &startMaintenanceUser
currenttMetricTest.MaintenanceWho.StartMaintenanceTime = &startMaintenanceTime
currenttMetricTest.MaintenanceWho.StopMaintenanceUser = &stopMaintenanceUser
currenttMetricTest.MaintenanceWho.StopMaintenanceTime = &stopMaintenanceTime
needSend, message := needSendEvent(currenttMetricTest.State, lastMetricTest.State, currenttMetricTest.Timestamp, lastMetricTest.GetEventTimestamp(), lastMetricTest.Suppressed, lastMetricTest.SuppressedState, currenttMetricTest.MaintenanceWho)
currentMetricTest.MaintenanceInfo.Set(&startMaintenanceUser, &startMaintenanceTime, &stopMaintenanceUser, &stopMaintenanceTime)
needSend, message := needSendEvent(currentMetricTest.State, lastMetricsTest.State, currentMetricTest.Timestamp, lastMetricsTest.GetEventTimestamp(), lastMetricsTest.Suppressed, lastMetricsTest.SuppressedState, currentMetricTest.MaintenanceInfo)
So(needSend, ShouldBeTrue)
So(*message, ShouldResemble, actual)
})
})

}

var lastCheckTest = moira.CheckData{
Score: 6000,
State: moira.StateOK,
Expand All @@ -607,15 +590,17 @@ var currentCheckTest = moira.CheckData {
Timestamp: 1504509981,
}

var lastMetricTest = moira.MetricState{
EventTimestamp: 1504449789,
State: moira.StateERROR,
Suppressed: true,
Timestamp: 1504509380,
}
var currenttMetricTest = moira.MetricState{
var currentMetricTest = moira.MetricState{
EventTimestamp: 1504449789,
State: moira.StateWARN,
Suppressed: true,
Timestamp: 1504509380,
}

var lastMetricsTest = moira.MetricState {
EventTimestamp: 1504449789,
State: moira.StateNODATA,
Suppressed: true,
Timestamp: 1504509380,
Maintenance: 1552723340,
}
22 changes: 2 additions & 20 deletions database/redis/last_check.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,14 +103,12 @@ func (connector *DbConnector) SetTriggerCheckMaintenance(triggerID string, metri
if !ok {
continue
}
setMaintenanceUserAndTime(&data, &value, userLogin, timeCallMaintenance)
data.Maintenance = value
moira.SetMaintenanceUserAndTime(&data, value, userLogin, timeCallMaintenance)
metricsCheck[metric] = data
}
}
if triggerMaintenance != nil {
lastCheck.Maintenance = *triggerMaintenance
setMaintenanceUserAndTime(&lastCheck, triggerMaintenance, userLogin, timeCallMaintenance)
moira.SetMaintenanceUserAndTime(&lastCheck, *triggerMaintenance, userLogin, timeCallMaintenance)
}
newLastCheck, err := json.Marshal(lastCheck)
if err != nil {
Expand All @@ -130,22 +128,6 @@ func (connector *DbConnector) SetTriggerCheckMaintenance(triggerID string, metri
return nil
}

func setMaintenanceUserAndTime(maintenaceCheck moira.MaintenanceCheck, triggerMaintenance *int64, user string, callMaintenance int64){
if user != "" && user != "anonymous" && callMaintenance != 0 {
var maintenanceWho moira.MaintenanceWho
if *triggerMaintenance < callMaintenance {
maintenanceWho.StopMaintenanceUser = &user
maintenanceWho.StopMaintenanceTime = &callMaintenance
} else {
maintenanceWho.StopMaintenanceUser = nil
maintenanceWho.StopMaintenanceTime = nil
maintenanceWho.StartMaintenanceUser = &user
maintenanceWho.StartMaintenanceTime = &callMaintenance
}
maintenaceCheck.SetMaintenanceWho(&maintenanceWho)
}
}

// checkDataScoreChanged returns true if checkData.Score changed since last check
func (connector *DbConnector) checkDataScoreChanged(triggerID string, checkData *moira.CheckData) bool {
c := connector.pool.Get()
Expand Down

0 comments on commit 712836c

Please sign in to comment.