Alerting: Expose info about notification delivery errors in a new /receivers endpoint

go.mod

@@ -366,4 +366,4 @@ replace xorm.io/xorm => github.com/grafana/xorm v0.8.3-0.20220614223926-2fcda756
 
 // Use our fork of the upstream alertmanagers.
 // This is required in order to get notification delivery errors from the receivers API.
-replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.24.0-rc.0.0.20220923162443-fdc1a480f964
+replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.24.0-rc.0.0.20220930143838-d75bdc5543c0

go.sum

@@ -1368,8 +1368,8 @@ github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58/go.m
 github.com/grafana/grafana-plugin-sdk-go v0.114.0/go.mod h1:D7x3ah+1d4phNXpbnOaxa/osSaZlwh9/ZUnGGzegRbk=
 github.com/grafana/grafana-plugin-sdk-go v0.139.0 h1:2RQKM2QpSaWTtaGN6sK+R7LO7zykOeTYF0QkAMA7JsI=
 github.com/grafana/grafana-plugin-sdk-go v0.139.0/go.mod h1:Y+Ps2sesZ62AyCnX+hzrYnyDQYe/ZZl+A8yKLOBm12c=
-github.com/grafana/prometheus-alertmanager v0.24.0-rc.0.0.20220923162443-fdc1a480f964 h1:KrD+gE1bR8SnxjkflSrc4cPodZC4ZtAmHRmUZORYSdY=
-github.com/grafana/prometheus-alertmanager v0.24.0-rc.0.0.20220923162443-fdc1a480f964/go.mod h1:xVHSIhcJ2xBqw8jSf7ZM+9NnTSV68dCLrj7MDiFEse8=
+github.com/grafana/prometheus-alertmanager v0.24.0-rc.0.0.20220930143838-d75bdc5543c0 h1:Ifcxl2wKT+UoJE+d2hsEZjH5FVdF5nML+1dtMliFk78=
+github.com/grafana/prometheus-alertmanager v0.24.0-rc.0.0.20220930143838-d75bdc5543c0/go.mod h1:xVHSIhcJ2xBqw8jSf7ZM+9NnTSV68dCLrj7MDiFEse8=
 github.com/grafana/saml v0.4.9-0.20220727151557-61cd9c9353fc h1:1PY8n+rXuBNr3r1JQhoytWDCpc+pq+BibxV0SZv+Cr4=
 github.com/grafana/saml v0.4.9-0.20220727151557-61cd9c9353fc/go.mod h1:9Zh6dWPtB3MSzTRt8fIFH60Z351QQ+s7hCU3J/tTlA4=
 github.com/grafana/thema v0.0.0-20220817114012-ebeee841c104 h1:dYpwFYIChrMfpq3wDa/ZBxAbUGSW5NYmYBeSezhaoao=

pkg/services/ngalert/api/tooling/api.json

@@ -3293,6 +3293,7 @@
    "type": "object"
   },
   "alertGroups": {
+   "description": "AlertGroups alert groups",
    "items": {
     "$ref": "#/definitions/alertGroup"
    },
@@ -3397,6 +3398,7 @@
    "type": "object"
   },
   "gettableAlert": {
+   "description": "GettableAlert gettable alert",
    "properties": {
     "annotations": {
      "$ref": "#/definitions/labelSet"
@@ -3459,6 +3461,7 @@
    "type": "array"
   },
   "gettableSilence": {
+   "description": "GettableSilence gettable silence",
    "properties": {
     "comment": {
      "description": "comment",
@@ -3507,24 +3510,25 @@
    "type": "object"
   },
   "gettableSilences": {
+   "description": "GettableSilences gettable silences",
    "items": {
     "$ref": "#/definitions/gettableSilence"
    },
    "type": "array"
   },
   "integration": {
    "properties": {
-    "lastError": {
-     "description": "last error",
+    "lastNotifyAttempt": {
+     "description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time",
+     "format": "date-time",
      "type": "string"
     },
-    "lastNotify": {
-     "description": "last notify\nFormat: date-time",
-     "format": "date-time",
+    "lastNotifyAttemptDuration": {
+     "description": "Duration of the last attempt to deliver a notification in humanized format (`1s` or `15ms`, etc).",
      "type": "string"
     },
-    "lastNotifyDuration": {
-     "description": "last notify duration",
+    "lastNotifyAttemptError": {
+     "description": "Error string for the last attempt to deliver a notification. Empty if the last attempt was successful.",
      "type": "string"
     },
     "name": {
@@ -3685,7 +3689,6 @@
    "type": "object"
   },
   "receiver": {
-   "description": "Receiver receiver",
    "properties": {
     "active": {
      "description": "active",

pkg/services/ngalert/api/tooling/post.json

@@ -3078,6 +3078,7 @@
    "type": "object"
   },
   "URL": {
+   "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
    "properties": {
     "ForceQuery": {
      "type": "boolean"
@@ -3113,7 +3114,7 @@
      "$ref": "#/definitions/Userinfo"
     }
    },
-   "title": "URL is a custom URL type that allows validation at configuration load time.",
+   "title": "A URL represents a parsed URL (technically, a URI reference).",
    "type": "object"
   },
   "Userinfo": {
@@ -3292,7 +3293,6 @@
    "type": "object"
   },
   "alertGroups": {
-   "description": "AlertGroups alert groups",
    "items": {
     "$ref": "#/definitions/alertGroup"
    },
@@ -3397,7 +3397,6 @@
    "type": "object"
   },
   "gettableAlert": {
-   "description": "GettableAlert gettable alert",
    "properties": {
     "annotations": {
      "$ref": "#/definitions/labelSet"
@@ -3508,7 +3507,6 @@
    "type": "object"
   },
   "gettableSilences": {
-   "description": "GettableSilences gettable silences",
    "items": {
     "$ref": "#/definitions/gettableSilence"
    },
@@ -3517,17 +3515,17 @@
   "integration": {
    "description": "Integration integration",
    "properties": {
-    "lastError": {
-     "description": "last error",
+    "lastNotifyAttempt": {
+     "description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time",
+     "format": "date-time",
      "type": "string"
     },
-    "lastNotify": {
-     "description": "last notify\nFormat: date-time",
-     "format": "date-time",
+    "lastNotifyAttemptDuration": {
+     "description": "Duration of the last attempt to deliver a notification in humanized format (`1s` or `15ms`, etc).",
      "type": "string"
     },
-    "lastNotifyDuration": {
-     "description": "last notify duration",
+    "lastNotifyAttemptError": {
+     "description": "Error string for the last attempt to deliver a notification. Empty if the last attempt was successful.",
      "type": "string"
     },
     "name": {
@@ -3650,7 +3648,6 @@
    "type": "array"
   },
   "postableSilence": {
-   "description": "PostableSilence postable silence",
    "properties": {
     "comment": {
      "description": "comment",

pkg/services/ngalert/api/tooling/spec.json

@@ -5617,8 +5617,9 @@
       }
     },
     "URL": {
+      "description": "The general form represented is:\n\n[scheme:][//[userinfo@]host][/]path[?query][#fragment]\n\nURLs that do not start with a slash after the scheme are interpreted as:\n\nscheme:opaque[?query][#fragment]\n\nNote that the Path field is stored in decoded form: /%47%6f%2f becomes /Go/.\nA consequence is that it is impossible to tell which slashes in the Path were\nslashes in the raw URL and which were %2f. This distinction is rarely important,\nbut when it is, the code should use RawPath, an optional field which only gets\nset if the default encoding is different from Path.\n\nURL's String method uses the EscapedPath method to obtain the path. See the\nEscapedPath method for more details.",
       "type": "object",
-      "title": "URL is a custom URL type that allows validation at configuration load time.",
+      "title": "A URL represents a parsed URL (technically, a URI reference).",
       "properties": {
         "ForceQuery": {
           "type": "boolean"
@@ -5832,7 +5833,6 @@
       "$ref": "#/definitions/alertGroup"
     },
     "alertGroups": {
-      "description": "AlertGroups alert groups",
       "type": "array",
       "items": {
         "$ref": "#/definitions/alertGroup"
@@ -5938,7 +5938,6 @@
       }
     },
     "gettableAlert": {
-      "description": "GettableAlert gettable alert",
       "type": "object",
       "required": [
         "labels",
@@ -6052,7 +6051,6 @@
       "$ref": "#/definitions/gettableSilence"
     },
     "gettableSilences": {
-      "description": "GettableSilences gettable silences",
       "type": "array",
       "items": {
         "$ref": "#/definitions/gettableSilence"
@@ -6067,17 +6065,17 @@
         "sendResolved"
       ],
       "properties": {
-        "lastError": {
-          "description": "last error",
-          "type": "string"
-        },
-        "lastNotify": {
-          "description": "last notify\nFormat: date-time",
+        "lastNotifyAttempt": {
+          "description": "A timestamp indicating the last attempt to deliver a notification regardless of the outcome.\nFormat: date-time",
           "type": "string",
           "format": "date-time"
         },
-        "lastNotifyDuration": {
-          "description": "last notify duration",
+        "lastNotifyAttemptDuration": {
+          "description": "Duration of the last attempt to deliver a notification in humanized format (`1s` or `15ms`, etc).",
+          "type": "string"
+        },
+        "lastNotifyAttemptError": {
+          "description": "Error string for the last attempt to deliver a notification. Empty if the last attempt was successful.",
           "type": "string"
         },
         "name": {
@@ -6196,7 +6194,6 @@
       }
     },
     "postableSilence": {
-      "description": "PostableSilence postable silence",
       "type": "object",
       "required": [
         "comment",

pkg/services/ngalert/notifier/receivers.go

@@ -210,15 +210,15 @@ func (am *Alertmanager) GetReceivers(ctx context.Context) apimodels.Receivers {
 		for _, integration := range rcv.Integrations() {
 			name := integration.Name()
 			sendResolved := integration.SendResolved()
-			lastNotify, lastNotifyDuration, lastNotifyError := integration.GetReport()
+			lastNotifyAttempt, lastNotifyAttemptDuration, lastNotifyAttemptError := integration.GetReport()
 			integrations = append(integrations, &apimodels.Integration{
-				Name:               &name,
-				SendResolved:       &sendResolved,
-				LastNotifyDuration: lastNotifyDuration.String(),
-				LastNotify:         strfmt.DateTime(lastNotify),
-				LastError: func() string {
-					if lastNotifyError != nil {
-						return lastNotifyError.Error()
+				Name:                      &name,
+				SendResolved:              &sendResolved,
+				LastNotifyAttempt:         strfmt.DateTime(lastNotifyAttempt),
+				LastNotifyAttemptDuration: lastNotifyAttemptDuration.String(),
+				LastNotifyAttemptError: func() string {
+					if lastNotifyAttemptError != nil {
+						return lastNotifyAttemptError.Error()
 					}
 					return ""
 				}(),

pkg/tests/api/alerting/api_notification_channel_test.go

@@ -787,9 +787,9 @@ func TestNotificationChannels(t *testing.T) {
 			for _, integration := range rcv.Integrations {
 				require.NotNil(t, integration.Name)
 				require.NotNil(t, integration.SendResolved)
-				require.Equal(t, "", integration.LastError)
-				require.Zero(t, integration.LastNotify)
-				require.Equal(t, "0s", integration.LastNotifyDuration)
+				require.Equal(t, "", integration.LastNotifyAttemptError)
+				require.Zero(t, integration.LastNotifyAttempt)
+				require.Equal(t, "0s", integration.LastNotifyAttemptDuration)
 			}
 		}
 	}
@@ -856,20 +856,20 @@ func TestNotificationChannels(t *testing.T) {
 
 			// If the receiver is not active, no attempts to send notifications should be registered.
 			if expActive {
-				require.NotZero(t, integration.LastNotify)
-				require.NotEqual(t, "0s", integration.LastNotifyDuration)
+				require.NotZero(t, integration.LastNotifyAttempt)
+				require.NotEqual(t, "0s", integration.LastNotifyAttemptDuration)
 			} else {
-				require.Zero(t, integration.LastNotify)
-				require.Equal(t, "0s", integration.LastNotifyDuration)
+				require.Zero(t, integration.LastNotifyAttempt)
+				require.Equal(t, "0s", integration.LastNotifyAttemptDuration)
 			}
 
 			// Check whether we're expecting an error on this integration.
 			if expErr {
 				for _, integration := range rcv.Integrations {
-					require.Equal(t, expNotificationErrors[*rcv.Name], integration.LastError)
+					require.Equal(t, expNotificationErrors[*rcv.Name], integration.LastNotifyAttemptError)
 				}
 			} else {
-				require.Equal(t, "", integration.LastError)
+				require.Equal(t, "", integration.LastNotifyAttemptError)
 			}
 		}
 	}