-
Notifications
You must be signed in to change notification settings - Fork 1.3k
/
notification.go
139 lines (118 loc) · 3.18 KB
/
notification.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
// Copyright (c) Tailscale Inc & AUTHORS
// SPDX-License-Identifier: BSD-3-Clause
package prober
import (
"bytes"
"encoding/json"
"errors"
"expvar"
"fmt"
"io"
"net/http"
"os"
"strings"
"time"
"github.com/google/uuid"
"tailscale.com/util/httpm"
)
var (
alertGenerated = expvar.NewInt("alert_generated")
alertFailed = expvar.NewInt("alert_failed")
warningGenerated = expvar.NewInt("warning_generated")
warningFailed = expvar.NewInt("warning_failed")
)
// SendAlert sends an alert to the incident response system, to
// page a human responder immediately.
// summary should be short and state the nature of the emergency.
// details can be longer, up to 29 KBytes.
func SendAlert(summary, details string) error {
type squadcastAlert struct {
Message string `json:"message"`
Description string `json:"description"`
Tags map[string]string `json:"tags,omitempty"`
Status string `json:"status"`
EventId string `json:"event_id"`
}
sqa := squadcastAlert{
Message: summary,
Description: details,
Tags: map[string]string{"severity": "critical"},
Status: "trigger",
EventId: uuid.New().String(),
}
sqBody, err := json.Marshal(sqa)
if err != nil {
alertFailed.Add(1)
return fmt.Errorf("encoding alert payload: %w", err)
}
webhookUrl := os.Getenv("SQUADCAST_WEBHOOK")
if webhookUrl == "" {
warningFailed.Add(1)
return errors.New("no SQUADCAST_WEBHOOK configured")
}
req, err := http.NewRequest(httpm.POST, webhookUrl, bytes.NewBuffer(sqBody))
if err != nil {
alertFailed.Add(1)
return err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
alertFailed.Add(1)
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
alertFailed.Add(1)
return errors.New(resp.Status)
}
body, _ := io.ReadAll(resp.Body)
if string(body) != "ok" {
alertFailed.Add(1)
return errors.New("non-ok response returned from Squadcast")
}
alertGenerated.Add(1)
return nil
}
// SendWarning will post a message to Slack.
// details should be a description of the issue.
func SendWarning(details string) error {
webhookUrl := os.Getenv("SLACK_WEBHOOK")
if webhookUrl == "" {
warningFailed.Add(1)
return errors.New("no SLACK_WEBHOOK configured")
}
type slackRequestBody struct {
Text string `json:"text"`
}
slackBody, err := json.Marshal(slackRequestBody{Text: details})
if err != nil {
warningFailed.Add(1)
return err
}
req, err := http.NewRequest("POST", webhookUrl, bytes.NewReader(slackBody))
if err != nil {
warningFailed.Add(1)
return err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{Timeout: 10 * time.Second}
resp, err := client.Do(req)
if err != nil {
warningFailed.Add(1)
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
warningFailed.Add(1)
return errors.New(resp.Status)
}
body, _ := io.ReadAll(resp.Body)
if s := strings.TrimSpace(string(body)); s != "ok" {
warningFailed.Add(1)
return errors.New("non-ok response returned from Slack")
}
warningGenerated.Add(1)
return nil
}