wip: add bg job cleanup

mr-karan · Feb 18, 2022 · 33cb34a · 33cb34a
1 parent 9271b28
commit 33cb34a
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 25 deletions.
diff --git a/cmd/init.go b/cmd/init.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/knadh/koanf"
 	"github.com/knadh/koanf/parsers/toml"
@@ -85,18 +86,20 @@ func initProviders(ko *koanf.Koanf, lo *logrus.Logger) []prvs.Provider {
 		case "google_chat":
 			gchat, err := prvs.NewGoogleChat(
 				prvs.GoogleChatOpts{
-					Log:         lo,
-					Timeout:     ko.MustDuration(fmt.Sprintf("%s.timeout", cfgKey)),
-					MaxIdleConn: ko.MustInt(fmt.Sprintf("%s.max_idle_conns", cfgKey)),
-					ProxyURL:    ko.String(fmt.Sprintf("%s.proxy_url", cfgKey)),
-					Endpoint:    ko.String(fmt.Sprintf("%s.endpoint", cfgKey)),
-					Room:        name,
-					Template:    ko.String(fmt.Sprintf("%s.template", cfgKey)),
+					Log:             lo,
+					Timeout:         ko.MustDuration(fmt.Sprintf("%s.timeout", cfgKey)),
+					MaxIdleConn:     ko.MustInt(fmt.Sprintf("%s.max_idle_conns", cfgKey)),
+					ProxyURL:        ko.String(fmt.Sprintf("%s.proxy_url", cfgKey)),
+					Endpoint:        ko.MustString(fmt.Sprintf("%s.endpoint", cfgKey)),
+					Room:            name,
+					Template:        ko.MustString(fmt.Sprintf("%s.template", cfgKey)),
+					ActiveAlertsTTL: ko.MustDuration(fmt.Sprintf("%s.active_alerts_ttl", cfgKey)),
 				},
 			)
 			if err != nil {
 				lo.WithError(err).Fatal("error initialising google chat provider")
 			}
+			go gchat.InitPruner(1 * time.Hour)
 			lo.WithField("room", gchat.GetRoom()).Info("initialised provider")
 			provs = append(provs, gchat)
 		}

diff --git a/config.sample.toml b/config.sample.toml
@@ -4,19 +4,22 @@ max_size = 4000
 address = "0.0.0.0:6000"
 server_timeout = "5s"
 enable_request_logs = true
+log_level = "debug"
 
 [providers.prod_alerts]
-endpoint = "https://chat.googleapis.com/v1/spaces/xxx/messages?key=key&token=token"
+endpoint = "https://chat.googleapis.com/v1/spaces/xxx/messages?key=key&token=token%3D"
 type = "google_chat"
 max_idle_conns =  50
 timeout = "7s"
 # proxy_url = "http://internal-squid-proxy.com:3128"
 template = "static/message.tmpl"
+active_alerts_ttl = "5s"
 
 [providers.dev_alerts]
-endpoint = "https://chat.googleapis.com/v1/spaces/xxx/messages?key=key&token=token"
+endpoint = "https://chat.googleapis.com/v1/spaces/xxx/messages?key=key&token=token%3D"
 type = "google_chat"
 max_idle_conns =  50
 timeout = "7s"
 # proxy_url = "http://internal-squid-proxy.com:3128"
 template = "static/message.tmpl"
+active_alerts_ttl = "12h"
diff --git a/docs/examples/mock_payload.json b/docs/examples/mock_payload.json
@@ -7,7 +7,7 @@
             "labels": {
                 "alertname": "DeadManAnotherSwitch",
                 "monitor": "docker-host-alpha",
-                "room": "dev-ops-alerts",
+                "room": "dev_alerts",
                 "severity": "deadman"
             },
             "annotations": {
@@ -27,7 +27,7 @@
             "labels": {
                 "alertname": "DeadManSwitch",
                 "monitor": "docker-host-alpha",
-                "room": "dev-ops-alerts",
+                "room": "dev_alerts",
                 "severity": "deadman"
             },
             "annotations": {
@@ -44,11 +44,11 @@
         }
     ],
     "groupLabels": {
-        "room": "dev-ops-alerts"
+        "room": "dev_alerts"
     },
     "commonLabels": {
         "monitor": "docker-host-alpha",
-        "room": "dev-ops-alerts",
+        "room": "dev_alerts",
         "severity": "deadman"
     },
     "commonAnnotations": {

diff --git a/docs/examples/send_alert.sh b/docs/examples/send_alert.sh
@@ -1 +1 @@
-curl -XPOST -d @mock_payload.json http://localhost:6000/create?room_name=alertManagerTestRoom -i
+curl -XPOST -d @mock_payload.json http://localhost:6000/dispatch -i
diff --git a/internal/providers/google_chat.go b/internal/providers/google_chat.go
@@ -71,8 +71,8 @@ func (d *ActiveAlerts) remove(fingerprint string) {
 
 // loookup retrievs the UUID for the alert based on the fingerprint.
 func (d *ActiveAlerts) loookup(fingerprint string) string {
-	d.Lock()
-	defer d.Unlock()
+	d.RLock()
+	defer d.RUnlock()
 	// Do a lookup for the provider by the room name and push the alerts.
 	if _, ok := d.alerts[fingerprint]; !ok {
 		return ""
@@ -83,21 +83,23 @@ func (d *ActiveAlerts) loookup(fingerprint string) string {
 // GoogleChatManager represents the various methods for interacting with Google Chat.
 type GoogleChatManager struct {
 	lo           *logrus.Logger
-	activeAlerts ActiveAlerts
+	activeAlerts *ActiveAlerts
 	endpoint     string
 	room         string
 	client       *http.Client
 	msgTmpl      *template.Template
+	ttl          time.Duration
 }
 
 type GoogleChatOpts struct {
-	Log         *logrus.Logger
-	MaxIdleConn int
-	Timeout     time.Duration
-	ProxyURL    string
-	Endpoint    string
-	Room        string
-	Template    string
+	Log             *logrus.Logger
+	MaxIdleConn     int
+	Timeout         time.Duration
+	ProxyURL        string
+	Endpoint        string
+	Room            string
+	Template        string
+	ActiveAlertsTTL time.Duration
 }
 
 // NewGoogleChat initializes a Google Chat provider object.
@@ -153,8 +155,9 @@ func NewGoogleChat(opts GoogleChatOpts) (*GoogleChatManager, error) {
 		client:       client,
 		endpoint:     opts.Endpoint,
 		room:         opts.Room,
-		activeAlerts: a,
+		activeAlerts: &a,
 		msgTmpl:      tmpl,
+		ttl:          opts.ActiveAlertsTTL,
 	}, nil
 }
 
@@ -260,3 +263,27 @@ func (m *GoogleChatManager) sendMessage(msg ChatMessage, threadKey string) error
 
 	return nil
 }
+
+// InitPruner is used to remove active alerts in the
+// map once their TTL is reached. The cleanup activity
+// happens at periodic intervals.
+// This is a blocking function
+// so the caller must invoke as a goroutine.
+func (m *GoogleChatManager) InitPruner(pruneInterval time.Duration) {
+	var (
+		evalTicker = time.NewTicker(pruneInterval).C
+	)
+	for range evalTicker {
+		// m.lo.Info("locking map")
+		now := time.Now()
+		expired := now.Add(-m.ttl)
+		m.lo.Info("pruning active alerts based on ttl")
+		for k, a := range m.activeAlerts.alerts {
+			fmt.Println("looping in prune", k, expired, a.StartsAt)
+			if a.StartsAt.Before(expired) {
+				m.lo.WithField("fingerprint", k).WithField("created", a.StartsAt).WithField("expired", expired).Info("removing alert from active alerts")
+				m.activeAlerts.remove(k)
+			}
+		}
+	}
+}