Purge old Rendered Templates

cmd/bosun/conf/conf.go

@@ -70,6 +70,8 @@ type SystemConfProvider interface {
 
 	GetAuthConf() *AuthConf
 
+	GetMaxRenderedTemplateAge() int
+
 	// Contexts
 	GetTSDBContext() opentsdb.Context
 	GetGraphiteContext() graphite.Context

cmd/bosun/conf/system.go

@@ -52,6 +52,8 @@ type SystemConf struct {
 
 	AuthConf *AuthConf
 
+	MaxRenderedTemplateAge int // in days
+
 	EnableSave      bool
 	EnableReload    bool
 	CommandHookPath string
@@ -427,6 +429,12 @@ func (sc *SystemConf) GetInternetProxy() string {
 	return sc.InternetProxy
 }
 
+// MaxRenderedTemplateAge returns the maximum time in days to keep rendered templates
+// after the incident end date.
+func (sc *SystemConf) GetMaxRenderedTemplateAge() int {
+	return sc.MaxRenderedTemplateAge
+}
+
 // SaveEnabled returns if saving via the UI and config editing API endpoints should be enabled
 func (sc *SystemConf) SaveEnabled() bool {
 	return sc.EnableSave

cmd/bosun/database/search_data.go

@@ -161,7 +161,6 @@ func (d *dataAccess) GetMetricTagSets(metric string, tags opentsdb.TagSet) (map[
 				result[mts] = t
 			}
 		}
-
 		if cursor == "" || cursor == "0" {
 			break
 		}

cmd/bosun/database/state_data.go

@@ -67,6 +67,9 @@ type StateDataAccess interface {
 
 	SetRenderedTemplates(incidentId int64, rt *models.RenderedTemplates) error
 	GetRenderedTemplates(incidentId int64) (*models.RenderedTemplates, error)
+	GetRenderedTemplateKeys() ([]string, error)
+	CleanupOldRenderedTemplates(olderThan time.Duration)
+	DeleteRenderedTemplates(incidentIds []int64) error
 
 	Forget(ak models.AlertKey) error
 	SetUnevaluated(ak models.AlertKey, uneval bool) error
@@ -93,16 +96,80 @@ func (d *dataAccess) GetRenderedTemplates(incidentId int64) (*models.RenderedTem
 	defer conn.Close()
 
 	b, err := redis.Bytes(conn.Do("GET", renderedTemplatesKey(incidentId)))
+	renderedT := &models.RenderedTemplates{}
 	if err != nil {
+		if err == redis.ErrNil {
+			return renderedT, nil
+		}
 		return nil, slog.Wrap(err)
 	}
-	renderedT := &models.RenderedTemplates{}
 	if err = json.Unmarshal(b, renderedT); err != nil {
 		return nil, slog.Wrap(err)
 	}
 	return renderedT, nil
 }
 
+func (d *dataAccess) GetRenderedTemplateKeys() ([]string, error) {
+	conn := d.Get()
+	defer conn.Close()
+
+	//ledis uses XSCAN cursor "KV" MATCH foo
+	//redis uses SCAN cursor MATCH foo
+	cmd := "SCAN"
+	args := []interface{}{"0", "MATCH", "renderedTemplatesById:*"}
+	cursorIdx := 0
+	if !d.isRedis {
+		cmd = "XSCAN"
+		args = append([]interface{}{"KV"}, args...)
+		cursorIdx = 1
+	}
+	found := []string{}
+	for {
+		vals, err := redis.Values(conn.Do(cmd, args...))
+		if err != nil {
+			return nil, slog.Wrap(err)
+		}
+		cursor, err := redis.String(vals[0], nil)
+		if err != nil {
+			return nil, slog.Wrap(err)
+		}
+		args[cursorIdx] = cursor
+		keys, err := redis.Strings(vals[1], nil)
+		if err != nil {
+			return nil, slog.Wrap(err)
+		}
+		found = append(found, keys...)
+		if cursor == "" || cursor == "0" {
+			break
+		}
+	}
+	return found, nil
+}
+
+func (d *dataAccess) DeleteRenderedTemplates(incidentIds []int64) error {
+	conn := d.Get()
+	defer conn.Close()
+	const batchSize = 1000
+	args := make([]interface{}, 0, batchSize)
+	for len(incidentIds) > 0 {
+		size := len(incidentIds)
+		if size > batchSize {
+			size = batchSize
+		}
+		thisBatch := incidentIds[:size]
+		incidentIds = incidentIds[size:]
+		args = args[:0]
+		for _, id := range thisBatch {
+			args = append(args, renderedTemplatesKey(id))
+		}
+		_, err := conn.Do("DEL", args...)
+		if err != nil {
+			return slog.Wrap(err)
+		}
+	}
+	return nil
+}
+
 func (d *dataAccess) State() StateDataAccess {
 	return d
 }
@@ -380,10 +447,12 @@ func (d *dataAccess) Forget(ak models.AlertKey) error {
 			return slog.Wrap(err)
 		}
 		for _, id := range ids {
-
 			if _, err = conn.Do("DEL", incidentStateKey(id)); err != nil {
 				return slog.Wrap(err)
 			}
+			if _, err = conn.Do("DEL", renderedTemplatesKey(id)); err != nil {
+				return slog.Wrap(err)
+			}
 		}
 		if _, err := conn.Do(d.LCLEAR(), incidentsForAlertKeyKey(ak)); err != nil {
 			return slog.Wrap(err)
@@ -447,3 +516,58 @@ func (d *dataAccess) transact(conn redis.Conn, f func() error) error {
 	}
 	return nil
 }
+
+// CleanupCleanupOldRenderedTemplates will in a loop purge any old rendered templates
+func (d *dataAccess) CleanupOldRenderedTemplates(olderThan time.Duration) {
+	// run after 5 minutes (to let bosun stabilize)
+	// and then every hour
+	time.Sleep(time.Minute * 5)
+	for {
+		conn := d.Get()
+		slog.Infof("Cleaning out old rendered templates")
+		earliestOk := time.Now().UTC().Add(-1 * olderThan)
+		func() {
+			toPurge := []int64{}
+			keys, err := d.GetRenderedTemplateKeys()
+			if err != nil {
+				slog.Error(err)
+				return
+			}
+			for _, key := range keys {
+				parts := strings.Split(key, ":")
+				if len(parts) != 2 {
+					slog.Errorf("Invalid rendered template redis key found: %s", key)
+					continue
+				}
+				id, err := strconv.ParseInt(parts[1], 10, 64)
+				if err != nil {
+					slog.Error(err)
+					continue
+				}
+				state, err := d.getIncident(id, conn)
+				if err != nil {
+					if strings.Contains(err.Error(), "nil returned") {
+						toPurge = append(toPurge, id)
+						continue
+					}
+					slog.Error(err)
+					continue
+				}
+				if state.End != nil && (*state.End).Before(earliestOk) {
+					toPurge = append(toPurge, id)
+				}
+			}
+			if len(toPurge) == 0 {
+				return
+			}
+			slog.Infof("Deleting %d old rendered templates", len(toPurge))
+			if err = d.DeleteRenderedTemplates(toPurge); err != nil {
+				slog.Error(err)
+				return
+			}
+		}()
+		conn.Close()
+		slog.Info("Done cleaning rendered templates")
+		time.Sleep(time.Hour)
+	}
+}

cmd/bosun/main.go

@@ -141,6 +141,9 @@ func main() {
 	if err != nil {
 		slog.Fatal(err)
 	}
+	if sysProvider.GetMaxRenderedTemplateAge() != 0 {
+		go da.State().CleanupOldRenderedTemplates(time.Hour * 24 * time.Duration(sysProvider.GetMaxRenderedTemplateAge()))
+	}
 	var annotateBackend backend.Backend
 	if sysProvider.AnnotateEnabled() {
 		index := sysProvider.GetAnnotateIndex()

cmd/bosun/web/static/js/bosun.js

@@ -3281,7 +3281,7 @@ bosunControllers.controller('IncidentCtrl', ['$scope', '$http', '$location', '$r
             }
             v.error = null;
             v.doneLoading = false;
-            if (i == $scope.lastNonUnknownAbnormalIdx) {
+            if (i == $scope.lastNonUnknownAbnormalIdx && $scope.body) {
                 v.subject = $scope.incident.Subject;
                 v.body = $scope.body;
                 v.doneLoading = true;
@@ -3317,7 +3317,7 @@ bosunControllers.controller('IncidentCtrl', ['$scope', '$http', '$location', '$r
             $scope.configLink = configUrl($scope.incident.AlertKey, moment.unix($scope.incident.LastAbnormalTime));
             for (var i = 0; i < $scope.events.length; i++) {
                 var e = $scope.events[i];
-                if (e.Status != 'normal' && e.Status != 'unknown') {
+                if (e.Status != 'normal' && e.Status != 'unknown' && $scope.body) {
                     $scope.lastNonUnknownAbnormalIdx = i;
                     $scope.collapse(i, e); // Expand the panel of the current body
                     break;