-
Notifications
You must be signed in to change notification settings - Fork 307
/
embeddedRecovery.go
48 lines (41 loc) · 1.33 KB
/
embeddedRecovery.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
package db
import (
"fmt"
"github.com/rudderlabs/rudder-server/rruntime"
"github.com/rudderlabs/rudder-go-kit/config"
)
// HandleEmbeddedRecovery decides the recovery Mode in which app should run based on earlier crashes
func HandleEmbeddedRecovery(forceNormal, forceDegraded bool, currTime int64, appType string) error {
enabled := config.GetBool("recovery.enabled", true)
if !enabled {
return nil
}
isForced := false
forceMode := getForceRecoveryMode(forceNormal, forceDegraded)
recoveryData, err := getRecoveryData()
if err != nil {
return fmt.Errorf("getting recovery data: %w", err)
}
if forceMode != "" {
isForced = true
recoveryData.Mode = forceMode
}
recoveryHandler := NewRecoveryHandler(&recoveryData)
if !isForced && recoveryHandler.HasThresholdReached() {
pkgLogger.Info("DB Recovery: Moving to next State. Threshold reached for " + recoveryData.Mode)
recoveryData.Mode = degradedMode
recoveryHandler = NewRecoveryHandler(&recoveryData)
alertOps(recoveryData.Mode)
}
recoveryHandler.RecordAppStart(currTime)
if err := saveRecoveryData(recoveryData); err != nil {
return fmt.Errorf("saving recovery data: %w", err)
}
recoveryHandler.Handle()
pkgLogger.Infof("Starting in %s mode", recoveryData.Mode)
CurrentMode = recoveryData.Mode
rruntime.Go(func() {
sendRecoveryModeStat(appType)
})
return nil
}