Skip to content

imp(auto-backup): remove backup of last N blocks instead of N records #22

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Jun 2, 2024
Merged

imp(auto-backup): remove backup of last N blocks instead of N records #22

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a6eb4ad
update message
0xbcdev Jun 2, 2024
f900d00
keep backup of last N blocks instead of just N records
0xbcdev Jun 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 31 additions & 19 deletions cmd/node/auto_backup_priv_validator_state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ func GetAutoBackupPrivValidatorStateCmd() *cobra.Command {
if keepRecent < 3 {
keepRecent = 3
}
fmt.Println("INF: keep the last", keepRecent, "backups")
fmt.Println("INF: keep backup of the last", keepRecent, "blocks")

binaryPathToKill, _ := cmd.Flags().GetString(flagBinaryKillByAutoBackup)
if binaryPathToKill == "" {
Expand Down Expand Up @@ -105,28 +105,35 @@ func GetAutoBackupPrivValidatorStateCmd() *cobra.Command {
fmt.Println("INF: latest state from backup:")
fmt.Println(latestBackupPvs.Json())

const interval = 800 * time.Millisecond
const interval = 200 * time.Millisecond
var lastExecution time.Time

var createdBackup []string
type backupPerHeight struct {
heightStr string
files []string
}
backupFilesByHeight := make([]*backupPerHeight, 0)

for {
if time.Since(lastExecution) < interval {
time.Sleep(50 * time.Millisecond)
time.Sleep(20 * time.Millisecond)
continue
}

lastExecution = time.Now().UTC()

// Remove old backups
if countCreatedBackups := len(createdBackup); countCreatedBackups > keepRecent {
for i := 0; i < countCreatedBackups-keepRecent; i++ {
err := os.Remove(createdBackup[i])
if err != nil {
utils.PrintlnStdErr("ERR: failed to remove backup file:", createdBackup[i], ":", err)
if numberOfBackupHeights := len(backupFilesByHeight); numberOfBackupHeights > keepRecent {
pruneSize := numberOfBackupHeights - keepRecent
for _, backupPerHeight := range backupFilesByHeight[:pruneSize] {
for _, file := range backupPerHeight.files {
if err := os.Remove(file); err != nil && !os.IsNotExist(err) {
utils.PrintlnStdErr("ERR: failed to remove backup file", file, ":", err)
// ignore error
}
}
}
createdBackup = createdBackup[countCreatedBackups-keepRecent:]
backupFilesByHeight = backupFilesByHeight[pruneSize:]
}

// Load the recent state
Expand Down Expand Up @@ -158,19 +165,24 @@ func GetAutoBackupPrivValidatorStateCmd() *cobra.Command {
continue
}

// backup the recent state to file, marked by time
backupFileNameMarkByTime := fmt.Sprintf(
"%s_%s.%s.json",
// backup the recent state to file, marked by time and height/round/step
backupFileNameMarkByTimeAndHrs := fmt.Sprintf(
"%s_%s_hrs_%s_%d_%d.json",
backupPrivValStateJsonPrefixFileName,
utils.GetDateTimeStringCompatibleWithFileName(time.Now().UTC(), time.DateTime),
constants.BINARY_NAME,
recentPvs.Height, recentPvs.Round, recentPvs.Step,
)
backupMarkByTimeFilePath := path.Join(backupDstPath, backupFileNameMarkByTime)
err = recentPvs.SaveToJSONFile(backupMarkByTimeFilePath)
backupMarkByTimeAndHrsFilePath := path.Join(backupDstPath, backupFileNameMarkByTimeAndHrs)
err = recentPvs.SaveToJSONFile(backupMarkByTimeAndHrsFilePath)
if err != nil {
utils.PrintlnStdErr("ERR: failed to save backup file", backupMarkByTimeFilePath, err)
utils.PrintlnStdErr("ERR: failed to save backup file", backupMarkByTimeAndHrsFilePath, err)
} else if size := len(backupFilesByHeight); size == 0 || backupFilesByHeight[size-1].heightStr != recentPvs.Height {
backupFilesByHeight = append(backupFilesByHeight, &backupPerHeight{
heightStr: recentPvs.Height,
files: []string{backupMarkByTimeAndHrsFilePath},
})
} else {
createdBackup = append(createdBackup, backupMarkByTimeFilePath)
backupFilesByHeight[size-1].files = append(backupFilesByHeight[size-1].files, backupMarkByTimeAndHrsFilePath)
}

if stateIncreased {
Expand Down Expand Up @@ -294,7 +306,7 @@ How to recover:
},
}

cmd.Flags().Int(flagKeep, 3, "Keep the last N backups")
cmd.Flags().Int(flagKeep, 3, "Keep backup of the last N blocks")
cmd.Flags().String(flagBinaryKillByAutoBackup, "", "Absolute path of the chain binary to be killed by process when priv_validator_state.json has problem")
cmd.Flags().Bool(flagGenSetup, false, "Display guide to setup instead of running business logic")

Expand Down