Skip to content

Restic: refactors the cluster backup and restic management code #1283

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 17 commits into
base: develop
Choose a base branch
from
Open

Restic: refactors the cluster backup and restic management code #1283

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
17 commits
Select commit Hold shift + click to select a range
d8f8f96
Refactor ResticRepo to ResticManager, update related methods and refe…
caffeinated92 Nov 29, 2025
1f9582e
feat(restic): enhance Restic task management with error handling and …
caffeinated92 Nov 30, 2025
a5bcab8
feat(backup): implement purge task management and enhance backup stat…
caffeinated92 Nov 30, 2025
a5c5aea
Refactor backup management and remove archiver module
caffeinated92 Nov 30, 2025
6121516
Refactor API endpoints for Restic management
caffeinated92 Nov 30, 2025
e8c5e51
feat: Implement Restic task management and enhance API endpoints
caffeinated92 Nov 30, 2025
f4d4f62
refactor: Rename log archive references to log restic for consistency
caffeinated92 Nov 30, 2025
dd44f84
feat(logging): Add Restic module to logging switch case for improved …
caffeinated92 Nov 30, 2025
43b6fd5
feat: Update log level for Restic and add confirmation modal for snap…
caffeinated92 Nov 30, 2025
8d385d8
refactor: Remove unused React import and close confirmation modal aft…
caffeinated92 Dec 1, 2025
3484940
feat: Implement Restic task queue management with API endpoints for r…
caffeinated92 Dec 1, 2025
ec4c180
feat: Add API endpoints for cancelling and resuming Restic tasks, and…
caffeinated92 Dec 1, 2025
ee1b8fd
feat: Implement Restic task queue management with new actions for can…
caffeinated92 Dec 1, 2025
b4bf1c4
Merge remote-tracking branch 'origin/develop' into restic
caffeinated92 Dec 2, 2025
c196e9b
feat(cluster): add FetchTask to BackupRestic for improved backup mana…
caffeinated92 Dec 2, 2025
9cd6309
feat(restic): add FetchTask invocation in AddBackupTask, AddPurgeTask…
caffeinated92 Dec 2, 2025
fd3fbfe
feat(restic): add deferred AddFetchTask invocation in InitRepo for im…
caffeinated92 Dec 2, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 12 additions & 8 deletions cluster/cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ import (
"github.com/signal18/replication-manager/router/maxscale"
"github.com/signal18/replication-manager/utils/alert/mailer"
"github.com/signal18/replication-manager/utils/alert/slackman"
"github.com/signal18/replication-manager/utils/archiver"
"github.com/signal18/replication-manager/utils/backupmgr"
"github.com/signal18/replication-manager/utils/cron"
"github.com/signal18/replication-manager/utils/dbhelper"
"github.com/signal18/replication-manager/utils/logrus/hooks/pushover"
Expand Down Expand Up @@ -206,7 +206,7 @@ type Cluster struct {
QueryRules map[uint32]config.QueryRule `json:"-"`
Backups []v3.Backup `json:"-"`
BackupStat v3.BackupStat `json:"backupStat" groups:"web"`
BackupMetaMap *config.BackupMetaMap `json:"backupList" groups:"web"`
BackupMetaMap *backupmgr.BackupMetaMap `json:"backupList" groups:"web"`
SLAHistory []state.Sla `json:"slaHistory" groups:"web"`
APIUsers map[string]APIUser `json:"apiUsers" groups:"web"`
Schedule map[string]cron.Entry `json:"-"`
Expand Down Expand Up @@ -245,7 +245,8 @@ type Cluster struct {
InResticBackup bool `json:"inResticBackup" groups:"web"`
InRollingRestart bool `json:"inRollingRestart" groups:"web"`
Mailer *mailer.Mailer `json:"-"`
ResticRepo *archiver.ResticRepo `json:"-"`
ResticManager *backupmgr.ResticManager `json:"-"`
MessageChan chan s18log.HttpMessage `json:"-"`
ErrorConfigs config.ErrorConfigs `json:"-"` //To store error config
Partner *config.Partner `json:"partner" groups:"web"`
ConfigManager *manager.ConfigManager `json:"-"`
Expand Down Expand Up @@ -371,6 +372,9 @@ func (cluster *Cluster) Init(confs *config.ConfVersion, cfgGroup string, tlog *s
cluster.AgentMaxFreq = make(map[string]int64)
cluster.ServiceTemplates = make([]string, 0)
cluster.OpenSVCStats.Store([]opensvc.DaemonNodeStats{})
cluster.MessageChan = make(chan s18log.HttpMessage, 10)

go cluster.ConsumeMessageChan()

*cluster.Conf = confs.ConfInit

Expand Down Expand Up @@ -410,7 +414,7 @@ func (cluster *Cluster) InitFromConf() {
cluster.runOnceAfterTopology = true
cluster.testStopCluster = true
cluster.testStartCluster = true
cluster.BackupMetaMap = config.NewBackupMetaMap()
cluster.BackupMetaMap = backupmgr.NewBackupMetaMap()
cluster.VersionsMap = config.NewVersionsMap()

cluster.WorkingDir = cluster.Conf.WorkingDir + "/" + cluster.Name
Expand Down Expand Up @@ -585,7 +589,7 @@ func (cluster *Cluster) InitFromConf() {
cluster.initScheduler()
cluster.CheckDefaultUser(true)
cluster.RefreshToolVersions()
cluster.StartResticRepo()
cluster.StartResticManager()

cluster.Conf.TopologyTarget = cluster.GetTopologyFromConf()
}
Expand Down Expand Up @@ -979,8 +983,8 @@ func (cluster *Cluster) StateProcessing() {
func (cluster *Cluster) Stop() {
cluster.Lock()
defer cluster.Unlock()
if cluster.ResticRepo != nil {
cluster.ResticRepo.ShutdownWorker()
if cluster.ResticManager != nil {
cluster.ResticManager.ShutdownWorker()
}
cluster.CloseRefreshTemplateMD5Worker()
cluster.ConfigManager.SaveConfig(cluster, true)
Expand Down Expand Up @@ -1757,7 +1761,7 @@ func (cluster *Cluster) MonitorSchema() {

cluster.WorkLoad.DBIndexSize = totindexsize
cluster.WorkLoad.DBTableSize = tottablesize
cmaster.DictTables = config.FromNormalTablesMap(cmaster.DictTables, tables)
cmaster.DictTables = dbhelper.FromNormalTablesMap(cmaster.DictTables, tables)
cluster.StateMachine.RemoveMonitorSchemaState()
}

Expand Down
11 changes: 3 additions & 8 deletions cluster/cluster_acl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -638,25 +638,20 @@ func (cluster *Cluster) IsURLPassACL(strUser string, URL string, errorPrint bool
return true
}
}
if cluster.APIUsers[strUser].Grants[config.GrantClusterShowBackups] {
if strings.Contains(URL, "/api/clusters/"+cluster.Name+"/backups") {
return true
}
}
if cluster.APIUsers[strUser].Grants[config.GrantClusterShowBackups] {
if strings.Contains(URL, "/api/clusters/"+cluster.Name+"/backups") {
return true
}

if strings.HasSuffix(strings.TrimSuffix(URL, "/"), "/api/clusters/"+cluster.Name+"/archives") {
if strings.HasSuffix(strings.TrimSuffix(URL, "/"), "/api/clusters/"+cluster.Name+"/restic/snapshots") {
return true
}
if strings.HasSuffix(strings.TrimSuffix(URL, "/"), "/api/clusters/"+cluster.Name+"/archives/stats") {
if strings.HasSuffix(strings.TrimSuffix(URL, "/"), "/api/clusters/"+cluster.Name+"/restic/stats") {
return true
}
}
if cluster.APIUsers[strUser].Grants[config.GrantClusterProcess] {
if strings.Contains(URL, "/api/clusters/"+cluster.Name+"/archives") {
if strings.Contains(URL, "/api/clusters/"+cluster.Name+"/restic") {
return true
}
}
Expand Down
7 changes: 4 additions & 3 deletions cluster/cluster_bash.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ import (

"github.com/signal18/replication-manager/config"
"github.com/signal18/replication-manager/utils/alert"
"github.com/signal18/replication-manager/utils/backupmgr"
"github.com/signal18/replication-manager/utils/state"
)

Expand Down Expand Up @@ -227,9 +228,9 @@ func (cluster *Cluster) BinlogCopyScript(server *ServerMonitor, binlog string, i
return nil
}

func (cluster *Cluster) BackupPostScript(server *ServerMonitor, backtype config.BackupMethod, filepath string) error {
func (cluster *Cluster) BackupPostScript(server *ServerMonitor, backtype backupmgr.BackupMethod, filepath string) error {
switch backtype {
case config.BackupMethodLogical:
case backupmgr.BackupMethodLogical:
if cluster.Conf.BackupLogicalPostScript != "" {
cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModGeneral, config.LvlInfo, "Calling backup logical post script at %s", cluster.Conf.BackupLogicalPostScript)
var out []byte
Expand All @@ -241,7 +242,7 @@ func (cluster *Cluster) BackupPostScript(server *ServerMonitor, backtype config.

cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModGeneral, "INFO", "Backup logical post script done: %s", string(out))
}
case config.BackupMethodPhysical:
case backupmgr.BackupMethodPhysical:
if cluster.Conf.BackupPhysicalPostScript != "" {
cluster.LogModulePrintf(cluster.Conf.Verbose, config.ConstLogModGeneral, config.LvlInfo, "Calling backup physical post script at %s", cluster.Conf.BackupPhysicalPostScript)
var out []byte
Expand Down
Loading