Prune/Retention Policy

README.md

@@ -175,6 +175,13 @@ rsync:
   #override_global_excluded: true
   #override_global_args:     true
 
+# FIXME needs more details (
+retention:
+  daily:    int       # number of daily backups to keep
+  weekly:   int       # number of weekly backups to keep
+  monthly:  int       # number of monthly backups to keep
+  yearly:   int       # number of yearly backups to keep
+
 # Inline scripts executed on the remote host before and after rsyncing,
 # and before any `pre.*.sh` and/or `post.*.sh` scripts for this host.
 pre_script:  string
@@ -222,6 +229,11 @@ rsync:
   - "--hard-links"
   - "--block-size=2048"
   - "--recursive"
+retention:
+  daily: 14
+  weekly: 4
+  monthly: 6
+  yearly: 5
 ```
 
 # Copyright

app/prune.go

@@ -0,0 +1,152 @@
+package app
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+	"time"
+
+	"github.com/digineo/zackup/config"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	patterns = map[string]string{
+		"daily":   "2006-01-02",
+		"weekly":  "", // See special case in keepers()
+		"monthly": "2006-01",
+		"yearly":  "2006",
+	}
+)
+
+type snapshot struct {
+	Name string    // Snapshot dataset name "backups/foo@RFC3339"
+	Time time.Time // Parsed timestamp from the dataset name
+}
+
+// FIXME PruneSnapshots does not actually perform any destructive operations
+// on your datasets at this time.
+func PruneSnapshots(job *config.JobConfig) {
+	var host = job.Host()
+
+	// Defaults: if config is not set
+	if job.Retention == nil {
+		job.Retention = &config.RetentionConfig{
+			Daily:   nil,
+			Weekly:  nil,
+			Monthly: nil,
+			Yearly:  nil,
+		}
+	}
+
+	var policies = map[string]*int{
+		"daily":   job.Retention.Daily,
+		"weekly":  job.Retention.Weekly,
+		"monthly": job.Retention.Monthly,
+		"yearly":  job.Retention.Yearly,
+	}
+
+	snapshots := listSnapshots(host)
+
+	for bucket, retention := range policies {
+		for _, snapshot := range listKeepers(snapshots, bucket, retention) {
+			l := log.WithFields(logrus.Fields{
+				"snapshot": snapshot,
+				"bucket":   bucket,
+			})
+
+			if retention == nil {
+				l = l.WithField("retention", "infinite")
+			} else {
+				l = l.WithField("retention", *retention)
+			}
+
+			l.Debug("keeping snapshot")
+		}
+	}
+
+	// TODO subtract keepers from the list of snapshots and rm -rf them
+}
+
+// listKeepers returns a list of snapshot that are not subject to deletion
+// for a given host, pattern, and retention.
+func listKeepers(snapshots []snapshot, bucket string, retention *int) []snapshot {
+	var keepers []snapshot
+	var last string
+
+	for _, snapshot := range snapshots {
+		var period string
+
+		// Weekly is special because golang doesn't have support for "week number in year"
+		// as Time.Format string pattern.
+		if bucket == "weekly" {
+			year, week := snapshot.Time.Local().ISOWeek()
+			period = fmt.Sprintf("%d-%d", year, week)
+		} else {
+			period = snapshot.Time.Local().Format(patterns[bucket])
+		}
+
+		if period != last {
+			last = period
+			keepers = append(keepers, snapshot)
+
+			// nil will keep infinite snapshots
+			if retention == nil {
+				continue
+			}
+
+			if len(keepers) == *retention {
+				break
+			}
+		}
+	}
+
+	return keepers
+}
+
+// listSnapshots calls out to ZFS for a list of snapshots for a given host.
+// Returned data will be sorted by time, most recent first.
+func listSnapshots(host string) []snapshot {
+	var snapshots []snapshot
+
+	ds := newDataset(host)
+
+	args := []string{
+		"list",
+		"-r",         // recursive
+		"-H",         // no field headers in output
+		"-o", "name", // only name field
+		"-t", "snapshot", // type snapshot
+		ds.Name,
+	}
+	o, e, err := execProgram("zfs", args...)
+	if err != nil {
+		f := appendStdlogs(logrus.Fields{
+			logrus.ErrorKey: err,
+			"prefix":        "zfs",
+			"command":       append([]string{"zfs"}, args...),
+		}, o, e)
+		log.WithFields(f).Errorf("executing zfs list failed")
+	}
+
+	for _, ss := range strings.Fields(o.String()) {
+		ts, err := time.Parse(time.RFC3339, strings.Split(ss, "@")[1])
+
+		if err != nil {
+			log.WithField("snapshot", ss).Error("Unable to parse timestamp from snapshot")
+			continue
+		}
+
+		snapshots = append(snapshots, snapshot{
+			Name: ss,
+			Time: ts,
+		})
+	}
+
+	// ZFS list _should_ be in chronological order but just in case ...
+	sort.Slice(snapshots, func(i, j int) bool {
+		return snapshots[i].Time.After(snapshots[j].Time)
+	})
+
+	return snapshots
+}

cmd/prune.go

@@ -0,0 +1,31 @@
+package cmd
+
+import (
+	"github.com/digineo/zackup/app"
+	"github.com/spf13/cobra"
+)
+
+// pruneCmd represents the prune command
+var pruneCmd = &cobra.Command{
+	Use:   "prune [host [...]]",
+	Short: "Prunes backups per-host ZFS dataset",
+	Run: func(cmd *cobra.Command, args []string) {
+		if len(args) == 0 {
+			args = tree.Hosts()
+		}
+
+		for _, host := range args {
+			job := tree.Host(host)
+			if job == nil {
+				log.WithField("prune", host).Warn("unknown host, ignoring")
+				continue
+			}
+
+			app.PruneSnapshots(job)
+		}
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(pruneCmd)
+}

config/job.go

@@ -4,8 +4,9 @@ package config
 type JobConfig struct {
 	host string
 
-	SSH   *SSHConfig   `yaml:"ssh"`
-	RSync *RsyncConfig `yaml:"rsync"`
+	SSH       *SSHConfig       `yaml:"ssh"`
+	RSync     *RsyncConfig     `yaml:"rsync"`
+	Retention *RetentionConfig `yaml:"retention"`
 
 	PreScript  Script `yaml:"pre_script"`  // from yaml file
 	PostScript Script `yaml:"post_script"` // from yaml file
@@ -18,6 +19,14 @@ type SSHConfig struct {
 	Timeout *uint  `yaml:"timeout"` // number of seconds, defaults to 15
 }
 
+// RetentionConfig holds backup retention periods
+type RetentionConfig struct {
+	Daily   *int `yaml:"daily"`   // defaults to infinite
+	Weekly  *int `yaml:"weekly"`  // defaults to infinite
+	Monthly *int `yaml:"monthly"` // defaults to infinite
+	Yearly  *int `yaml:"yearly"`  // defaults to infinite
+}
+
 // Host returns the hostname for this job.
 func (j *JobConfig) Host() string {
 	return j.host
@@ -59,6 +68,26 @@ func (j *JobConfig) mergeGlobals(globals *JobConfig) {
 		}
 	}
 
+	if globals.Retention != nil {
+		if j.Retention == nil {
+			dup := *globals.Retention
+			j.Retention = &dup
+		} else {
+			if j.Retention.Daily == nil {
+				j.Retention.Daily = globals.Retention.Daily
+			}
+			if j.Retention.Weekly == nil {
+				j.Retention.Weekly = globals.Retention.Weekly
+			}
+			if j.Retention.Monthly == nil {
+				j.Retention.Monthly = globals.Retention.Monthly
+			}
+			if j.Retention.Yearly == nil {
+				j.Retention.Yearly = globals.Retention.Yearly
+			}
+		}
+	}
+
 	// globals.PreScript
 	j.PreScript.inline = append(globals.PreScript.inline, j.PreScript.inline...)
 	j.PreScript.scripts = append(globals.PreScript.scripts, j.PreScript.scripts...)

go.mod

@@ -35,3 +35,5 @@ require (
 	gopkg.in/gemnasium/logrus-graylog-hook.v2 v2.0.7
 	gopkg.in/yaml.v2 v2.2.2
 )
+
+go 1.13

testdata/globals.yml

@@ -4,6 +4,12 @@ ssh:
   port: 22
   identity_file: /etc/zackup/id_rsa.pub
 
+retention:
+  daily: 14
+  weekly: 4
+  monthly: 6
+  yearly: 5
+
 rsync:
   included:
   - /etc