feat: automatic backup/restore of db from s3

backup/backup_test.go

@@ -0,0 +1,55 @@
+package backup
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestCompress(t *testing.T) {
+	type args struct {
+		b []byte
+	}
+	tests := []struct {
+		name    string
+		args    args
+		want    int
+		wantErr bool
+	}{
+		// TODO: Add test cases.
+		{
+			"Compress and decress text",
+			args{
+				b: []byte("Foo"),
+			},
+			3,
+			false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+
+			w := &bytes.Buffer{}
+			r := bytes.NewReader(tt.args.b)
+			got, err := Compress(r, w)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("Compress() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("Compress() = '%v', want '%v'", got, tt.want)
+			}
+			rw := &bytes.Buffer{}
+			rw.Write(w.Bytes())
+			ww := &bytes.Buffer{}
+			n, err := Decompress(rw, ww)
+
+			if n != len(tt.args.b) {
+				t.Errorf("Decompress() n = '%v', want '%v'", n, len(tt.args.b))
+			}
+
+			if ww.String() != string(tt.args.b) {
+				t.Errorf("Decompress() = '%v', want '%v'", ww.String(), string(tt.args.b))
+			}
+		})
+	}
+}

bboltStorage/backup.go

@@ -0,0 +1,64 @@
+package bboltStorage
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/dustin/go-humanize"
+	bolt "go.etcd.io/bbolt"
+)
+
+// Backups the database to a writer.
+// Note that bbolt seems to always change the file (statistics?),
+// so creating a hash of the the database does not
+func (s *BBolter) Backup(w io.Writer) (int64, error) {
+	var originalSize int64 = -1
+	var compactSize int64 = -1
+	s.DB.View(func(tx *bolt.Tx) error {
+		originalSize = tx.Size()
+		return nil
+	})
+	s.l.Info().Msg("Creating backup of database, by first compacting the current database to a new temporary database")
+	compactPath := "__compact-backup.bbolt"
+	if fileExists(compactPath) {
+		if err := os.Remove(compactPath); err != nil {
+			return 0, err
+		}
+	}
+	defer os.Remove(compactPath)
+
+	compactDb, err := s.copyCompact(compactPath)
+	if compactDb != nil {
+		defer compactDb.Close()
+	}
+	if err != nil {
+		return 0, err
+	}
+	var written int64
+	err = compactDb.View((func(tx *bolt.Tx) error {
+		compactSize = tx.Size()
+		n, err := tx.WriteTo(w)
+		written = n
+		return err
+	}))
+	if err != nil {
+		return written, err
+	}
+	if written == 0 {
+		err = fmt.Errorf("The data written was of zero size")
+		return written, err
+	}
+	s.l.Debug().
+		Str("originalSize", humanize.Bytes(uint64(originalSize))).
+		Str("compactSize", humanize.Bytes(uint64(compactSize))).
+		Str("percentage", fmt.Sprintf("%2f", float64(compactSize)/float64(originalSize)*100)).
+		Msg("Compact-size-difference")
+	return written, err
+}
+
+func fileExists(filePath string) bool {
+	_, error := os.Stat(filePath)
+	return !errors.Is(error, os.ErrNotExist)
+}

bboltStorage/cleanup.go

@@ -14,24 +14,34 @@ import (
 // Note that performance does not suffer, this is only to reduce disk-pressure
 // If the database gets too big.
 
-func (s *BBolter) compactDatabase() error {
-	path := "_compact.bbolt"
-	originalPath := s.Path()
+func (s *BBolter) copyCompact(path string) (*bolt.DB, error) {
 	compactDb, err := bolt.Open(path, 0666, &bolt.Options{
 		Timeout: 1 * time.Second,
 	})
 	if err != nil {
 		s.l.Error().Err(err).Msg("Failed to create a new database before compacting this one")
-		return fmt.Errorf("Failed to create a new database before compacting this one")
+		return compactDb, fmt.Errorf("Failed to create a new database before compacting this one")
 	}
 	s.l.Warn().Msg("New database was opened")
 	err = bolt.Compact(compactDb, s.DB, 0)
 	if err != nil {
 		s.l.Error().Err(err).Msg("Failed to compact database")
-		return fmt.Errorf("Failed to compact database")
+		return compactDb, fmt.Errorf("Failed to compact database")
 	}
-	s.l.Warn().Msg("New database was compacted. Will now close existing database.")
+	return compactDb, nil
+}
+func (s *BBolter) compactDatabase() error {
+	path := "_compact.bbolt"
+	compactDb, err := s.copyCompact(path)
+	if compactDb != nil {
+		defer compactDb.Close()
+	}
+	if err != nil {
+		return err
+	}
+	originalPath := s.Path()
 	compactDb.Close()
+	s.l.Warn().Msg("New database was compacted. Will now close existing database.")
 	s.Close()
 	s.l.Warn().Msg("Closed databases. Will now rename databases on disk")
 	err = os.Rename(originalPath, originalPath+".bk")

bboltStorage/storage.go

@@ -4,6 +4,7 @@ import (
 	"encoding/gob"
 	"errors"
 	"os"
+	"sync"
 	"time"
 
 	"github.com/dustin/go-humanize"
@@ -74,7 +75,17 @@ func NewBbolt(l logger.AppLogger, path string, pubsub PubSubPublisher, options .
 	return
 }
 
+func (s *BBolter) WriteState() WriteStats {
+	s.writeStats.Lock()
+	defer s.writeStats.Unlock()
+	return s.writeStats.WriteStats
+}
 func (s *BBolter) PublishChange(kind PubType, variant PubVerb, contents interface{}) {
+	s.writeStats.Lock()
+	now := time.Now()
+	s.writeStats.LastWrite = &now
+	s.writeStats.Unlock()
+
 	if s.pubsub == nil {
 		return
 	}
@@ -219,12 +230,21 @@ func (bb *BBolter) Iterate(bucket []byte, f func(key, b []byte) bool) error {
 	return err
 }
 
+type writeStats struct {
+	WriteStats
+	sync.Mutex
+}
+type WriteStats struct {
+	LastWrite *time.Time
+}
+
 type BBolter struct {
 	*bolt.DB
 	pubsub PubSubPublisher
 	l      logger.AppLogger
 	Marshaller
 	idgenerator IDGenerator
+	writeStats  writeStats
 }
 
 type IDGenerator interface {

config/config.go

@@ -56,6 +56,9 @@ type BackupConfig struct {
 	// The database can be backed up as often as every write, but can be relaxed with this value.
 	// Defaults to 10 minutes
 	MaxInterval Duration `json:"maxInterval" help:"The database can be backed up as often as every write, but can be relaxed with this value. Defaults to 10 minutes."`
+	// Can be used to set a custom objectkey.
+	// defaults to "skiver.bbolt"
+	FileName string
 }
 
 type Uploader struct {

config/duration.go

@@ -20,14 +20,7 @@ func (d Duration) Duration() time.Duration {
 	return time.Duration(d)
 }
 func (d *Duration) UnmarshalText(b []byte) error {
-	fmt.Println("fooish", string(b))
 	return d.UnmarshalJSON(b)
-	// x, err := time.ParseDuration(string(b))
-	// if err != nil {
-	// 	return err
-	// }
-	// *d = Duration(x)
-	// return nil
 }
 func (d Duration) MarshalText() (text []byte, err error) {
 	return []byte(time.Duration(d).String()), nil
@@ -98,7 +91,7 @@ func (Duration) JSONSchema() *jsonschema.Schema {
 	return &jsonschema.Schema{
 		Type:        "string",
 		Title:       "Duration-type",
-		Description: fmt.Sprintf("Textual representation of a duration", Examples...),
+		Description: fmt.Sprintf("Textual representation of a duration %s", Examples),
 		Examples:    Examples,
 	}
 }

go.mod

@@ -74,6 +74,7 @@ require (
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/Masterminds/sprig v2.22.0+incompatible
 	github.com/NYTimes/gziphandler v1.1.1
+	github.com/bep/debounce v1.2.1
 	github.com/dustin/go-humanize v1.0.0
 	github.com/fsnotify/fsnotify v1.5.1
 	github.com/ghodss/yaml v1.0.0

go.sum

@@ -79,6 +79,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bep/debounce v1.2.1 h1:v67fRdBA9UQu2NhLFXrSg0Brw7CexQekrBwDMM8bzeY=
+github.com/bep/debounce v1.2.1/go.mod h1:H8yggRPQKLUhUoqrJC1bO2xNya7vanpDl7xR3ISbCJ0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE=

main.go

@@ -381,6 +381,18 @@ func main() {
 
 	shutdown := make(chan os.Signal, 1)
 	signal.Notify(shutdown, syscall.SIGINT, syscall.SIGTERM)
+	type BackupHandler interface {
+		AutoCreateBackupAndSaveIfRequired(source backup.BackerUpper) error
+		WriteNewestBackup(path string) error
+	}
+	var bak BackupHandler
+	if len(config.DatabaseBackups) > 0 {
+		bak = backup.NewBackHandler(logger.GetLogger("backup"), config.DatabaseBackups)
+		err := bak.WriteNewestBackup(apiConfig.DBLocation)
+		if err != nil {
+			l.Fatal().Err(err).Msg("Failed to get the newest backup from backupsource")
+		}
+	}
 	// IMPORTANT: database publishes changes, but for performance-reasons, it should not be used until the listener (ws) is started.
 	db, err := bboltStorage.NewBbolt(l, apiConfig.DBLocation, &events)
 	if err != nil {
@@ -431,10 +443,8 @@ func main() {
 	if l.HasDebug() {
 		events.AddSubscriber("log", &logPublisher{logger.GetLogger("events")})
 	}
-	if len(config.DatabaseBackups) > 0 {
+	if bak != nil {
 		l.Info().Msg("creating backuphandler")
-		bak := backup.NewBackHandler(logger.GetLogger("backup"), config.DatabaseBackups)
-		go bak.AutoCreateBackupAndSaveIfRequired(&db)
 		events.AddSubscriberFunc("backups", func(kind, variant string, contents interface{}) {
 			go bak.AutoCreateBackupAndSaveIfRequired(&db)
 		})

types/storage.go

@@ -1,10 +1,14 @@
 package types
 
 import (
+	"io"
 	"reflect"
 	"time"
 )
 
+type DatabaseBackup interface {
+	Backup(w io.Writer) (int64, error)
+}
 type UserStorage interface {
 	GetUser(userId string) (*User, error)
 	FindUsers(max int, filter ...User) (map[string]User, error)