Delete target backup

.github/workflows/dockertests.yml

@@ -93,6 +93,7 @@ jobs:
                    'make TEST="pg_wale_compatibility_test" pg_integration_test',
                    'make TEST="pg_delete_before_permanent_full_test" pg_integration_test',
                    'make TEST="pg_delete_before_permanent_delta_test" pg_integration_test',
+                   'make TEST="pg_delete_target_test" pg_integration_test',
                    'make mongo_test',
                    'make MONGO_VERSION="4.4.3" MONGO_MAJOR="4.4" mongo_features',
                    'make MONGO_VERSION="4.2.12" MONGO_MAJOR="4.2" mongo_features',

cmd/fdb/delete.go

@@ -1,8 +1,6 @@
 package fdb
 
 import (
-	"time"
-
 	"github.com/spf13/cobra"
 	"github.com/wal-g/storages/storage"
 	"github.com/wal-g/tracelog"
@@ -103,7 +101,7 @@ func newFdbDeleteHandler(folder storage.Folder) (*internal.DeleteHandler, error)
 
 	backupObjects := make([]internal.BackupObject, 0, len(backups))
 	for _, object := range backups {
-		backupObjects = append(backupObjects, FdbBackupObject{object})
+		backupObjects = append(backupObjects, internal.NewDefaultBackupObject(object))
 	}
 
 	return internal.NewDeleteHandler(folder, backupObjects, makeLessFunc()), nil
@@ -119,15 +117,3 @@ func makeLessFunc() func(object1, object2 storage.Object) bool {
 		return time1 < time2
 	}
 }
-
-type FdbBackupObject struct {
-	storage.Object
-}
-
-func (o FdbBackupObject) IsFullBackup() bool {
-	return true
-}
-
-func (o FdbBackupObject) GetBackupTime() time.Time {
-	return o.Object.GetLastModified()
-}

cmd/mysql/delete.go

@@ -78,27 +78,16 @@ func runDeleteEverything(cmd *cobra.Command, args []string) {
 
 	deleteHandler.DeleteEverything(confirmed)
 }
-func (h *DeleteHandler) deleteTarget(bobj internal.BackupObject, confirmed bool) error {
-	return storage.DeleteObjectsWhere(h.Folder.GetSubFolder(utility.BaseBackupPath), confirmed, func(object storage.Object) bool {
-		return strings.HasPrefix(object.GetName(), strings.TrimSuffix(bobj.GetName(), utility.SentinelSuffix))
-	})
-}
 
 func runDeleteTarget(cmd *cobra.Command, args []string) {
 	deleteHandler, err := NewMySqlDeleteHandler()
 	tracelog.ErrorLogger.FatalOnError(err)
-	bname := args[0] // backup name
 
-	for e := range deleteHandler.permanentObjects {
-		if e == bname {
-			tracelog.InfoLogger.Fatalf("unable to delete permanent backup %s\n", bname)
-		}
-	}
-	if bobj, err := deleteHandler.FindTargetByName(bname); err != nil {
-		tracelog.ErrorLogger.FatalfOnError("unable to delete target backup", err)
-	} else if err := deleteHandler.deleteTarget(bobj, confirmed); err != nil {
-		tracelog.ErrorLogger.FatalfOnError("unable to delete target backup", err)
-	}
+	bname := args[0]                                             // backup name
+	backupSelector, err := internal.NewBackupNameSelector(bname) //todo: add selection by userdata
+	tracelog.ErrorLogger.PrintOnError(err)
+
+	deleteHandler.HandleDeleteTarget(backupSelector, confirmed, false)
 }
 
 func runDeleteBefore(cmd *cobra.Command, args []string) {
@@ -212,7 +201,7 @@ func NewMySqlDeleteHandler() (*DeleteHandler, error) {
 
 	backupObjects := make([]internal.BackupObject, 0, len(backups))
 	for _, object := range backups {
-		b := mysql.BackupObject{Object: object}
+		b := internal.NewDefaultBackupObject(object)
 		backupObjects = append(backupObjects, b)
 	}
 

cmd/pg/backup_fetch.go

@@ -3,7 +3,6 @@ package pg
 import (
 	"fmt"
 
-	"github.com/pkg/errors"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/wal-g/storages/storage"
@@ -26,17 +25,17 @@ var fileMask string
 var restoreSpec string
 var reverseDeltaUnpack bool
 var skipRedundantTars bool
-var targetUserData string
+var fetchTargetUserData string
 
 var backupFetchCmd = &cobra.Command{
 	Use:   "backup-fetch destination_directory [backup_name | --target-user-data <data>]",
 	Short: backupFetchShortDescription, // TODO : improve description
 	Args:  cobra.RangeArgs(1, 2),
 	Run: func(cmd *cobra.Command, args []string) {
-		if targetUserData == "" {
-			targetUserData = viper.GetString(internal.TargetUserDataSetting)
+		if fetchTargetUserData == "" {
+			fetchTargetUserData = viper.GetString(internal.FetchTargetUserDataSetting)
 		}
-		targetBackupSelector, err := createTargetBackupSelector(cmd, args)
+		targetBackupSelector, err := createTargetFetchBackupSelector(cmd, args, fetchTargetUserData)
 		tracelog.ErrorLogger.FatalOnError(err)
 
 		folder, err := internal.ConfigureFolder()
@@ -56,29 +55,18 @@ var backupFetchCmd = &cobra.Command{
 }
 
 // create the BackupSelector to select the backup to fetch
-func createTargetBackupSelector(cmd *cobra.Command, args []string) (internal.BackupSelector, error) {
-	var err error
-	switch {
-	case len(args) == 2 && targetUserData != "":
-		err = errors.New("Incorrect arguments. Specify backup_name OR target flag, not both.")
-
-	case len(args) == 2 && args[1] == internal.LatestString:
-		tracelog.InfoLogger.Printf("Fetching the latest backup...\n")
-		return internal.NewLatestBackupSelector(), nil
-
-	case len(args) == 2:
-		tracelog.InfoLogger.Printf("Fetching the backup with name %s...\n", args[1])
-		return internal.NewBackupNameSelector(args[1])
-
-	case len(args) == 1 && targetUserData != "":
-		tracelog.InfoLogger.Println("Fetching the backup with the specified user data...")
-		return internal.NewUserDataBackupSelector(targetUserData), nil
+func createTargetFetchBackupSelector(cmd *cobra.Command, args []string, targetUserData string) (internal.BackupSelector, error) {
+	targetName := ""
+	if len(args) >= 2 {
+		targetName = args[1]
+	}
 
-	default:
-		err = errors.New("Insufficient arguments.")
+	backupSelector, err := internal.NewTargetBackupSelector(targetUserData, targetName)
+	if err != nil {
+		fmt.Println(cmd.UsageString())
+		return nil, err
 	}
-	fmt.Println(cmd.UsageString())
-	return nil, err
+	return backupSelector, nil
 }
 
 func init() {
@@ -88,7 +76,7 @@ func init() {
 		false, reverseDeltaUnpackDescription)
 	backupFetchCmd.Flags().BoolVar(&skipRedundantTars, "skip-redundant-tars",
 		false, skipRedundantTarsDescription)
-	backupFetchCmd.Flags().StringVar(&targetUserData, "target-user-data",
+	backupFetchCmd.Flags().StringVar(&fetchTargetUserData, "target-user-data",
 		"", targetUserDataDescription)
 	cmd.AddCommand(backupFetchCmd)
 }

cmd/pg/delete.go

@@ -2,7 +2,6 @@ package pg
 
 import (
 	"fmt"
-	"regexp"
 	"time"
 
 	"github.com/pkg/errors"
@@ -18,8 +17,8 @@ const UseSentinelTimeDescription = "Use backup creation time from sentinel for b
 
 var confirmed = false
 var useSentinelTime = false
-var patternBackupName = fmt.Sprintf("base_%[1]s(_D_%[1]s)?", internal.PatternTimelineAndLogSegNo)
-var regexpBackupName = regexp.MustCompile(patternBackupName)
+var deleteTargetUserData = ""
+var findFullBackup = false
 
 // deleteCmd represents the delete command
 var deleteCmd = &cobra.Command{
@@ -49,6 +48,13 @@ var deleteEverythingCmd = &cobra.Command{
 	Run:       runDeleteEverything,
 }
 
+var deleteSingleCmd = &cobra.Command{
+	Use:     internal.DeleteTargetUsageExample, // TODO : improve description
+	Example: internal.DeleteTargetExamples,
+	Args:    cobra.RangeArgs(0, 1),
+	Run:     runDeleteTarget,
+}
+
 func runDeleteBefore(cmd *cobra.Command, args []string) {
 	folder, err := internal.ConfigureFolder()
 	tracelog.ErrorLogger.FatalOnError(err)
@@ -107,10 +113,32 @@ func runDeleteEverything(cmd *cobra.Command, args []string) {
 	deleteHandler.DeleteEverything(confirmed)
 }
 
+func runDeleteTarget(cmd *cobra.Command, args []string) {
+	folder, err := internal.ConfigureFolder()
+	tracelog.ErrorLogger.FatalOnError(err)
+
+	permanentBackups, permanentWals := internal.GetPermanentObjects(folder)
+	if len(permanentBackups) > 0 {
+		tracelog.InfoLogger.Printf("Found permanent objects: backups=%v, wals=%v\n",
+			permanentBackups, permanentWals)
+	}
+
+	deleteHandler, err := newPostgresDeleteHandler(folder, permanentBackups, permanentWals)
+	tracelog.ErrorLogger.FatalOnError(err)
+	targetBackupSelector, err := createTargetDeleteBackupSelector(cmd, args, deleteTargetUserData)
+	tracelog.ErrorLogger.FatalOnError(err)
+	deleteHandler.HandleDeleteTarget(targetBackupSelector, confirmed, findFullBackup)
+}
+
 func init() {
 	cmd.AddCommand(deleteCmd)
 
-	deleteCmd.AddCommand(deleteRetainCmd, deleteBeforeCmd, deleteEverythingCmd)
+	deleteSingleCmd.Flags().StringVar(
+		&deleteTargetUserData, internal.DeleteTargetUserDataFlag, "", internal.DeleteTargetUserDataDescription)
+	deleteSingleCmd.Flags().BoolVar(
+		&findFullBackup, internal.FindFullBackupFlag, false, internal.FindFullBackupDescription)
+
+	deleteCmd.AddCommand(deleteRetainCmd, deleteBeforeCmd, deleteEverythingCmd, deleteSingleCmd)
 	deleteCmd.PersistentFlags().BoolVar(&confirmed, internal.ConfirmFlag, false, "Confirms backup deletion")
 	deleteCmd.PersistentFlags().BoolVar(&useSentinelTime, UseSentinelTimeFlag, false, UseSentinelTimeDescription)
 }
@@ -136,7 +164,10 @@ func newPostgresDeleteHandler(folder storage.Folder, permanentBackups, permanent
 			lessFunc = makeLessFunc(startTimeByBackupName)
 		}
 	}
-	postgresBackups := makePostgresBackupObjects(folder, backups, startTimeByBackupName)
+	postgresBackups, err := makePostgresBackupObjects(folder, backups, startTimeByBackupName)
+	if err != nil {
+		return nil, err
+	}
 
 	deleteHandler := internal.NewDeleteHandler(
 		folder,
@@ -149,41 +180,58 @@ func newPostgresDeleteHandler(folder storage.Folder, permanentBackups, permanent
 	return deleteHandler, nil
 }
 
-func newPostgresBackupObject(isFullBackup bool, creationTime time.Time, object storage.Object) PostgresBackupObject {
+func newPostgresBackupObject(incrementBase string, isFullBackup bool, creationTime time.Time, object storage.Object) PostgresBackupObject {
 	return PostgresBackupObject{
-		Object:       object,
-		isFullBackup: isFullBackup,
-		creationTime: creationTime,
+		Object:         object,
+		isFullBackup:   isFullBackup,
+		baseBackupName: incrementBase,
+		creationTime:   creationTime,
+		BackupName:     internal.FetchPgBackupName(object),
 	}
 }
 
 type PostgresBackupObject struct {
 	storage.Object
-	isFullBackup bool
-	creationTime time.Time
+	BackupName     string
+	isFullBackup   bool
+	baseBackupName string
+	creationTime   time.Time
 }
 
 func (o PostgresBackupObject) IsFullBackup() bool {
 	return o.isFullBackup
 }
 
+func (o PostgresBackupObject) GetBaseBackupName() string {
+	return o.baseBackupName
+}
+
 func (o PostgresBackupObject) GetBackupTime() time.Time {
 	return o.creationTime
 }
 
-func makePostgresBackupObjects(folder storage.Folder, objects []storage.Object, startTimeByBackupName map[string]time.Time) []internal.BackupObject {
+func (o PostgresBackupObject) GetBackupName() string {
+	return o.BackupName
+}
+
+func makePostgresBackupObjects(
+	folder storage.Folder, objects []storage.Object, startTimeByBackupName map[string]time.Time,
+) ([]internal.BackupObject, error) {
 	backupObjects := make([]internal.BackupObject, 0, len(objects))
 	for _, object := range objects {
+		incrementBase, isFullBackup, err := postgresGetIncrementInfo(folder, object)
+		if err != nil {
+			return nil, err
+		}
 		postgresBackup := newPostgresBackupObject(
-			postgresIsFullBackup(folder, object), object.GetLastModified(), object)
+			incrementBase, isFullBackup, object.GetLastModified(), object)
 
 		if startTimeByBackupName != nil {
-			backupName := fetchBackupName(object)
-			postgresBackup.creationTime = startTimeByBackupName[backupName]
+			postgresBackup.creationTime = startTimeByBackupName[postgresBackup.BackupName]
 		}
 		backupObjects = append(backupObjects, postgresBackup)
 	}
-	return backupObjects
+	return backupObjects, nil
 }
 
 func makePostgresPermanentFunc(permanentBackups, permanentWals map[string]bool) func(object storage.Object) bool {
@@ -194,13 +242,13 @@ func makePostgresPermanentFunc(permanentBackups, permanentWals map[string]bool)
 
 func makeLessFunc(startTimeByBackupName map[string]time.Time) func(storage.Object, storage.Object) bool {
 	return func(object1 storage.Object, object2 storage.Object) bool {
-		backupName1 := fetchBackupName(object1)
+		backupName1 := internal.FetchPgBackupName(object1)
 		if backupName1 == "" {
 			// we can't compare non-backup storage objects (probably WAL segments) by start time,
 			// so use the segment number comparator instead
 			return postgresSegmentNoLess(object1, object2)
 		}
-		backupName2 := fetchBackupName(object2)
+		backupName2 := internal.FetchPgBackupName(object2)
 		if backupName2 == "" {
 			return postgresSegmentNoLess(object1, object2)
 		}
@@ -259,12 +307,30 @@ func postgresTimelineAndSegmentNoLess(object1 storage.Object, object2 storage.Ob
 	return tl1 < tl2 || tl1 == tl2 && segNo1 < segNo2
 }
 
-func postgresIsFullBackup(folder storage.Folder, object storage.Object) bool {
-	backup := internal.NewBackup(folder.GetSubFolder(utility.BaseBackupPath), fetchBackupName(object))
-	sentinel, _ := backup.GetSentinel()
-	return !sentinel.IsIncremental()
+func postgresGetIncrementInfo(folder storage.Folder, object storage.Object) (string, bool, error) {
+	backup := internal.NewBackup(folder.GetSubFolder(utility.BaseBackupPath), internal.FetchPgBackupName(object))
+	sentinel, err := backup.GetSentinel()
+	if err != nil {
+		return "", true, err
+	}
+	if !sentinel.IsIncremental() {
+		return "", true, nil
+	}
+
+	return *sentinel.IncrementFullName, false, nil
 }
 
-func fetchBackupName(object storage.Object) string {
-	return regexpBackupName.FindString(object.GetName())
+// create the BackupSelector to select the backup to delete
+func createTargetDeleteBackupSelector(cmd *cobra.Command, args []string, targetUserData string) (internal.BackupSelector, error) {
+	targetName := ""
+	if len(args) > 0 {
+		targetName = args[0]
+	}
+
+	backupSelector, err := internal.NewTargetBackupSelector(targetUserData, targetName)
+	if err != nil {
+		fmt.Println(cmd.UsageString())
+		return nil, err
+	}
+	return backupSelector, nil
 }

cmd/sqlserver/delete.go

@@ -1,8 +1,6 @@
 package sqlserver
 
 import (
-	"time"
-
 	"github.com/spf13/cobra"
 	"github.com/wal-g/storages/storage"
 	"github.com/wal-g/tracelog"
@@ -79,24 +77,12 @@ func newSqlServerDeleteHandler() (*internal.DeleteHandler, error) {
 
 	backupObjects := make([]internal.BackupObject, 0, len(backups))
 	for _, object := range backups {
-		backupObjects = append(backupObjects, SqlServerBackupObject{object})
+		backupObjects = append(backupObjects, internal.NewDefaultBackupObject(object))
 	}
 
 	return internal.NewDeleteHandler(folder, backupObjects, makeLessFunc()), nil
 }
 
-type SqlServerBackupObject struct {
-	storage.Object
-}
-
-func (o SqlServerBackupObject) IsFullBackup() bool {
-	return true
-}
-
-func (o SqlServerBackupObject) GetBackupTime() time.Time {
-	return o.Object.GetLastModified()
-}
-
 func makeLessFunc() func(object1, object2 storage.Object) bool {
 	return func(object1, object2 storage.Object) bool {
 		time1, ok1 := utility.TryFetchTimeRFC3999(object1.GetName())