Delete target backup (improvements) (#917)

* Delete the dependant delta backups when no find-full flag is specified

* Replace --find-full with FIND_FULL to match the other delete commands

* Add docs

* Add integration test

* Add FIND_FULL integration test

.github/workflows/dockertests.yml

@@ -94,6 +94,8 @@ jobs:
                    'make TEST="pg_delete_before_permanent_full_test" pg_integration_test',
                    'make TEST="pg_delete_before_permanent_delta_test" pg_integration_test',
                    'make TEST="pg_delete_target_test" pg_integration_test',
+                   'make TEST="pg_delete_target_delta_test" pg_integration_test',
+                   'make TEST="pg_delete_target_delta_find_full_test" pg_integration_test',
                    'make mongo_test',
                    'make MONGO_VERSION="4.4.3" MONGO_MAJOR="4.4" mongo_features',
                    'make MONGO_VERSION="4.2.12" MONGO_MAJOR="4.2" mongo_features',

README.md

@@ -198,7 +198,7 @@ Lists names and creation time of available backups.
 
 Is used to delete backups and WALs before them. By default ``delete`` will perform a dry run. If you want to execute deletion, you have to add ``--confirm`` flag at the end of the command. Backups marked as permanent will not be deleted.
 
-``delete`` can operate in three modes: ``retain``, ``before`` and ``everything``.
+``delete`` can operate in four modes: ``retain``, ``before``, ``everything`` and ``target``.
 
 ``retain`` [FULL|FIND_FULL] %number% [--after %name|time%]
 
@@ -211,6 +211,10 @@ If `FIND_FULL` is specified, WAL-G will calculate minimum backup needed to keep
 
 ``everything`` [FORCE]
 
+``target`` [FIND_FULL] %name% | --target-user-data %data% will delete the backup specified by name or user data.
+
+(Only in Postgres) By default, if delta backup is provided as the target, WAL-G will also delete all the dependant delta backups. If `FIND_FULL` is specified, WAL-G will delete all backups with the same base backup as the target.
+
 Examples:
 
 ``everything`` all backups will be deleted (if there are no permanent backups)
@@ -229,6 +233,14 @@ Examples:
 
 ``before FIND_FULL base_000010000123123123`` will keep everything after base of base_000010000123123123
 
+``target base_0000000100000000000000C9`` delete the base backup and all dependant delta backups
+
+``  target --target-user-data "{ \"x\": [3], \"y\": 4 }"`` 	delete backup specified by user data
+
+``target base_0000000100000000000000C9_D_0000000100000000000000C4``	delete delta backup and all dependant delta backups
+
+``target FIND_FULL base_0000000100000000000000C9_D_0000000100000000000000C4`` delete delta backup and all delta backups with the same base backup
+
 **More commands are available for the chosen database engine. See it in [Databases](#databases)**
 
 Databases

cmd/pg/delete.go

@@ -18,7 +18,6 @@ const UseSentinelTimeDescription = "Use backup creation time from sentinel for b
 var confirmed = false
 var useSentinelTime = false
 var deleteTargetUserData = ""
-var findFullBackup = false
 
 // deleteCmd represents the delete command
 var deleteCmd = &cobra.Command{
@@ -48,10 +47,10 @@ var deleteEverythingCmd = &cobra.Command{
 	Run:       runDeleteEverything,
 }
 
-var deleteSingleCmd = &cobra.Command{
+var deleteTargetCmd = &cobra.Command{
 	Use:     internal.DeleteTargetUsageExample, // TODO : improve description
 	Example: internal.DeleteTargetExamples,
-	Args:    cobra.RangeArgs(0, 1),
+	Args:    internal.DeleteTargetArgsValidator,
 	Run:     runDeleteTarget,
 }
 
@@ -123,6 +122,14 @@ func runDeleteTarget(cmd *cobra.Command, args []string) {
 			permanentBackups, permanentWals)
 	}
 
+	findFullBackup := false
+	modifier := internal.ExtractDeleteTargetModifierFromArgs(args)
+	if modifier == internal.FindFullDeleteModifier {
+		findFullBackup = true
+		// remove the extracted modifier from args
+		args = args[1:]
+	}
+
 	deleteHandler, err := newPostgresDeleteHandler(folder, permanentBackups, permanentWals)
 	tracelog.ErrorLogger.FatalOnError(err)
 	targetBackupSelector, err := createTargetDeleteBackupSelector(cmd, args, deleteTargetUserData)
@@ -133,12 +140,10 @@ func runDeleteTarget(cmd *cobra.Command, args []string) {
 func init() {
 	cmd.AddCommand(deleteCmd)
 
-	deleteSingleCmd.Flags().StringVar(
+	deleteTargetCmd.Flags().StringVar(
 		&deleteTargetUserData, internal.DeleteTargetUserDataFlag, "", internal.DeleteTargetUserDataDescription)
-	deleteSingleCmd.Flags().BoolVar(
-		&findFullBackup, internal.FindFullBackupFlag, false, internal.FindFullBackupDescription)
 
-	deleteCmd.AddCommand(deleteRetainCmd, deleteBeforeCmd, deleteEverythingCmd, deleteSingleCmd)
+	deleteCmd.AddCommand(deleteRetainCmd, deleteBeforeCmd, deleteEverythingCmd, deleteTargetCmd)
 	deleteCmd.PersistentFlags().BoolVar(&confirmed, internal.ConfirmFlag, false, "Confirms backup deletion")
 	deleteCmd.PersistentFlags().BoolVar(&useSentinelTime, UseSentinelTimeFlag, false, UseSentinelTimeDescription)
 }
@@ -180,22 +185,25 @@ func newPostgresDeleteHandler(folder storage.Folder, permanentBackups, permanent
 	return deleteHandler, nil
 }
 
-func newPostgresBackupObject(incrementBase string, isFullBackup bool, creationTime time.Time, object storage.Object) PostgresBackupObject {
+func newPostgresBackupObject(incrementBase, incrementFrom string,
+	isFullBackup bool, creationTime time.Time, object storage.Object) PostgresBackupObject {
 	return PostgresBackupObject{
-		Object:         object,
-		isFullBackup:   isFullBackup,
-		baseBackupName: incrementBase,
-		creationTime:   creationTime,
-		BackupName:     internal.FetchPgBackupName(object),
+		Object:            object,
+		isFullBackup:      isFullBackup,
+		baseBackupName:    incrementBase,
+		incrementFromName: incrementFrom,
+		creationTime:      creationTime,
+		BackupName:        internal.FetchPgBackupName(object),
 	}
 }
 
 type PostgresBackupObject struct {
 	storage.Object
-	BackupName     string
-	isFullBackup   bool
-	baseBackupName string
-	creationTime   time.Time
+	BackupName        string
+	isFullBackup      bool
+	baseBackupName    string
+	incrementFromName string
+	creationTime      time.Time
 }
 
 func (o PostgresBackupObject) IsFullBackup() bool {
@@ -214,17 +222,21 @@ func (o PostgresBackupObject) GetBackupName() string {
 	return o.BackupName
 }
 
+func (o PostgresBackupObject) GetIncrementFromName() string {
+	return o.incrementFromName
+}
+
 func makePostgresBackupObjects(
 	folder storage.Folder, objects []storage.Object, startTimeByBackupName map[string]time.Time,
 ) ([]internal.BackupObject, error) {
 	backupObjects := make([]internal.BackupObject, 0, len(objects))
 	for _, object := range objects {
-		incrementBase, isFullBackup, err := postgresGetIncrementInfo(folder, object)
+		incrementBase, incrementFrom, isFullBackup, err := postgresGetIncrementInfo(folder, object)
 		if err != nil {
 			return nil, err
 		}
 		postgresBackup := newPostgresBackupObject(
-			incrementBase, isFullBackup, object.GetLastModified(), object)
+			incrementBase, incrementFrom, isFullBackup, object.GetLastModified(), object)
 
 		if startTimeByBackupName != nil {
 			postgresBackup.creationTime = startTimeByBackupName[postgresBackup.BackupName]
@@ -307,17 +319,17 @@ func postgresTimelineAndSegmentNoLess(object1 storage.Object, object2 storage.Ob
 	return tl1 < tl2 || tl1 == tl2 && segNo1 < segNo2
 }
 
-func postgresGetIncrementInfo(folder storage.Folder, object storage.Object) (string, bool, error) {
+func postgresGetIncrementInfo(folder storage.Folder, object storage.Object) (string, string, bool, error) {
 	backup := internal.NewBackup(folder.GetSubFolder(utility.BaseBackupPath), internal.FetchPgBackupName(object))
 	sentinel, err := backup.GetSentinel()
 	if err != nil {
-		return "", true, err
+		return "", "", true, err
 	}
 	if !sentinel.IsIncremental() {
-		return "", true, nil
+		return "", "", true, nil
 	}
 
-	return *sentinel.IncrementFullName, false, nil
+	return *sentinel.IncrementFullName, *sentinel.IncrementFrom, false, nil
 }
 
 // create the BackupSelector to select the backup to delete

docker-compose.yml

@@ -280,6 +280,28 @@ services:
     links:
       - s3
 
+  pg_delete_target_delta_test:
+    build:
+      dockerfile: docker/pg_tests/Dockerfile_delete_target_delta_test
+      context: .
+    image: wal-g/delete_target_delta_test
+    container_name: wal-g_pg_delete_target_delta_test
+    depends_on:
+      - s3
+    links:
+      - s3
+
+  pg_delete_target_delta_find_full_test:
+    build:
+      dockerfile: docker/pg_tests/Dockerfile_delete_target_delta_find_full_test
+      context: .
+    image: wal-g/delete_target_delta_find_full_test
+    container_name: wal-g_pg_delete_target_delta_find_full_test
+    depends_on:
+      - s3
+    links:
+      - s3
+
   pg_ghost_table_test:
     build:
       dockerfile: docker/pg_tests/Dockerfile_ghost_table_test

docker/pg_tests/Dockerfile_delete_target_delta_find_full_test

@@ -0,0 +1,3 @@
+FROM wal-g/docker_prefix:latest
+
+CMD su postgres -c "/tmp/tests/delete_target_delta_find_full_test.sh"

docker/pg_tests/Dockerfile_delete_target_delta_test

@@ -0,0 +1,3 @@
+FROM wal-g/docker_prefix:latest
+
+CMD su postgres -c "/tmp/tests/delete_target_delta_test.sh"

docker/pg_tests/scripts/configs/delete_target_delta_find_full_test_config.json

@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "100",
+"WALE_S3_PREFIX": "s3://deletetargetbucket",
+"WALG_USE_WAL_DELTA": "true"

docker/pg_tests/scripts/configs/delete_target_delta_test_config.json

@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "100",
+"WALE_S3_PREFIX": "s3://deletetargetbucket",
+"WALG_USE_WAL_DELTA": "true"

docker/pg_tests/scripts/tests/delete_target_delta_find_full_test.sh

@@ -0,0 +1,76 @@
+#!/bin/sh
+set -e -x
+  CONFIG_FILE="/tmp/configs/delete_target_delta_find_full_test_config.json"
+COMMON_CONFIG="/tmp/configs/common_config.json"
+TMP_CONFIG="/tmp/configs/tmp_config.json"
+cat ${CONFIG_FILE} > ${TMP_CONFIG}
+echo "," >> ${TMP_CONFIG}
+cat ${COMMON_CONFIG} >> ${TMP_CONFIG}
+/tmp/scripts/wrap_config_file.sh ${TMP_CONFIG}
+
+/usr/lib/postgresql/10/bin/initdb ${PGDATA}
+
+echo "archive_mode = on" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_command = '/usr/bin/timeout 600 /usr/bin/wal-g --config=${TMP_CONFIG} wal-push %p'" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_timeout = 600" >> /var/lib/postgresql/10/main/postgresql.conf
+
+/usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
+
+/tmp/scripts/wait_while_pg_not_ready.sh
+
+wal-g --config=${TMP_CONFIG} delete everything FORCE --confirm
+
+# create full backup and incremental
+for i in 1 2
+do
+    pgbench -i -s 1 postgres &
+    sleep 1
+    wal-g --config=${TMP_CONFIG} backup-push ${PGDATA}
+done
+
+# remember the backup-list output
+# later in the test we create new backups which should be deleted so lists should be identical
+lines_before_delete=`wal-g --config=${TMP_CONFIG} backup-list | wc -l`
+wal-g --config=${TMP_CONFIG} backup-list | tail -n 2 > /tmp/list_before_delete
+
+# create one full and two increments
+for i in 1 2 3
+do
+    if [ $i -eq 1 ]; then
+       modifier='--full'
+    else
+       modifier=''
+    fi
+    pgbench -i -s 1 postgres &
+    sleep 1
+    wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} ${modifier}
+done
+
+# get the name of the second incremental backup
+SECOND_INCREMENT=$(wal-g --config=${TMP_CONFIG} backup-list | awk 'NR==5 {print $1}')
+
+# make two increments from the SECOND_INCREMENT
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --delta-from-name ${SECOND_INCREMENT}
+
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --delta-from-name ${SECOND_INCREMENT}
+
+# delete the SECOND_INCREMENT with FIND_FULL, should leave only the first full backup w/ first increment
+wal-g --config=${TMP_CONFIG} delete target FIND_FULL ${SECOND_INCREMENT} --confirm
+
+lines_after_delete=`wal-g --config=${TMP_CONFIG} backup-list | wc -l`
+wal-g --config=${TMP_CONFIG} backup-list | tail -n 2 > /tmp/list_after_delete
+
+if [ $(($lines_before_delete)) -ne $lines_after_delete ];
+then
+    echo $(($lines_before_delete)) > /tmp/before_delete
+    echo $lines_after_delete > /tmp/after_delete
+    echo "Wrong number of deleted lines"
+    diff /tmp/before_delete /tmp/after_delete
+fi
+
+
+diff /tmp/list_before_delete /tmp/list_after_delete
+/tmp/scripts/drop_pg.sh
+rm ${TMP_CONFIG}

docker/pg_tests/scripts/tests/delete_target_delta_test.sh

@@ -0,0 +1,77 @@
+#!/bin/sh
+set -e -x
+  CONFIG_FILE="/tmp/configs/delete_target_delta_test_config.json"
+COMMON_CONFIG="/tmp/configs/common_config.json"
+TMP_CONFIG="/tmp/configs/tmp_config.json"
+cat ${CONFIG_FILE} > ${TMP_CONFIG}
+echo "," >> ${TMP_CONFIG}
+cat ${COMMON_CONFIG} >> ${TMP_CONFIG}
+/tmp/scripts/wrap_config_file.sh ${TMP_CONFIG}
+
+/usr/lib/postgresql/10/bin/initdb ${PGDATA}
+
+echo "archive_mode = on" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_command = '/usr/bin/timeout 600 /usr/bin/wal-g --config=${TMP_CONFIG} wal-push %p'" >> /var/lib/postgresql/10/main/postgresql.conf
+echo "archive_timeout = 600" >> /var/lib/postgresql/10/main/postgresql.conf
+
+/usr/lib/postgresql/10/bin/pg_ctl -D ${PGDATA} -w start
+
+/tmp/scripts/wait_while_pg_not_ready.sh
+
+wal-g --config=${TMP_CONFIG} delete everything FORCE --confirm
+
+# create one base backup and one increment
+for i in 1 2
+do
+    pgbench -i -s 1 postgres &
+    sleep 1
+    wal-g --config=${TMP_CONFIG} backup-push ${PGDATA}
+done
+
+# remember the name of the first increment
+FIRST_INCREMENT=$(wal-g --config=${TMP_CONFIG} backup-list | awk 'END {print $1}')
+
+# make the second full backup
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --full
+
+# make the increment from the second full backup
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA}
+
+# remember the backup-list output with two full backups and two increments.
+# later in the test we create new backups which should be deleted so lists should be identical
+lines_before_delete=`wal-g --config=${TMP_CONFIG} backup-list | wc -l`
+wal-g --config=${TMP_CONFIG} backup-list | tail -n 4 > /tmp/list_before_delete
+
+# now make increment from the FIRST_INCREMENT, which will be deleted later
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --delta-from-name ${FIRST_INCREMENT}
+INCREMENT_TO_DELETE=$(wal-g --config=${TMP_CONFIG} backup-list | awk 'END {print $1}')
+
+# make the increment from the INCREMENT_TO_DELETE
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --delta-from-name ${INCREMENT_TO_DELETE}
+
+# make one more increment from the INCREMENT_TO_DELETE
+pgbench -i -s 1 postgres & sleep 1
+wal-g --config=${TMP_CONFIG} backup-push ${PGDATA} --delta-from-name ${INCREMENT_TO_DELETE}
+
+# delete the INCREMENT_TO_DELETE, should leave only the first full backup w/ first increment and the second full backup w/ first increment
+wal-g --config=${TMP_CONFIG} delete target ${INCREMENT_TO_DELETE} --confirm
+
+lines_after_delete=`wal-g --config=${TMP_CONFIG} backup-list | wc -l`
+wal-g --config=${TMP_CONFIG} backup-list | tail -n 4 > /tmp/list_after_delete
+
+if [ $(($lines_before_delete)) -ne $lines_after_delete ];
+then
+    echo $(($lines_before_delete)) > /tmp/before_delete
+    echo $lines_after_delete > /tmp/after_delete
+    echo "Wrong number of deleted lines"
+    diff /tmp/before_delete /tmp/after_delete
+fi
+
+
+diff /tmp/list_before_delete /tmp/list_after_delete
+/tmp/scripts/drop_pg.sh
+rm ${TMP_CONFIG}

internal/backup_object.go

@@ -23,6 +23,10 @@ func (o DefaultBackupObject) GetBaseBackupName() string {
 	return o.GetBackupName()
 }
 
+func (o DefaultBackupObject) GetIncrementFromName() string {
+	return o.GetBackupName()
+}
+
 func (o DefaultBackupObject) IsFullBackup() bool {
 	return true
 }