Replace wal-purge with delete garbage command

.github/workflows/dockertests.yml

@@ -100,7 +100,7 @@ jobs:
                    'make TEST="pg_delete_target_delta_test" pg_integration_test',
                    'make TEST="pg_delete_target_delta_find_full_test" pg_integration_test',
                    'make TEST="pg_backup_mark_permanent_no_error_test" pg_integration_test',
-                   'make TEST="pg_wal_purge_test" pg_integration_test',
+                   'make TEST="pg_delete_garbage_test" pg_integration_test',
                    'make mongo_test',
                    'make MONGO_VERSION="5.0.1" MONGO_MAJOR="5.0" MONGO_REPO="repo.mongodb.org" MONGO_PACKAGE="mongodb-org" mongo_features',
                    'make MONGO_VERSION="5.0.1" MONGO_MAJOR="5.0" MONGO_REPO="repo.mongodb.com" MONGO_PACKAGE="mongodb-enterprise" mongo_features',

cmd/pg/delete.go

@@ -1,15 +1,18 @@
 package pg
 
 import (
-	"github.com/wal-g/wal-g/internal/databases/postgres"
-
 	"github.com/spf13/cobra"
 	"github.com/wal-g/tracelog"
 	"github.com/wal-g/wal-g/internal"
+	"github.com/wal-g/wal-g/internal/databases/postgres"
 )
 
 const UseSentinelTimeFlag = "use-sentinel-time"
 const UseSentinelTimeDescription = "Use backup creation time from sentinel for backups ordering."
+const DeleteGarbageExamples = `  garbage           Deletes outdated WAL archives and leftover backups files from storage
+  garbage ARCHIVES  Deletes only outdated WAL archives from storage
+  garbage BACKUPS   Deletes only leftover backups files from storage`
+const DeleteGarbageUse = "garbage [ARCHIVES|BACKUPS"
 
 var confirmed = false
 var useSentinelTime = false
@@ -50,6 +53,13 @@ var deleteTargetCmd = &cobra.Command{
 	Run:     runDeleteTarget,
 }
 
+var deleteGarbageCmd = &cobra.Command{
+	Use:     DeleteGarbageUse,
+	Example: DeleteGarbageExamples,
+	Args:    DeleteGarbageArgsValidator,
+	Run:     runDeleteGarbage,
+}
+
 func runDeleteBefore(cmd *cobra.Command, args []string) {
 	folder, err := internal.ConfigureFolder()
 	tracelog.ErrorLogger.FatalOnError(err)
@@ -107,13 +117,31 @@ func runDeleteTarget(cmd *cobra.Command, args []string) {
 	deleteHandler.HandleDeleteTarget(targetBackupSelector, confirmed, findFullBackup)
 }
 
+func runDeleteGarbage(cmd *cobra.Command, args []string) {
+	folder, err := internal.ConfigureFolder()
+	tracelog.ErrorLogger.FatalOnError(err)
+
+	permanentBackups, permanentWals := postgres.GetPermanentBackupsAndWals(folder)
+
+	deleteHandler, err := postgres.NewDeleteHandler(folder, permanentBackups, permanentWals, false)
+	tracelog.ErrorLogger.FatalOnError(err)
+
+	err = deleteHandler.HandleDeleteGarbage(args, folder, confirmed)
+	tracelog.ErrorLogger.FatalOnError(err)
+}
+
+func DeleteGarbageArgsValidator(cmd *cobra.Command, args []string) error {
+	modifiers := []string{postgres.DeleteGarbageArchivesModifier, postgres.DeleteGarbageBackupsModifier}
+	return internal.DeleteArgsValidator(args, modifiers, 0, 1)
+}
+
 func init() {
 	Cmd.AddCommand(deleteCmd)
 
 	deleteTargetCmd.Flags().StringVar(
 		&deleteTargetUserData, internal.DeleteTargetUserDataFlag, "", internal.DeleteTargetUserDataDescription)
 
-	deleteCmd.AddCommand(deleteRetainCmd, deleteBeforeCmd, deleteEverythingCmd, deleteTargetCmd)
+	deleteCmd.AddCommand(deleteRetainCmd, deleteBeforeCmd, deleteEverythingCmd, deleteTargetCmd, deleteGarbageCmd)
 	deleteCmd.PersistentFlags().BoolVar(&confirmed, internal.ConfirmFlag, false, "Confirms backup deletion")
 	deleteCmd.PersistentFlags().BoolVar(&useSentinelTime, UseSentinelTimeFlag, false, UseSentinelTimeDescription)
 }

docker-compose.yml

@@ -52,7 +52,7 @@ services:
       &&  mkdir -p /export/deletewithoutconfirm
       &&  mkdir -p /export/deleteendtoendbucket
       &&  mkdir -p /export/deletetargetbucket
-      &&  mkdir -p /export/walpurgebucket
+      &&  mkdir -p /export/deletegarbagebucket
       &&  mkdir -p /export/mysqlfullxtrabackupbucket
       &&  mkdir -p /export/mysqlfullxtrabackupwithrangesbucket
       &&  mkdir -p /export/mysqlfullmysqldumpbucket
@@ -368,12 +368,12 @@ services:
     links:
       - s3
 
-  pg_wal_purge_test:
+  pg_delete_garbage_test:
     build:
-      dockerfile: docker/pg_tests/Dockerfile_wal_purge_test
+      dockerfile: docker/pg_tests/Dockerfile_delete_garbage_test
       context: .
-    image: wal-g/wal_purge_test
-    container_name: wal-g_pg_wal_purge_test
+    image: wal-g/delete_garbage_test
+    container_name: wal-g_pg_delete_garbage_test
     depends_on:
       - s3
     links:

docker/pg_tests/Dockerfile_delete_garbage_test

@@ -0,0 +1,3 @@
+FROM wal-g/docker_prefix:latest
+
+CMD su postgres -c "/tmp/tests/delete_garbage_test.sh"

docker/pg_tests/scripts/configs/delete_garbage_test_config.json

@@ -0,0 +1,3 @@
+"WALG_DELTA_MAX_STEPS": "0",
+"WALE_S3_PREFIX": "s3://deletegarbagebucket",
+"WALG_USE_WAL_DELTA": "true"

docker/pg_tests/scripts/tests/wal_purge_test.sh → docker/pg_tests/scripts/tests/delete_garbage_test.sh

@@ -1,7 +1,7 @@
 #!/bin/sh
 set -e -x
 
-CONFIG_FILE="/tmp/configs/wal_purge_test_config.json"
+CONFIG_FILE="/tmp/configs/delete_garbage_test_config.json"
 COMMON_CONFIG="/tmp/configs/common_config.json"
 TMP_CONFIG="/tmp/configs/tmp_config.json"
 cat ${CONFIG_FILE} > ${TMP_CONFIG}
@@ -47,13 +47,13 @@ FIRST_NON_PERMANENT_BACKUP=$(wal-g --config=${TMP_CONFIG} backup-list | awk 'NR=
 wal-g --config=${TMP_CONFIG} delete target ${FIRST_NON_PERMANENT_BACKUP} --confirm
 
 # should delete WALs in ranges (0, PERMANENT_BACKUP) and (PERMANENT_BACKUP, second non-permanent backup)
-wal-g --config=${TMP_CONFIG} wal-purge --confirm
+wal-g --config=${TMP_CONFIG} delete garbage --confirm
 
 FIRST_BACKUP=$(wal-g --config=${TMP_CONFIG} backup-list | awk 'NR==2{print $1}')
 
 if [ "$PERMANENT_BACKUP" != "$FIRST_BACKUP" ];
 then
-    echo "oh no! wal-purge deleted the permanent backup!"
+    echo "oh no! delete garbage deleted the permanent backup!"
     exit 1
 fi
 

docs/PostgreSQL.md

@@ -501,13 +501,15 @@ Flags:
 - `-t, --to string` Storage config to where should copy backup
 - `-w, --without-history` Copy backup without history (wal files)
 
-### ``wal-purge``
+### ``delete garbage``
 
-Purges outdated WAL archives from storage. Will remove all WAL archives before the earliest non-permanent backup.
+Deletes outdated WAL archives and backups leftover files from storage, e.g. unsuccessfully backups or partially deleted ones. Will remove all non-permanent objects before the earliest non-permanent backup. This command is useful when backups are being deleted by the `delete target` command.
 
 Usage:
 ```bash
-wal-g wal-purge
+wal-g delete garbage           # Deletes outdated WAL archives and leftover backups files from storage
+wal-g delete garbage ARCHIVES      # Deletes only outdated WAL archives from storage
+wal-g delete garbage BACKUPS       # Deletes only leftover (partially deleted or unsuccessful) backups files from storage
 ```
 
 ### ``wal-restore``

internal/databases/postgres/delete.go

@@ -10,9 +10,18 @@ import (
 	"github.com/wal-g/wal-g/utility"
 )
 
+type DeleteHandler struct {
+	internal.DeleteHandler
+}
+
+const (
+	DeleteGarbageArchivesModifier = "ARCHIVES"
+	DeleteGarbageBackupsModifier  = "BACKUPS"
+)
+
 func NewDeleteHandler(folder storage.Folder, permanentBackups, permanentWals map[string]bool,
 	useSentinelTime bool,
-) (*internal.DeleteHandler, error) {
+) (*DeleteHandler, error) {
 	backups, err := internal.GetBackupSentinelObjects(folder)
 	if err != nil {
 		return nil, err
@@ -37,13 +46,15 @@ func NewDeleteHandler(folder storage.Folder, permanentBackups, permanentWals map
 		return nil, err
 	}
 
-	deleteHandler := internal.NewDeleteHandler(
-		folder,
-		postgresBackups,
-		lessFunc,
-		internal.IsPermanentFunc(
-			makePermanentFunc(permanentBackups, permanentWals)),
-	)
+	deleteHandler :=
+		&DeleteHandler{
+			*internal.NewDeleteHandler(
+				folder,
+				postgresBackups,
+				lessFunc,
+				internal.IsPermanentFunc(
+					makePermanentFunc(permanentBackups, permanentWals))),
+		}
 
 	return deleteHandler, nil
 }
@@ -192,3 +203,75 @@ func getIncrementInfo(folder storage.Folder, object storage.Object) (string, str
 
 	return *sentinel.IncrementFullName, *sentinel.IncrementFrom, false, nil
 }
+
+// HandleDeleteGarbage delete outdated WAL archives and leftover backup files
+func (dh *DeleteHandler) HandleDeleteGarbage(args []string, folder storage.Folder, confirm bool) error {
+	predicate := ExtractDeleteGarbagePredicate(args)
+	oldestBackup, err := findOldestNonPermanentBackup(folder.GetSubFolder(utility.BaseBackupPath))
+	if err != nil {
+		if _, ok := err.(internal.NoBackupsFoundError); ok {
+			tracelog.InfoLogger.Println("Couldn't find any non-permanent backups in storage. Not doing anything.")
+			return nil
+		}
+		return err
+	}
+
+	target, err := dh.FindTargetByName(oldestBackup.BackupName)
+	if err != nil {
+		return err
+	}
+
+	return dh.DeleteBeforeTargetWhere(target, confirm, predicate)
+}
+
+// ExtractDeleteGarbagePredicate extracts delete modifier the "delete garbage" command
+func ExtractDeleteGarbagePredicate(args []string) func(storage.Object) bool {
+	switch {
+	case len(args) == 2 && args[1] == DeleteGarbageArchivesModifier:
+		return storagePrefixFilter(utility.WalPath)
+	case len(args) == 2 && args[1] == DeleteGarbageBackupsModifier:
+		return storagePrefixFilter(utility.BaseBackupPath)
+	default:
+		return func(storage.Object) bool { return true }
+	}
+}
+
+func storagePrefixFilter(prefix string) func(storage.Object) bool {
+	return func(object storage.Object) bool {
+		objectName := object.GetName()
+		return len(objectName) >= len(prefix) && objectName[:len(prefix)] == prefix
+	}
+}
+
+// findOldestNonPermanentBackup finds oldest non-permanent backup available in storage.
+func findOldestNonPermanentBackup(
+	folder storage.Folder,
+) (*BackupDetail, error) {
+	backups, err := internal.GetBackups(folder)
+	if err != nil {
+		// this also includes the zero backups case
+		return nil, err
+	}
+
+	backupDetails, err := GetBackupsDetails(folder, backups)
+	if err != nil {
+		return nil, err
+	}
+
+	SortBackupDetails(backupDetails)
+
+	for i := range backupDetails {
+		currBackup := &backupDetails[i]
+
+		if currBackup.IsPermanent {
+			tracelog.InfoLogger.Printf(
+				"Backup %s is permanent, it is not eligible to be selected "+
+					"as the oldest backup for delete garbage.\n", currBackup.BackupName)
+			continue
+		}
+		tracelog.InfoLogger.Printf("Found earliest non-permanent backup: %s\n", currBackup.BackupName)
+		return currBackup, nil
+	}
+
+	return nil, internal.NewNoBackupsFoundError()
+}