Fix of deleting WAL-E backups

cmd/mysql/delete.go

@@ -115,7 +115,8 @@ func NewMySQLDeleteHandler() (*DeleteHandler, error) {
 	backupObjects, err := internal.FindBackupObjects(folder)
 	tracelog.ErrorLogger.FatalOnError(err)
 
-	permanentBackups := internal.FindPermanentBackups(folder, mysql.NewGenericMetaFetcher())
+	permanentBackups := internal.GetPermanentBackups(folder.GetSubFolder(utility.BaseBackupPath),
+		mysql.NewGenericMetaFetcher())
 
 	return &DeleteHandler{
 		DeleteHandler: internal.NewDeleteHandler(folder, backupObjects, makeLessFunc(folder),

docker/pg_tests/scripts/tests/wale_compatibility_test.sh

@@ -44,6 +44,25 @@ PGHOST=/var/run/postgresql \
 WALE_FILE_PREFIX=file://localhost/tmp \
 wal-e backup-push ${PGDATA}
 
+pgbench -i -s 10 postgres
+pg_dumpall -f /tmp/dump1
+pgbench -c 2 -T 100000000 -S &
+sleep 1
+AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE \
+AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY \
+AWS_ENDPOINT=http://s3:9000 \
+AWS_S3_FORCE_PATH_STYLE=true \
+WALG_COMPRESSION_METHOD=brotli \
+WALG_DELTA_MAX_STEPS=6 \
+WALG_UPLOAD_CONCURRENCY=10 \
+WALG_DISK_RATE_LIMIT=41943040 \
+WALG_NETWORK_RATE_LIMIT=10485760 \
+PGSSLMODE=allow \
+PGDATABASE=postgres \
+PGHOST=/var/run/postgresql \
+WALE_FILE_PREFIX=file://localhost/tmp \
+wal-e backup-push ${PGDATA}
+
 pkill -9 postgres
 
 mkdir /tmp/conf_files
@@ -96,6 +115,11 @@ pg_dumpall -f /tmp/dump2
 
 diff /tmp/dump1 /tmp/dump2
 rm -rf /tmp/conf_files
+
+
+WALE_FILE_PREFIX=file://localhost/tmp \
+wal-g delete retain 1 --confirm
+
 /tmp/scripts/drop_pg.sh
 
 echo "WAL-E compatible backup-fetch success!!!!!!"

internal/backup_mark.go

@@ -198,8 +198,7 @@ func GetPermanentBackups(folder storage.Folder, metaFetcher GenericMetaFetcher)
 	for _, backupTime := range backupTimes {
 		meta, err := metaFetcher.Fetch(backupTime.BackupName, folder)
 		if err != nil {
-			tracelog.ErrorLogger.Printf("failed to fetch backup meta for backup %s with error %s, ignoring...",
-				backupTime.BackupName, err.Error())
+			FatalOnUnrecoverableMetadataError(backupTime, err)
 			continue
 		}
 		if meta.IsPermanent {

internal/databases/greenplum/delete_handler.go

@@ -29,7 +29,8 @@ func NewDeleteHandler(folder storage.Folder) (*DeleteHandler, error) {
 		return obj1.GetLastModified().Before(obj2.GetLastModified())
 	}
 
-	permanentBackups := internal.FindPermanentBackups(folder, NewGenericMetaFetcher())
+	permanentBackups := internal.GetPermanentBackups(folder.GetSubFolder(utility.BaseBackupPath),
+		NewGenericMetaFetcher())
 	isPermanentFunc := func(obj storage.Object) bool {
 		return internal.IsPermanent(obj.GetName(), permanentBackups, BackupNameLength)
 	}

internal/databases/postgres/backup.go

@@ -28,7 +28,7 @@ var UtilityFilePaths = map[string]bool{
 	TablespaceMapFilename: true,
 }
 
-var patternPgBackupName = fmt.Sprintf("base_%[1]s(_D_%[1]s)?", PatternTimelineAndLogSegNo)
+var patternPgBackupName = fmt.Sprintf("base_%[1]s(_D_%[1]s)?(_%[2]s)?", PatternTimelineAndLogSegNo, PatternLSN)
 var regexpPgBackupName = regexp.MustCompile(patternPgBackupName)
 
 // Backup contains information about a valid Postgres backup
@@ -176,7 +176,7 @@ func (backup *Backup) FetchMeta() (ExtendedMetadataDto, error) {
 	extendedMetadataDto := ExtendedMetadataDto{}
 	err := backup.FetchMetadata(&extendedMetadataDto)
 	if err != nil {
-		return ExtendedMetadataDto{}, errors.Wrap(err, "failed to unmarshal metadata")
+		return ExtendedMetadataDto{}, err
 	}
 
 	return extendedMetadataDto, nil
@@ -435,6 +435,6 @@ func GetLastWalFilename(backup Backup) (string, error) {
 	return endWalSegmentNo.getFilename(timelineID), nil
 }
 
-func FetchPgBackupName(object storage.Object) string {
+func DeduceBackupName(object storage.Object) string {
 	return regexpPgBackupName.FindString(object.GetName())
 }

internal/databases/postgres/delete.go

@@ -68,7 +68,7 @@ func newBackupObject(incrementBase, incrementFrom string,
 		baseBackupName:    incrementBase,
 		incrementFromName: incrementFrom,
 		creationTime:      creationTime,
-		BackupName:        FetchPgBackupName(object),
+		BackupName:        DeduceBackupName(object),
 	}
 }
 
@@ -129,13 +129,13 @@ func makePermanentFunc(permanentBackups, permanentWals map[string]bool) func(obj
 
 func makeLessFunc(startTimeByBackupName map[string]time.Time) func(storage.Object, storage.Object) bool {
 	return func(object1 storage.Object, object2 storage.Object) bool {
-		backupName1 := FetchPgBackupName(object1)
+		backupName1 := DeduceBackupName(object1)
 		if backupName1 == "" {
 			// we can't compare non-backup storage objects (probably WAL segments) by start time,
 			// so use the segment number comparator instead
 			return segmentNoLess(object1, object2)
 		}
-		backupName2 := FetchPgBackupName(object2)
+		backupName2 := DeduceBackupName(object2)
 		if backupName2 == "" {
 			return segmentNoLess(object1, object2)
 		}
@@ -193,7 +193,7 @@ func timelineAndSegmentNoLess(object1 storage.Object, object2 storage.Object) bo
 }
 
 func getIncrementInfo(folder storage.Folder, object storage.Object) (string, string, bool, error) {
-	backup := NewBackup(folder.GetSubFolder(utility.BaseBackupPath), FetchPgBackupName(object))
+	backup := NewBackup(folder.GetSubFolder(utility.BaseBackupPath), DeduceBackupName(object))
 	sentinel, err := backup.GetSentinel()
 	if err != nil {
 		return "", "", true, err

internal/databases/postgres/delete_util.go

@@ -22,8 +22,7 @@ func GetPermanentBackupsAndWals(folder storage.Folder) (map[string]bool, map[str
 		backup := NewBackup(folder.GetSubFolder(utility.BaseBackupPath), backupTime.BackupName)
 		meta, err := backup.FetchMeta()
 		if err != nil {
-			tracelog.ErrorLogger.Printf("failed to fetch backup meta for backup %s with error %s, ignoring...",
-				backupTime.BackupName, err.Error())
+			internal.FatalOnUnrecoverableMetadataError(backupTime, err)
 			continue
 		}
 		if meta.IsPermanent {

internal/databases/postgres/timeline.go

@@ -11,6 +11,7 @@ import (
 )
 
 const PatternTimelineAndLogSegNo = "[0-9A-F]{24}"
+const PatternLSN = "[0-9A-F]{8}"
 
 var regexpTimelineAndLogSegNo = regexp.MustCompile(PatternTimelineAndLogSegNo)
 

internal/delete_util.go

@@ -10,27 +10,14 @@ import (
 	"github.com/wal-g/wal-g/utility"
 )
 
-func FindPermanentBackups(folder storage.Folder, metaFetcher GenericMetaFetcher) map[string]bool {
-	tracelog.InfoLogger.Println("retrieving permanent objects")
-	backupTimes, err := GetBackups(folder.GetSubFolder(utility.BaseBackupPath))
-	if err != nil {
-		return map[string]bool{}
-	}
-
-	permanentBackups := map[string]bool{}
-	for _, backupTime := range backupTimes {
-		meta, err := metaFetcher.Fetch(
-			backupTime.BackupName, folder.GetSubFolder(utility.BaseBackupPath))
-		if err != nil {
-			tracelog.ErrorLogger.Printf("failed to fetch backup meta for backup %s with error %s, ignoring...",
-				backupTime.BackupName, err.Error())
-			continue
-		}
-		if meta.IsPermanent {
-			permanentBackups[backupTime.BackupName] = true
-		}
+func FatalOnUnrecoverableMetadataError(backupTime BackupTime, err error) {
+	if _, ok := err.(storage.ObjectNotFoundError); ok {
+		tracelog.InfoLogger.Printf("Backup %s lacks metadata to check if it's permanent, ignoring...",
+			backupTime.BackupName)
+	} else {
+		tracelog.ErrorLogger.Fatalf("failed to fetch backup meta for backup %s with error %s",
+			backupTime.BackupName, err.Error())
 	}
-	return permanentBackups
 }
 
 // IsPermanent is a generic function to determine if the storage object is permanent.