Minor fixes

cmd/pg/backup_push.go

@@ -73,6 +73,7 @@ var (
 			}
 			withoutFilesMetadata = withoutFilesMetadata || viper.GetBool(internal.WithoutFilesMetadataSetting)
 			if withoutFilesMetadata {
+				// files metadata tracking is required for delta backups and copy/rating composers
 				if useRatingComposer || useCopyComposer {
 					tracelog.ErrorLogger.Fatalf(
 						"%s option cannot be used with %s, %s options",
@@ -83,6 +84,8 @@ var (
 						"%s option cannot be used with %s, %s, %s options",
 						withoutFilesMetadataFlag, deltaFromNameFlag, deltaFromUserDataFlag, addUserDataFlag)
 				}
+				tracelog.InfoLogger.Print("Files metadata tracking is disabled")
+				fullBackup = true
 			}
 
 			deltaBaseSelector, err := createDeltaBaseSelector(cmd, deltaFromName, deltaFromUserData)
@@ -132,7 +135,6 @@ func createDeltaBaseSelector(cmd *cobra.Command,
 		return internal.NewUserDataBackupSelector(targetUserData, postgres.NewGenericMetaFetcher())
 
 	default:
-		tracelog.InfoLogger.Println("Selecting the latest backup as the base for the current delta backup...")
 		return internal.NewLatestBackupSelector(), nil
 	}
 }

internal/databases/postgres/backup.go

@@ -125,6 +125,14 @@ func (backup *Backup) GetSentinelAndFilesMetadata() (BackupSentinelDto, FilesMet
 	}
 
 	var filesMetadata FilesMetadataDto
+
+	// skip the files metadata fetch if backup was taken without it
+	if sentinel.FilesMetadataDisabled {
+		tracelog.InfoLogger.Printf("Files metadata tracking was disabled, skipping the download of %s", FilesMetadataName)
+		backup.FilesMetadataDto = &filesMetadata
+		return sentinel, filesMetadata, nil
+	}
+
 	err = backup.FetchDto(&filesMetadata, getFilesMetadataPath(backup.Name))
 	if err != nil {
 		// double-check that this is not V2 backup

internal/databases/postgres/backup_fetch_handler.go

@@ -62,9 +62,8 @@ func chooseTablespaceSpecification(sentinelDtoSpec, spec *TablespaceSpec) *Table
 
 // TODO : unit tests
 // deltaFetchRecursion function composes Backup object and recursively searches for necessary base backup
-func deltaFetchRecursionOld(backupName string, folder storage.Folder, dbDataDirectory string,
+func deltaFetchRecursionOld(backup Backup, folder storage.Folder, dbDataDirectory string,
 	tablespaceSpec *TablespaceSpec, filesToUnwrap map[string]bool) error {
-	backup := NewBackup(folder.GetSubFolder(utility.BaseBackupPath), backupName)
 	sentinelDto, filesMetaDto, err := backup.GetSentinelAndFilesMetadata()
 	if err != nil {
 		return err
@@ -78,7 +77,8 @@ func deltaFetchRecursionOld(backupName string, folder storage.Folder, dbDataDire
 		if err != nil {
 			return err
 		}
-		err = deltaFetchRecursionOld(*sentinelDto.IncrementFrom, folder, dbDataDirectory, tablespaceSpec, baseFilesToUnwrap)
+		incrementFrom := NewBackup(folder.GetSubFolder(utility.BaseBackupPath), *sentinelDto.IncrementFrom)
+		err = deltaFetchRecursionOld(incrementFrom, folder, dbDataDirectory, tablespaceSpec, baseFilesToUnwrap)
 		if err != nil {
 			return err
 		}
@@ -102,7 +102,7 @@ func GetPgFetcherOld(dbDataDirectory, fileMask, restoreSpecPath string) func(roo
 			errMessege := fmt.Sprintf("Invalid restore specification path %s\n", restoreSpecPath)
 			tracelog.ErrorLogger.FatalfOnError(errMessege, err)
 		}
-		err = deltaFetchRecursionOld(backup.Name, rootFolder, utility.ResolveSymlink(dbDataDirectory), spec, filesToUnwrap)
+		err = deltaFetchRecursionOld(pgBackup, rootFolder, utility.ResolveSymlink(dbDataDirectory), spec, filesToUnwrap)
 		tracelog.ErrorLogger.FatalfOnError("Failed to fetch backup: %v\n", err)
 	}
 }

internal/databases/postgres/backup_push_handler.go

@@ -218,7 +218,7 @@ func (bh *BackupHandler) setupDTO(tarFileSets TarFileSets) (sentinelDto BackupSe
 	}
 	sentinelDto = NewBackupSentinelDto(bh, tablespaceSpec)
 	filesMeta.setFiles(bh.workers.bundle.GetFiles())
-	filesMeta.TarFileSets = tarFileSets
+	filesMeta.TarFileSets = tarFileSets.Get()
 	return sentinelDto, filesMeta
 }
 
@@ -355,7 +355,7 @@ func (bh *BackupHandler) createAndPushRemoteBackup() {
 	bh.curBackupInfo.compressedSize, err = bh.workers.uploader.UploadedDataSize()
 	tracelog.ErrorLogger.FatalOnError(err)
 	sentinelDto := NewBackupSentinelDto(bh, baseBackup.GetTablespaceSpec())
-	filesMetadataDto := NewFilesMetadataDto(baseBackup.Files, TarFileSets{})
+	filesMetadataDto := NewFilesMetadataDto(baseBackup.Files, tarFileSets)
 	bh.curBackupInfo.name = baseBackup.BackupName()
 	tracelog.InfoLogger.Println("Uploading metadata")
 	bh.uploadMetadata(sentinelDto, filesMetadataDto)
@@ -580,6 +580,11 @@ func (bh *BackupHandler) uploadExtendedMetadata(meta ExtendedMetadataDto) (err e
 }
 
 func (bh *BackupHandler) uploadFilesMetadata(filesMetaDto FilesMetadataDto) (err error) {
+	if bh.arguments.withoutFilesMetadata {
+		tracelog.InfoLogger.Printf("Files metadata tracking is disabled, will not upload the %s", FilesMetadataName)
+		return nil
+	}
+
 	dtoBody, err := json.Marshal(filesMetaDto)
 	if err != nil {
 		return err

internal/databases/postgres/backup_sentinel_dto.go

@@ -30,6 +30,8 @@ type BackupSentinelDto struct {
 	TablespaceSpec   *TablespaceSpec `json:"Spec"`
 
 	UserData interface{} `json:"UserData,omitempty"`
+
+	FilesMetadataDisabled bool `json:"FilesMetadataDisabled,omitempty"`
 }
 
 func NewBackupSentinelDto(bh *BackupHandler, tbsSpec *TablespaceSpec) BackupSentinelDto {
@@ -54,6 +56,7 @@ func NewBackupSentinelDto(bh *BackupHandler, tbsSpec *TablespaceSpec) BackupSent
 	sentinel.SystemIdentifier = bh.pgInfo.systemIdentifier
 	sentinel.UncompressedSize = bh.curBackupInfo.uncompressedSize
 	sentinel.CompressedSize = bh.curBackupInfo.compressedSize
+	sentinel.FilesMetadataDisabled = bh.arguments.withoutFilesMetadata
 	return sentinel
 }
 
@@ -117,11 +120,11 @@ func (dto *BackupSentinelDto) IsIncremental() (isIncremental bool) {
 // It can be pretty large on some databases, sometimes more than 1GB
 type FilesMetadataDto struct {
 	Files       internal.BackupFileList `json:"Files,omitempty"`
-	TarFileSets TarFileSets             `json:"TarFileSets,omitempty"`
+	TarFileSets map[string][]string     `json:"TarFileSets,omitempty"`
 }
 
 func NewFilesMetadataDto(files internal.BackupFileList, tarFileSets TarFileSets) FilesMetadataDto {
-	return FilesMetadataDto{TarFileSets: tarFileSets, Files: files}
+	return FilesMetadataDto{TarFileSets: tarFileSets.Get(), Files: files}
 }
 
 func (dto *FilesMetadataDto) setFiles(p *sync.Map) {

internal/databases/postgres/copy_tar_ball_composer.go

@@ -176,7 +176,7 @@ func (c *CopyTarBallComposer) copyTar(tarName string) error {
 	if err != nil {
 		return err
 	}
-	for _, fileName := range c.prevTarFileSets.GetFiles()[tarName] {
+	for _, fileName := range c.prevTarFileSets.Get()[tarName] {
 		if file, exists := c.fileInfo[fileName]; exists {
 			file.status = processed
 			c.tarFileSets.AddFile(newTarName, fileName)
@@ -199,7 +199,7 @@ func (c *CopyTarBallComposer) getTarBall() internal.TarBall {
 func (c *CopyTarBallComposer) copyUnchangedTars() error {
 	for tarName, cnt := range c.tarUnchangedFilesCount {
 		if cnt != 0 {
-			for _, fileName := range c.prevTarFileSets.GetFiles()[tarName] {
+			for _, fileName := range c.prevTarFileSets.Get()[tarName] {
 				if _, exists := c.fileInfo[fileName]; exists {
 					c.fileInfo[fileName].status = doNotCopy
 				} else if _, exists := c.headerInfos[fileName]; exists {

internal/databases/postgres/tar_file_sets.go

@@ -3,29 +3,25 @@ package postgres
 type TarFileSets interface {
 	AddFile(name string, file string)
 	AddFiles(name string, files []string)
-	GetFiles() map[string][]string
+	Get() map[string][]string
 }
 
-type RegularTarFileSets struct {
-	files map[string][]string
-}
+type RegularTarFileSets map[string][]string
 
 func NewRegularTarFileSets() *RegularTarFileSets {
-	return &RegularTarFileSets{
-		files: make(map[string][]string),
-	}
+	return &RegularTarFileSets{}
 }
 
 func (tarFileSets *RegularTarFileSets) AddFile(name string, file string) {
-	tarFileSets.files[name] = append(tarFileSets.files[name], file)
+	(*tarFileSets)[name] = append((*tarFileSets)[name], file)
 }
 
 func (tarFileSets *RegularTarFileSets) AddFiles(name string, files []string) {
-	tarFileSets.files[name] = append(tarFileSets.files[name], files...)
+	(*tarFileSets)[name] = append((*tarFileSets)[name], files...)
 }
 
-func (tarFileSets *RegularTarFileSets) GetFiles() map[string][]string {
-	return tarFileSets.files
+func (tarFileSets *RegularTarFileSets) Get() map[string][]string {
+	return *tarFileSets
 }
 
 type NopTarFileSets struct {
@@ -41,6 +37,6 @@ func (tarFileSets *NopTarFileSets) AddFile(name string, file string) {
 func (tarFileSets *NopTarFileSets) AddFiles(name string, files []string) {
 }
 
-func (tarFileSets *NopTarFileSets) GetFiles() map[string][]string {
+func (tarFileSets *NopTarFileSets) Get() map[string][]string {
 	return make(map[string][]string)
 }

internal/databases/postgres/walk_test.go

@@ -378,16 +378,20 @@ func testWalk(t *testing.T, composer postgres.TarBallComposerType, withoutFilesM
 		t.Log(err)
 	}
 
-	backupFileList := postgres.MakeBackupFileList(bundle.GetFiles())
+	backupFileListEmpty := true
+	bundle.GetFiles().Range(func(key, value interface{}) bool {
+		backupFileListEmpty = false
+		return false
+	})
 
 	if withoutFilesMetadata {
 		// Test tarFileSets is not tracked
-		assert.True(t, len(tarFileSets.GetFiles()) == 0)
+		assert.True(t, len(tarFileSets.Get()) == 0)
 		// Test BackupFileList is not tracked
-		assert.True(t, len(backupFileList) == 0)
+		assert.True(t, backupFileListEmpty)
 	} else {
-		assert.True(t, len(tarFileSets.GetFiles()) > 0)
-		assert.True(t, len(backupFileList) > 0)
+		assert.True(t, len(tarFileSets.Get()) > 0)
+		assert.False(t, backupFileListEmpty)
 	}
 
 	err = bundle.FinishQueue()