Add WALG_GP_AOSEG_SIZE_THRESHOLD setting

docker/gp_tests/scripts/configs/common_config.json

@@ -11,4 +11,5 @@
 "WALG_DELTA_MAX_STEPS": "0",
 "WALG_PGP_KEY_PATH": "/tmp/PGP_KEY",
 "WALG_GP_SEG_POLL_INTERVAL": "5s",
-"WALG_GP_SEG_UPD_INTERVAL": "1s"
+"WALG_GP_SEG_UPD_INTERVAL": "1s",
+"WALG_GP_AOSEG_SIZE_THRESHOLD": "0"

docker/gp_tests/scripts/tests/ao_storage_test.sh

@@ -19,7 +19,14 @@ wal-g --config=${TMP_CONFIG} delete everything FORCE --confirm
 
 # 1st backup (init tables heap, ao, co)
 insert_data
-run_backup_logged ${TMP_CONFIG} ${PGDATA}
+WALG_GP_AOSEG_SIZE_THRESHOLD=1048576 wal-g --config=${TMP_CONFIG} backup-push ${PGDATA}
+
+wal-g st ls -r --config=${TMP_CONFIG}
+# AO/AOCS storage should be empty (all objects are below the threshold)
+if (wal-g st ls -r --config=${TMP_CONFIG} | grep -q "_aoseg") then
+  echo "Error: AO/AOCS shared storage is not empty"
+  exit 1
+fi
 
 # 2nd backup (populate the co table)
 psql -p 6000 -d test -c "INSERT INTO co select i, i FROM generate_series(1,10)i;"

docs/Greenplum.md

@@ -127,4 +127,8 @@ wal-g backup-fetch LATEST --in-place --config=/path/to/config.yaml
 ```
 
 #### Delete concurrency
-During the delete execution, WAL-G can process segments in parallel mode. To control, how many segments will be processed simultaneously, use the `WALG_GP_DELETE_CONCURRENCY` setting. The default value is `1`. 
+During the delete execution, WAL-G can process segments in parallel mode. To control, how many segments will be processed simultaneously, use the `WALG_GP_DELETE_CONCURRENCY` setting. The default value is `1`. 
+
+
+#### AO/AOCS size threshold
+To control the minimal size of the AO/AOCS segment file to be uploaded into the shared storage, use the `WALG_GP_AOSEG_SIZE_THRESHOLD`. The higher this value, the bigger the size of a single backup and the smaller the size of the shared AO/AOCS storage folder. Default value is `1048576 (1MB)`.

internal/config.go

@@ -121,6 +121,7 @@ const (
 	GPSegmentsUpdInterval  = "WALG_GP_SEG_UPD_INTERVAL"
 	GPSegmentStatesDir     = "WALG_GP_SEG_STATES_DIR"
 	GPDeleteConcurrency    = "WALG_GP_DELETE_CONCURRENCY"
+	GPAoSegSizeThreshold   = "WALG_GP_AOSEG_SIZE_THRESHOLD"
 
 	GoMaxProcs = "GOMAXPROCS"
 
@@ -209,6 +210,7 @@ var (
 		GPSegmentsPollRetries:  "5",
 		GPSegmentStatesDir:     "/tmp",
 		GPDeleteConcurrency:    "1",
+		GPAoSegSizeThreshold:   "1048576", // (1 << 20)
 	}
 
 	AllowedSettings map[string]bool
@@ -410,6 +412,7 @@ var (
 		GPSegmentsUpdInterval:  true,
 		GPSegmentStatesDir:     true,
 		GPDeleteConcurrency:    true,
+		GPAoSegSizeThreshold:   true,
 	}
 
 	RequiredSettings       = make(map[string]bool)

internal/databases/postgres/greenplum_tar_ball_composer.go

@@ -8,6 +8,8 @@ import (
 	"path"
 	"sync"
 
+	"github.com/spf13/viper"
+
 	"github.com/wal-g/wal-g/internal/walparser"
 
 	"github.com/wal-g/tracelog"
@@ -84,7 +86,8 @@ type GpTarBallComposer struct {
 
 	uploader *internal.Uploader
 	// Separate uploader for AO/AOCS relfiles with disabled file size tracking
-	aoSegUploader *internal.Uploader
+	aoSegUploader      *internal.Uploader
+	aoSegSizeThreshold int64
 
 	files            internal.BundleFiles
 	tarFileSets      internal.TarFileSets
@@ -106,19 +109,20 @@ func NewGpTarBallComposer(
 	aoSegUploader.DisableSizeTracking()
 
 	composer := &GpTarBallComposer{
-		backupName:    backupName,
-		tarBallQueue:  tarBallQueue,
-		tarFilePacker: packer,
-		crypter:       crypter,
-		relStorageMap: relStorageMap,
-		files:         bundleFiles,
-		aoFiles:       aoFiles,
-		baseAoFiles:   baseAoFiles,
-		uploader:      uploader.Clone(),
-		aoSegUploader: aoSegUploader,
-		tarFileSets:   tarFileSets,
-		errorGroup:    errorGroup,
-		ctx:           ctx,
+		backupName:         backupName,
+		tarBallQueue:       tarBallQueue,
+		tarFilePacker:      packer,
+		crypter:            crypter,
+		relStorageMap:      relStorageMap,
+		files:              bundleFiles,
+		aoFiles:            aoFiles,
+		baseAoFiles:        baseAoFiles,
+		uploader:           uploader.Clone(),
+		aoSegUploader:      aoSegUploader,
+		aoSegSizeThreshold: viper.GetInt64(internal.GPAoSegSizeThreshold),
+		tarFileSets:        tarFileSets,
+		errorGroup:         errorGroup,
+		ctx:                ctx,
 	}
 
 	maxUploadDiskConcurrency, err := internal.GetMaxUploadDiskConcurrency()
@@ -199,7 +203,8 @@ func (c *GpTarBallComposer) addFileWorker(tasks <-chan *internal.ComposeFileInfo
 
 func (c *GpTarBallComposer) addFile(cfi *internal.ComposeFileInfo) error {
 	// WAL-G uploads AO/AOCS relfiles to different location
-	if isAo, meta, location := c.relStorageMap.getAOStorageMetadata(cfi.Path); isAo {
+	isAo, meta, location := c.relStorageMap.getAOStorageMetadata(cfi.Path)
+	if isAo && cfi.FileInfo.Size() >= c.aoSegSizeThreshold {
 		return c.addAOFile(cfi, meta, location)
 	}