From 784f457f66c7a67a5d1ddcd6ea9c22bde13f2735 Mon Sep 17 00:00:00 2001 From: Remco Beckers Date: Tue, 14 Apr 2026 12:49:38 +0200 Subject: [PATCH] STAC-23457 Dropping multipart archive for stackgraph The tooling now uses native s3 multipart upload instead --- cmd/elasticsearch/list_test.go | 2 - cmd/settings/list.go | 6 +- cmd/stackgraph/list.go | 4 +- cmd/stackgraph/restore.go | 5 +- internal/clients/s3/filter.go | 112 ++------- internal/clients/s3/filter_test.go | 212 ++---------------- internal/foundation/config/config.go | 7 +- internal/foundation/config/config_test.go | 10 +- .../config/testdata/validConfigMapConfig.yaml | 3 - .../config/testdata/validConfigMapOnly.yaml | 1 - .../testdata/validStorageConfigMapConfig.yaml | 2 - .../validStorageConfigMapNoStackpacks.yaml | 1 - .../testdata/validStorageConfigMapOnly.yaml | 1 - .../scripts/restore-stackgraph-backup.sh | 19 +- 14 files changed, 46 insertions(+), 339 deletions(-) diff --git a/cmd/elasticsearch/list_test.go b/cmd/elasticsearch/list_test.go index d616be1..3a7b45e 100644 --- a/cmd/elasticsearch/list_test.go +++ b/cmd/elasticsearch/list_test.go @@ -64,7 +64,6 @@ minio: secretKey: minioadmin stackgraph: bucket: stackgraph-bucket - multipartArchive: true restore: scaleDownLabelSelector: "app=stackgraph" loggingConfigConfigMap: logging-config @@ -148,7 +147,6 @@ storage: secretKey: storageadmin stackgraph: bucket: stackgraph-bucket - multipartArchive: true restore: scaleDownLabelSelector: "app=stackgraph" loggingConfigConfigMap: logging-config diff --git a/cmd/settings/list.go b/cmd/settings/list.go index da8c341..a82f369 100644 --- a/cmd/settings/list.go +++ b/cmd/settings/list.go @@ -27,7 +27,6 @@ import ( ) const ( - isMultiPartArchive = false expectedListJobPodCount = 1 expectedListJobContainerCount = 1 backupFileNameRegex = `^sts-backup-.*\.sty$` @@ -183,8 +182,7 @@ func getBackupListFromS3(appCtx *app.Context) ([]BackupFileInfo, error) { return nil, fmt.Errorf("failed to list S3 objects: %w", err) } - // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) + filteredObjects := s3client.FilterBackupObjects(result.Contents) // Filter to only include direct children of the prefix that match the backup filename pattern, // and strip the prefix from the key @@ -238,7 +236,7 @@ func getBackupListFromLocalBucket(appCtx *app.Context) ([]BackupFileInfo, error) return nil, fmt.Errorf("failed to list objects in local bucket: %w", err) } - filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, isMultiPartArchive) + filteredObjects := s3client.FilterBackupObjects(result.Contents) filteredObjects, err = s3client.FilterByPrefixAndRegex(filteredObjects, "", backupFileNameRegex) if err != nil { diff --git a/cmd/stackgraph/list.go b/cmd/stackgraph/list.go index c9b86b7..50c0ed7 100644 --- a/cmd/stackgraph/list.go +++ b/cmd/stackgraph/list.go @@ -52,7 +52,6 @@ func runList(appCtx *app.Context) error { // List objects in bucket bucket := appCtx.Config.Stackgraph.Bucket prefix := appCtx.Config.Stackgraph.S3Prefix - multipartArchive := appCtx.Config.Stackgraph.MultipartArchive appCtx.Logger.Infof("Listing Stackgraph backups in bucket '%s'...", bucket) @@ -66,8 +65,7 @@ func runList(appCtx *app.Context) error { return fmt.Errorf("failed to list S3 objects: %w", err) } - // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) + filteredObjects := s3client.FilterBackupObjects(result.Contents) // Filter to only include direct children of the prefix that match the backup filename pattern, // and strip the prefix from the key diff --git a/cmd/stackgraph/restore.go b/cmd/stackgraph/restore.go index 3226831..8391ceb 100644 --- a/cmd/stackgraph/restore.go +++ b/cmd/stackgraph/restore.go @@ -169,7 +169,6 @@ func getLatestBackup(k8sClient *k8s.Client, namespace string, config *config.Con // List objects in bucket bucket := config.Stackgraph.Bucket prefix := config.Stackgraph.S3Prefix - multipartArchive := config.Stackgraph.MultipartArchive input := &s3.ListObjectsV2Input{ Bucket: aws.String(bucket), @@ -181,8 +180,7 @@ func getLatestBackup(k8sClient *k8s.Client, namespace string, config *config.Con return "", fmt.Errorf("failed to list S3 objects: %w", err) } - // Filter objects based on whether the archive is split or not - filteredObjects := s3client.FilterMultipartBackupObjects(result.Contents, multipartArchive) + filteredObjects := s3client.FilterBackupObjects(result.Contents) // Filter to only include direct children of the prefix that match the backup filename pattern, // and strip the prefix from the key @@ -278,7 +276,6 @@ func buildRestoreEnvVars(backupFile string, config *config.Config) []corev1.EnvV {Name: "FORCE_DELETE", Value: purgeStackgraphDataFlag}, {Name: "BACKUP_STACKGRAPH_BUCKET_NAME", Value: config.Stackgraph.Bucket}, {Name: "BACKUP_STACKGRAPH_S3_PREFIX", Value: config.Stackgraph.S3Prefix}, - {Name: "BACKUP_STACKGRAPH_MULTIPART_ARCHIVE", Value: strconv.FormatBool(config.Stackgraph.MultipartArchive)}, {Name: "MINIO_ENDPOINT", Value: fmt.Sprintf("%s:%d", storageService.Name, storageService.Port)}, {Name: "STACKSTATE_BASE_URL", Value: config.GetBaseURL()}, {Name: "RECEIVER_BASE_URL", Value: config.GetReceiverBaseURL()}, diff --git a/internal/clients/s3/filter.go b/internal/clients/s3/filter.go index f44ceb7..34e9e1e 100644 --- a/internal/clients/s3/filter.go +++ b/internal/clients/s3/filter.go @@ -10,10 +10,6 @@ import ( s3types "github.com/aws/aws-sdk-go-v2/service/s3/types" ) -const ( - multipartArchiveSuffixLength = 2 -) - // Object represents a simplified S3 object with key metadata type Object struct { Key string @@ -21,37 +17,15 @@ type Object struct { Size int64 } -// FilterMultipartBackupObjects filters S3 objects based on whether the archive is split or not -// If it is not multipartArchive, it filters out multipart archives (files ending with .digits) -// Otherwise, it groups multipart archives by base name and sums their sizes -func FilterMultipartBackupObjects(objects []s3types.Object, multipartArchive bool) []Object { - if !multipartArchive { - return filterNonMultipart(objects) - } - return aggregateMultipart(objects) -} - -// filterNonMultipart filters out multipart archives (files ending with .digits) -func filterNonMultipart(objects []s3types.Object) []Object { +// FilterBackupObjects filters out backup part files ending with .digits. +func FilterBackupObjects(objects []s3types.Object) []Object { var filteredObjects []Object for _, obj := range objects { key := aws.ToString(obj.Key) - // Skip if it ends with .digits (multipart archive) - if strings.Contains(key, ".") { - parts := strings.Split(key, ".") - lastPart := parts[len(parts)-1] - isDigits := true - for _, c := range lastPart { - if c < '0' || c > '9' { - isDigits = false - break - } - } - if isDigits && len(lastPart) > 0 { - continue - } + if hasNumericFileSuffix(key) { + continue } filteredObjects = append(filteredObjects, Object{ @@ -64,83 +38,25 @@ func filterNonMultipart(objects []s3types.Object) []Object { return filteredObjects } -// aggregateMultipart groups multipart archives by base name and sums their sizes -func aggregateMultipart(objects []s3types.Object) []Object { - // Map to group objects by base name - archiveMap := make(map[string]*Object) - - for _, obj := range objects { - key := aws.ToString(obj.Key) - - // Check if this is a multipart file (ends with .NN where NN are digits) - baseName, isMultipart := getBaseName(key) - if !isMultipart { - // Not a multipart file, include as-is - archiveMap[key] = &Object{ - Key: key, - LastModified: aws.ToTime(obj.LastModified), - Size: aws.ToInt64(obj.Size), - } - continue - } - - // Group multipart files by base name - if existing, exists := archiveMap[baseName]; exists { - // Add size to existing entry - existing.Size += aws.ToInt64(obj.Size) - // Keep the most recent LastModified time - if aws.ToTime(obj.LastModified).After(existing.LastModified) { - existing.LastModified = aws.ToTime(obj.LastModified) - } - } else { - // Create new entry - archiveMap[baseName] = &Object{ - Key: baseName, - LastModified: aws.ToTime(obj.LastModified), - Size: aws.ToInt64(obj.Size), - } - } - } - - // Convert map to slice - var filteredObjects []Object - for _, obj := range archiveMap { - filteredObjects = append(filteredObjects, *obj) - } - - return filteredObjects -} - -// getBaseName extracts the base name from a multipart archive filename -// Returns (baseName, isMultipart) -// Example: "backup.graph.00" -> ("backup.graph", true) -// -// "backup.graph" -> ("backup.graph", false) -func getBaseName(key string) (string, bool) { +func hasNumericFileSuffix(key string) bool { if !strings.Contains(key, ".") { - return key, false + return false } parts := strings.Split(key, ".") lastPart := parts[len(parts)-1] - // Check if last part is all digits (2 digits for part numbers like .00, .01, etc.) - if len(lastPart) == multipartArchiveSuffixLength { - isDigits := true - for _, c := range lastPart { - if c < '0' || c > '9' { - isDigits = false - break - } - } - if isDigits { - // Remove the .NN suffix to get base name - baseName := strings.Join(parts[:len(parts)-1], ".") - return baseName, true + if len(lastPart) == 0 { + return false + } + + for _, c := range lastPart { + if c < '0' || c > '9' { + return false } } - return key, false + return true } // FilterByPrefixAndRegex filters objects to only include direct children of the given prefix diff --git a/internal/clients/s3/filter_test.go b/internal/clients/s3/filter_test.go index 7880c3a..03eb3b6 100644 --- a/internal/clients/s3/filter_test.go +++ b/internal/clients/s3/filter_test.go @@ -9,8 +9,7 @@ import ( "github.com/stretchr/testify/assert" ) -// TestFilterBackupObjects_SingleFileMode tests filtering when it is not multipartArchive -func TestFilterBackupObjects_SingleFileMode(t *testing.T) { +func TestFilterBackupObjects(t *testing.T) { tests := []struct { name string objects []s3types.Object @@ -79,7 +78,7 @@ func TestFilterBackupObjects_SingleFileMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterMultipartBackupObjects(tt.objects, false) + result := FilterBackupObjects(tt.objects) assert.Equal(t, tt.expectedCount, len(result)) @@ -93,89 +92,8 @@ func TestFilterBackupObjects_SingleFileMode(t *testing.T) { } } -// TestFilterBackupObjects_MultipartMode tests filtering when it is multipartArchive -func TestFilterBackupObjects_MultipartMode(t *testing.T) { - tests := []struct { - name string - objects []s3types.Object - multipartArchive bool - expectedCount int - expectedKeys []string - }{ - { - name: "groups multipart archives and sums their sizes", - objects: []s3types.Object{ - {Key: aws.String("backup-2024-01-01.00"), Size: aws.Int64(500000000)}, - {Key: aws.String("backup-2024-01-01.01"), Size: aws.Int64(500000000)}, - {Key: aws.String("backup-2024-01-01.02"), Size: aws.Int64(300000000)}, - {Key: aws.String("backup-2024-01-02.00"), Size: aws.Int64(500000000)}, - {Key: aws.String("backup-2024-01-02.01"), Size: aws.Int64(400000000)}, - }, - multipartArchive: true, - expectedCount: 2, - expectedKeys: []string{"backup-2024-01-01", "backup-2024-01-02"}, - }, - { - name: "includes both multipart and single files", - objects: []s3types.Object{ - {Key: aws.String("backup.tar.gz"), Size: aws.Int64(1000)}, - {Key: aws.String("backup-split.00"), Size: aws.Int64(500000000)}, - {Key: aws.String("backup-split.01"), Size: aws.Int64(500000000)}, - {Key: aws.String("backup-single"), Size: aws.Int64(1000000)}, - }, - multipartArchive: true, - expectedCount: 3, - expectedKeys: []string{"backup.tar.gz", "backup-split", "backup-single"}, - }, - { - name: "handles different split size values", - objects: []s3types.Object{ - {Key: aws.String("backup-1G.00"), Size: aws.Int64(1000000000)}, - {Key: aws.String("backup-1G.01"), Size: aws.Int64(500000000)}, - }, - multipartArchive: true, - expectedCount: 1, - expectedKeys: []string{"backup-1G"}, - }, - { - name: "handles empty object list", - objects: []s3types.Object{}, - multipartArchive: true, - expectedCount: 0, - expectedKeys: []string{}, - }, - { - name: "handles objects with .00 in middle of filename", - objects: []s3types.Object{ - {Key: aws.String("backup.00.tar"), Size: aws.Int64(1000)}, - {Key: aws.String("backup-final.00"), Size: aws.Int64(500000000)}, - }, - multipartArchive: true, - expectedCount: 2, - expectedKeys: []string{"backup.00.tar", "backup-final"}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) - - assert.Equal(t, tt.expectedCount, len(result)) - - resultKeys := make([]string, len(result)) - for i, obj := range result { - resultKeys[i] = obj.Key - } - - assert.ElementsMatch(t, tt.expectedKeys, resultKeys) - }) - } -} - -// TestFilterBackupObjects_ObjectMetadata tests that metadata is preserved correctly func TestFilterBackupObjects_ObjectMetadata(t *testing.T) { now := time.Now() - yesterday := now.Add(-24 * time.Hour) objects := []s3types.Object{ { @@ -183,112 +101,65 @@ func TestFilterBackupObjects_ObjectMetadata(t *testing.T) { Size: aws.Int64(1234567890), LastModified: aws.Time(now), }, - { - Key: aws.String("backup-2024-01-02.00"), - Size: aws.Int64(100000000), - LastModified: aws.Time(yesterday), - }, - { - Key: aws.String("backup-2024-01-02.01"), - Size: aws.Int64(50000000), - LastModified: aws.Time(yesterday.Add(1 * time.Minute)), // Slightly later - }, } - // Test single file mode - result := FilterMultipartBackupObjects(objects, false) + result := FilterBackupObjects(objects) assert.Equal(t, 1, len(result)) assert.Equal(t, "backup-2024-01-01.tar.gz", result[0].Key) assert.Equal(t, int64(1234567890), result[0].Size) assert.Equal(t, now.Unix(), result[0].LastModified.Unix()) - - // Test multipart mode - should group parts and sum sizes - result = FilterMultipartBackupObjects(objects, true) - assert.Equal(t, 2, len(result)) // tar.gz file + grouped multipart - - // Find the multipart archive result - var multipartResult *Object - var singleResult *Object - for i := range result { - switch result[i].Key { - case "backup-2024-01-02": - multipartResult = &result[i] - case "backup-2024-01-01.tar.gz": - singleResult = &result[i] - } - } - - assert.NotNil(t, multipartResult, "Should find grouped multipart archive") - assert.NotNil(t, singleResult, "Should find single file") - - // Verify multipart archive has summed size - assert.Equal(t, "backup-2024-01-02", multipartResult.Key) - assert.Equal(t, int64(150000000), multipartResult.Size) // 100M + 50M - assert.Equal(t, yesterday.Add(1*time.Minute).Unix(), multipartResult.LastModified.Unix()) // Most recent timestamp - - // Verify single file - assert.Equal(t, "backup-2024-01-01.tar.gz", singleResult.Key) - assert.Equal(t, int64(1234567890), singleResult.Size) } -// TestFilterBackupObjects_EdgeCases tests edge cases and boundary conditions func TestFilterBackupObjects_EdgeCases(t *testing.T) { tests := []struct { - name string - objects []s3types.Object - multipartArchive bool - expectedCount int + name string + objects []s3types.Object + expectedCount int }{ { name: "file ending with just a dot", objects: []s3types.Object{ {Key: aws.String("backup."), Size: aws.Int64(1000)}, }, - multipartArchive: false, - expectedCount: 1, // Empty extension, should be included + expectedCount: 1, // Empty extension, should be included }, { name: "file with mixed alphanumeric extension", objects: []s3types.Object{ {Key: aws.String("backup.123abc"), Size: aws.Int64(1000)}, }, - multipartArchive: false, - expectedCount: 1, // Contains non-digits, should be included + expectedCount: 1, // Contains non-digits, should be included }, { name: "very long numeric extension", objects: []s3types.Object{ {Key: aws.String("backup.00000000000000000001"), Size: aws.Int64(1000)}, }, - multipartArchive: false, - expectedCount: 0, // All digits, should be filtered + expectedCount: 0, // All digits, should be filtered }, { - name: "multipart with .00 but it is not multipartArchive", + name: "multipart part with .00 suffix", objects: []s3types.Object{ {Key: aws.String("backup.00"), Size: aws.Int64(1000)}, }, - multipartArchive: false, - expectedCount: 0, // Numeric extension in single file mode, should be filtered + expectedCount: 0, // Numeric extension should be filtered }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) + result := FilterBackupObjects(tt.objects) assert.Equal(t, tt.expectedCount, len(result)) }) } } -// TestFilterBackupObjects_RealWorldScenarios tests realistic backup scenarios func TestFilterBackupObjects_RealWorldScenarios(t *testing.T) { tests := []struct { - name string - scenario string - objects []s3types.Object - multipartArchive bool - expectedCount int + name string + scenario string + objects []s3types.Object + expectedCount int }{ { name: "stackgraph backups without splitting", @@ -298,25 +169,11 @@ func TestFilterBackupObjects_RealWorldScenarios(t *testing.T) { {Key: aws.String("stackgraph-backup-2024-01-02.tar.gz"), Size: aws.Int64(12000000)}, {Key: aws.String("stackgraph-backup-2024-01-03.tar.gz"), Size: aws.Int64(11000000)}, }, - multipartArchive: false, - expectedCount: 3, - }, - { - name: "stackgraph backups with 500M splitting", - scenario: "Split archives at 500M", - objects: []s3types.Object{ - {Key: aws.String("stackgraph-backup-2024-01-01.00"), Size: aws.Int64(500000000)}, - {Key: aws.String("stackgraph-backup-2024-01-01.01"), Size: aws.Int64(500000000)}, - {Key: aws.String("stackgraph-backup-2024-01-01.02"), Size: aws.Int64(200000000)}, - {Key: aws.String("stackgraph-backup-2024-01-02.00"), Size: aws.Int64(500000000)}, - {Key: aws.String("stackgraph-backup-2024-01-02.01"), Size: aws.Int64(300000000)}, - }, - multipartArchive: true, - expectedCount: 2, // Two grouped multipart archives + expectedCount: 3, }, { name: "mixed backup types in same bucket", - scenario: "Single and split backups mixed", + scenario: "Single archives remain while multipart parts are ignored", objects: []s3types.Object{ {Key: aws.String("backup-old.tar.gz"), Size: aws.Int64(1000000)}, {Key: aws.String("backup-new-split.00"), Size: aws.Int64(500000000)}, @@ -324,47 +181,18 @@ func TestFilterBackupObjects_RealWorldScenarios(t *testing.T) { {Key: aws.String("backup-old-split.00"), Size: aws.Int64(1000)}, {Key: aws.String("backup-old-split.01"), Size: aws.Int64(1000)}, }, - multipartArchive: true, - expectedCount: 3, // One single file + two grouped multipart archives + expectedCount: 1, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - result := FilterMultipartBackupObjects(tt.objects, tt.multipartArchive) + result := FilterBackupObjects(tt.objects) assert.Equal(t, tt.expectedCount, len(result), "Scenario: %s", tt.scenario) }) } } -// TestFilterBackupObjects_SizeSummation tests that sizes are correctly summed for multipart archives -func TestFilterBackupObjects_SizeSummation(t *testing.T) { - objects := []s3types.Object{ - {Key: aws.String("sts-backup-20251028-1546.graph.00"), Size: aws.Int64(104857600)}, - {Key: aws.String("sts-backup-20251028-1546.graph.01"), Size: aws.Int64(6885342)}, - {Key: aws.String("sts-backup-20251029-0300.graph.00"), Size: aws.Int64(104857600)}, - {Key: aws.String("sts-backup-20251029-0300.graph.01"), Size: aws.Int64(4348555)}, - {Key: aws.String("sts-backup-20251029-0924.graph.00"), Size: aws.Int64(104857600)}, - {Key: aws.String("sts-backup-20251029-0924.graph.01"), Size: aws.Int64(6567239)}, - } - - result := FilterMultipartBackupObjects(objects, true) - - // Should have 3 grouped archives - assert.Equal(t, 3, len(result)) - - // Create a map for easier lookup - sizeMap := make(map[string]int64) - for _, obj := range result { - sizeMap[obj.Key] = obj.Size - } - - // Verify sizes are correctly summed - assert.Equal(t, int64(111742942), sizeMap["sts-backup-20251028-1546.graph"]) // 104857600 + 6885342 - assert.Equal(t, int64(109206155), sizeMap["sts-backup-20251029-0300.graph"]) // 104857600 + 4348555 - assert.Equal(t, int64(111424839), sizeMap["sts-backup-20251029-0924.graph"]) // 104857600 + 6567239 -} - // TestFilterByPrefixAndRegex tests the combined filtering by prefix and regex pattern func TestFilterByPrefixAndRegex(t *testing.T) { //nolint:funlen // Table-driven test now := time.Now() diff --git a/internal/foundation/config/config.go b/internal/foundation/config/config.go index 07d8a90..ab9ca8e 100644 --- a/internal/foundation/config/config.go +++ b/internal/foundation/config/config.go @@ -176,10 +176,9 @@ type StorageConfig struct { // StackgraphConfig holds Stackgraph backup-specific configuration type StackgraphConfig struct { - Bucket string `yaml:"bucket" validate:"required"` - S3Prefix string `yaml:"s3Prefix"` - MultipartArchive bool `yaml:"multipartArchive" validate:"boolean"` - Restore StackgraphRestoreConfig `yaml:"restore" validate:"required"` + Bucket string `yaml:"bucket" validate:"required"` + S3Prefix string `yaml:"s3Prefix"` + Restore StackgraphRestoreConfig `yaml:"restore" validate:"required"` } type VictoriaMetricsConfig struct { diff --git a/internal/foundation/config/config_test.go b/internal/foundation/config/config_test.go index 07fecc9..2860159 100644 --- a/internal/foundation/config/config_test.go +++ b/internal/foundation/config/config_test.go @@ -654,9 +654,8 @@ func TestConfig_StructValidation(t *testing.T) { SecretKey: "minioadmin", }, Stackgraph: StackgraphConfig{ - Bucket: "stackgraph-bucket", - S3Prefix: "", - MultipartArchive: true, + Bucket: "stackgraph-bucket", + S3Prefix: "", Restore: StackgraphRestoreConfig{ ScaleDownLabelSelector: "app=stackgraph", LoggingConfigConfigMapName: "logging-config", @@ -801,9 +800,8 @@ func TestConfig_StructValidation(t *testing.T) { SecretKey: "storageadmin", }, Stackgraph: StackgraphConfig{ - Bucket: "stackgraph-bucket", - S3Prefix: "", - MultipartArchive: true, + Bucket: "stackgraph-bucket", + S3Prefix: "", Restore: StackgraphRestoreConfig{ ScaleDownLabelSelector: "app=stackgraph", LoggingConfigConfigMapName: "logging-config", diff --git a/internal/foundation/config/testdata/validConfigMapConfig.yaml b/internal/foundation/config/testdata/validConfigMapConfig.yaml index 4437c07..4308734 100644 --- a/internal/foundation/config/testdata/validConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validConfigMapConfig.yaml @@ -84,9 +84,6 @@ stackgraph: bucket: sts-stackgraph-backup # S3 prefix path for backups s3Prefix: "" - # S3 prefix path for stackpacks backups - # Archive split to multiple parts - multipartArchive: true # Restore configuration restore: # Label selector for deployments to scale down during restore diff --git a/internal/foundation/config/testdata/validConfigMapOnly.yaml b/internal/foundation/config/testdata/validConfigMapOnly.yaml index 17ded94..3003119 100644 --- a/internal/foundation/config/testdata/validConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validConfigMapOnly.yaml @@ -86,7 +86,6 @@ minio: stackgraph: bucket: sts-stackgraph-backup s3Prefix: "" - multipartArchive: true restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging diff --git a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml index 3affec5..dc2f72e 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapConfig.yaml @@ -86,8 +86,6 @@ stackgraph: bucket: sts-stackgraph-backup # S3 prefix path for backups s3Prefix: "" - # Archive split to multiple parts - multipartArchive: true # Restore configuration restore: # Label selector for deployments to scale down during restore diff --git a/internal/foundation/config/testdata/validStorageConfigMapNoStackpacks.yaml b/internal/foundation/config/testdata/validStorageConfigMapNoStackpacks.yaml index 0c058f0..6983ecd 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapNoStackpacks.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapNoStackpacks.yaml @@ -48,7 +48,6 @@ storage: stackgraph: bucket: sts-stackgraph-backup s3Prefix: "" - multipartArchive: true restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging diff --git a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml index 036f06f..ae82bcf 100644 --- a/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml +++ b/internal/foundation/config/testdata/validStorageConfigMapOnly.yaml @@ -89,7 +89,6 @@ storage: stackgraph: bucket: sts-stackgraph-backup s3Prefix: "" - multipartArchive: true restore: scaleDownLabelSelector: "observability.suse.com/scalable-during-stackgraph-restore=true" loggingConfigConfigMap: suse-observability-logging diff --git a/internal/scripts/scripts/restore-stackgraph-backup.sh b/internal/scripts/scripts/restore-stackgraph-backup.sh index db11cef..3da0526 100644 --- a/internal/scripts/scripts/restore-stackgraph-backup.sh +++ b/internal/scripts/scripts/restore-stackgraph-backup.sh @@ -9,24 +9,7 @@ export AWS_SECRET_ACCESS_KEY AWS_SECRET_ACCESS_KEY="$(cat /aws-keys/secretkey)" echo "=== Downloading StackGraph backup \"${BACKUP_FILE}\" from bucket \"${BACKUP_STACKGRAPH_BUCKET_NAME}\"..." - -if [ "${BACKUP_STACKGRAPH_MULTIPART_ARCHIVE:-false}" == "false" ]; then - sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_FILE}" "${TMP_DIR}/${BACKUP_FILE}" -else - # Check if the filename of the snapshot is one of the multiparts - # sts-backup-20240222-0730.graph.00 -> sts-backup-20240222-0730.graph - BACKUP_FILE="${BACKUP_FILE/%.[0-9]*/}" - rm -f "${TMP_DIR}/${BACKUP_FILE}.*" - sts-toolbox aws s3 ls --endpoint "http://${MINIO_ENDPOINT}" --region minio --bucket "${BACKUP_STACKGRAPH_BUCKET_NAME}" --prefix "${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_FILE}" | while read -r backup_file - do - sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}${backup_file}" "${TMP_DIR}/${backup_file}" - done - # Concatenate a multipart arhive - find ${TMP_DIR} -name "${BACKUP_FILE}.*" | sort | while read -r multipart - do - cat "${multipart}" >> "${TMP_DIR}/${BACKUP_FILE}" - done -fi +sts-toolbox aws s3 cp --endpoint "http://${MINIO_ENDPOINT}" --region minio "s3://${BACKUP_STACKGRAPH_BUCKET_NAME}/${BACKUP_STACKGRAPH_S3_PREFIX}${BACKUP_FILE}" "${TMP_DIR}/${BACKUP_FILE}" echo "=== Importing StackGraph data from \"${BACKUP_FILE}\"..." /opt/docker/bin/stackstate-server -Dlogback.configurationFile=/opt/docker/etc_log/logback.xml -import "${TMP_DIR}/${BACKUP_FILE}" "${FORCE_DELETE}"