Skip to content

Commit

Permalink
fix: ignore workspace if no backup config is found
Browse files Browse the repository at this point in the history
  • Loading branch information
Sidddddarth authored and cisse21 committed Jul 31, 2023
1 parent bef1280 commit c5d00be
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 11 deletions.
16 changes: 10 additions & 6 deletions jobsdb/backup.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,19 +4,21 @@ import (
"context"
"database/sql"
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
"time"

"github.com/cenkalti/backoff"
"golang.org/x/sync/errgroup"

"github.com/cenkalti/backoff"

"github.com/rudderlabs/rudder-go-kit/config"
"github.com/rudderlabs/rudder-go-kit/filemanager"
"github.com/rudderlabs/rudder-go-kit/stats"
fileuploader "github.com/rudderlabs/rudder-server/services/fileuploader"
"github.com/rudderlabs/rudder-server/services/fileuploader"
"github.com/rudderlabs/rudder-server/utils/misc"
)

Expand Down Expand Up @@ -552,18 +554,20 @@ func (jd *HandleT) createTableDumps(queryFunc func(int64) string, pathFunc func(
return fmt.Errorf("scanning row failed with error : %w", err)
}
preferences, err := jd.fileUploaderProvider.GetStoragePreferences(workspaceID)
if errors.Is(err, fileuploader.NoStorageForWorkspaceError) {
offset++
jd.logger.Infof("getting storage preferences failed with error: %w for workspaceID: %s", err, workspaceID)
continue
}
if err != nil {
return fmt.Errorf("getting storage preferences failed with error : %w", err)
return fmt.Errorf("getting storage preferences failed with error: %w for workspaceID: %s", err, workspaceID)
}
if !preferences.Backup(jd.tablePrefix) {
offset++
jd.logger.Infof("Skipping backup for workspace: %s. Preferences: %v and tablePrefix: %s", workspaceID, preferences, jd.tablePrefix)
continue
}
rawJSONRows = append(rawJSONRows, '\n') // appending '\n'
if err != nil {
return fmt.Errorf("error while appending '\n': %w", err)
}
path, err := pathFunc(workspaceID)
if err != nil {
return fmt.Errorf("error while getting path: %w", err)
Expand Down
4 changes: 2 additions & 2 deletions services/fileuploader/fileuploader.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ type StorageSettings struct {
Preferences backendconfig.StoragePreferences
}

var noStorageForWorkspaceErrorString string = "no storage settings found for workspace: %s"
var NoStorageForWorkspaceError = fmt.Errorf("no storage settings found for workspace")

// Provider is an interface that provides file managers and storage preferences for a given workspace.
type Provider interface {
Expand Down Expand Up @@ -67,7 +67,7 @@ func (p *provider) getStorageSettings(workspaceID string) (StorageSettings, erro
defer p.mu.RUnlock()
settings, ok := p.storageSettings[workspaceID]
if !ok {
return StorageSettings{}, fmt.Errorf(noStorageForWorkspaceErrorString, workspaceID)
return StorageSettings{}, NoStorageForWorkspaceError
}
return settings, nil
}
Expand Down
6 changes: 3 additions & 3 deletions services/fileuploader/fileuploader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package fileuploader

import (
"context"
"fmt"
"sync"
"testing"

Expand Down Expand Up @@ -125,7 +124,7 @@ func TestFileUploaderUpdatingWithConfigBackend(t *testing.T) {
Expect(fm3.Prefix()).To(Equal("defaultPrefixWithStorageTTL"))

fm0, err := fileUploaderProvider.GetFileManager("testWorkspaceId-0")
Expect(err).To(Equal(fmt.Errorf(noStorageForWorkspaceErrorString, "testWorkspaceId-0")))
Expect(err).To(Equal(NoStorageForWorkspaceError))
Expect(fm0).To(BeNil())

storageSettings.Wait()
Expand All @@ -137,7 +136,7 @@ func TestFileUploaderUpdatingWithConfigBackend(t *testing.T) {
))

preferences, err = fileUploaderProvider.GetStoragePreferences("testWorkspaceId-0")
Expect(err).To(Equal(fmt.Errorf(noStorageForWorkspaceErrorString, "testWorkspaceId-0")))
Expect(err).To(Equal(NoStorageForWorkspaceError))
Expect(preferences).To(BeEquivalentTo(backendconfig.StoragePreferences{}))

preferences, err = fileUploaderProvider.GetStoragePreferences("testWorkspaceId-2")
Expand Down Expand Up @@ -251,6 +250,7 @@ func TestOverride(t *testing.T) {
"d": "5",
},
}

bucket = overrideWithSettings(config, settings, "wrk-1")
Expect(bucket.Config).To(Equal(map[string]interface{}{
"a": "1",
Expand Down

0 comments on commit c5d00be

Please sign in to comment.