Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
use the default backup storage location for restic
Signed-off-by: Steve Kriss <steve@heptio.com>
  • Loading branch information
skriss committed Aug 28, 2018
1 parent 833a630 commit 20f89fb
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 45 deletions.
19 changes: 12 additions & 7 deletions pkg/cmd/server/server.go
Expand Up @@ -271,6 +271,11 @@ func (s *server) run() error {

s.watchConfig(originalConfig)

backupStorageLocation, err := s.arkClient.ArkV1().BackupStorageLocations(s.namespace).Get(s.defaultBackupLocation, metav1.GetOptions{})
if err != nil {
return errors.WithStack(err)
}

objectStore, err := getObjectStore(config.BackupStorageProvider.CloudProviderConfig, s.pluginManager)
if err != nil {
return err
Expand All @@ -288,13 +293,13 @@ func (s *server) run() error {
s.blockStore = blockStore
}

if config.BackupStorageProvider.ResticLocation != "" {
if err := s.initRestic(config.BackupStorageProvider); err != nil {
if backupStorageLocation.Spec.Config[restic.ResticLocationConfigKey] != "" {
if err := s.initRestic(backupStorageLocation.Spec.Provider); err != nil {
return err
}
}

if err := s.runControllers(config); err != nil {
if err := s.runControllers(config, backupStorageLocation); err != nil {
return err
}

Expand Down Expand Up @@ -525,7 +530,7 @@ func durationMin(a, b time.Duration) time.Duration {
return b
}

func (s *server) initRestic(config api.ObjectStorageProviderConfig) error {
func (s *server) initRestic(providerName string) error {
// warn if restic daemonset does not exist
if _, err := s.kubeClient.AppsV1().DaemonSets(s.namespace).Get(restic.DaemonSet, metav1.GetOptions{}); apierrors.IsNotFound(err) {
s.logger.Warn("Ark restic daemonset not found; restic backups/restores will not work until it's created")
Expand All @@ -539,7 +544,7 @@ func (s *server) initRestic(config api.ObjectStorageProviderConfig) error {
}

// set the env vars that restic uses for creds purposes
if config.Name == string(restic.AzureBackend) {
if providerName == string(restic.AzureBackend) {
os.Setenv("AZURE_ACCOUNT_NAME", os.Getenv("AZURE_STORAGE_ACCOUNT_ID"))
os.Setenv("AZURE_ACCOUNT_KEY", os.Getenv("AZURE_STORAGE_KEY"))
}
Expand Down Expand Up @@ -578,7 +583,7 @@ func (s *server) initRestic(config api.ObjectStorageProviderConfig) error {
return nil
}

func (s *server) runControllers(config *api.Config) error {
func (s *server) runControllers(config *api.Config, defaultBackupLocation *api.BackupStorageLocation) error {
s.logger.Info("Starting controllers")

ctx := s.ctx
Expand Down Expand Up @@ -755,7 +760,7 @@ func (s *server) runControllers(config *api.Config) error {
s.logger,
s.sharedInformerFactory.Ark().V1().ResticRepositories(),
s.arkClient.ArkV1(),
config.BackupStorageProvider,
defaultBackupLocation,
s.resticManager,
)
wg.Add(1)
Expand Down
8 changes: 4 additions & 4 deletions pkg/controller/restic_repository_controller.go
Expand Up @@ -44,7 +44,7 @@ type resticRepositoryController struct {

resticRepositoryClient arkv1client.ResticRepositoriesGetter
resticRepositoryLister listers.ResticRepositoryLister
objectStorageConfig arkv1api.ObjectStorageProviderConfig
storageLocation *arkv1api.BackupStorageLocation
repositoryManager restic.RepositoryManager

clock clock.Clock
Expand All @@ -55,14 +55,14 @@ func NewResticRepositoryController(
logger logrus.FieldLogger,
resticRepositoryInformer informers.ResticRepositoryInformer,
resticRepositoryClient arkv1client.ResticRepositoriesGetter,
objectStorageConfig arkv1api.ObjectStorageProviderConfig,
storageLocation *arkv1api.BackupStorageLocation,
repositoryManager restic.RepositoryManager,
) Interface {
c := &resticRepositoryController{
genericController: newGenericController("restic-repository", logger),
resticRepositoryClient: resticRepositoryClient,
resticRepositoryLister: resticRepositoryInformer.Lister(),
objectStorageConfig: objectStorageConfig,
storageLocation: storageLocation,
repositoryManager: repositoryManager,
clock: &clock.RealClock{},
}
Expand Down Expand Up @@ -139,7 +139,7 @@ func (c *resticRepositoryController) initializeRepo(req *v1.ResticRepository, lo

// defaulting - if the patch fails, return an error so the item is returned to the queue
if err := c.patchResticRepository(req, func(r *v1.ResticRepository) {
r.Spec.ResticIdentifier = restic.GetRepoIdentifier(c.objectStorageConfig, r.Name)
r.Spec.ResticIdentifier = restic.GetRepoIdentifier(c.storageLocation, r.Name)

if r.Spec.MaintenanceFrequency.Duration <= 0 {
r.Spec.MaintenanceFrequency = metav1.Duration{Duration: restic.DefaultMaintenanceFrequency}
Expand Down
1 change: 1 addition & 0 deletions pkg/restic/common.go
Expand Up @@ -36,6 +36,7 @@ const (
DaemonSet = "restic"
InitContainer = "restic-wait"
DefaultMaintenanceFrequency = 24 * time.Hour
ResticLocationConfigKey = "restic-location"

podAnnotationPrefix = "snapshot.ark.heptio.com/"
volumesToBackupAnnotation = "backup.ark.heptio.com/backup-volumes"
Expand Down
17 changes: 9 additions & 8 deletions pkg/restic/config.go
Expand Up @@ -38,9 +38,10 @@ var getAWSBucketRegion = aws.GetBucketRegion

// getRepoPrefix returns the prefix of the value of the --repo flag for
// restic commands, i.e. everything except the "/<repo-name>".
func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string {
func getRepoPrefix(location *arkv1api.BackupStorageLocation) string {
var (
parts = strings.SplitN(config.ResticLocation, "/", 2)
resticLocation = location.Spec.Config[ResticLocationConfigKey]
parts = strings.SplitN(resticLocation, "/", 2)
bucket, path, prefix string
)

Expand All @@ -51,13 +52,13 @@ func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string {
path = parts[1]
}

switch BackendType(config.Name) {
switch BackendType(location.Spec.Provider) {
case AWSBackend:
var url string
switch {
// non-AWS, S3-compatible object store
case config.Config["s3Url"] != "":
url = config.Config["s3Url"]
case location.Spec.Config["s3Url"] != "":
url = location.Spec.Config["s3Url"]
default:
region, err := getAWSBucketRegion(bucket)
if err != nil {
Expand All @@ -68,7 +69,7 @@ func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string {
url = fmt.Sprintf("s3-%s.amazonaws.com", region)
}

return fmt.Sprintf("s3:%s/%s", url, config.ResticLocation)
return fmt.Sprintf("s3:%s/%s", url, resticLocation)
case AzureBackend:
prefix = "azure"
case GCPBackend:
Expand All @@ -80,8 +81,8 @@ func getRepoPrefix(config arkv1api.ObjectStorageProviderConfig) string {

// GetRepoIdentifier returns the string to be used as the value of the --repo flag in
// restic commands for the given repository.
func GetRepoIdentifier(config arkv1api.ObjectStorageProviderConfig, name string) string {
prefix := getRepoPrefix(config)
func GetRepoIdentifier(location *arkv1api.BackupStorageLocation, name string) string {
prefix := getRepoPrefix(location)

return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name)
}
65 changes: 39 additions & 26 deletions pkg/restic/config_test.go
Expand Up @@ -30,47 +30,60 @@ func TestGetRepoIdentifier(t *testing.T) {
getAWSBucketRegion = func(string) (string, error) {
return "", errors.New("no region found")
}
config := arkv1api.ObjectStorageProviderConfig{
CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "aws"},
ResticLocation: "bucket/prefix",

backupLocation := &arkv1api.BackupStorageLocation{
Spec: arkv1api.BackupStorageLocationSpec{
Provider: "aws",
Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"},
},
}
assert.Equal(t, "s3:s3.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(config, "repo-1"))
assert.Equal(t, "s3:s3.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1"))

// stub implementation of getAWSBucketRegion
getAWSBucketRegion = func(string) (string, error) {
return "us-west-2", nil
}

config = arkv1api.ObjectStorageProviderConfig{
CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "aws"},
ResticLocation: "bucket",
backupLocation = &arkv1api.BackupStorageLocation{
Spec: arkv1api.BackupStorageLocationSpec{
Provider: "aws",
Config: map[string]string{ResticLocationConfigKey: "bucket"},
},
}
assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/repo-1", GetRepoIdentifier(config, "repo-1"))
assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/repo-1", GetRepoIdentifier(backupLocation, "repo-1"))

config = arkv1api.ObjectStorageProviderConfig{
CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "aws"},
ResticLocation: "bucket/prefix",
backupLocation = &arkv1api.BackupStorageLocation{
Spec: arkv1api.BackupStorageLocationSpec{
Provider: "aws",
Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"},
},
}
assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(config, "repo-1"))
assert.Equal(t, "s3:s3-us-west-2.amazonaws.com/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1"))

config = arkv1api.ObjectStorageProviderConfig{
CloudProviderConfig: arkv1api.CloudProviderConfig{
Name: "aws",
Config: map[string]string{"s3Url": "alternate-url"},
backupLocation = &arkv1api.BackupStorageLocation{
Spec: arkv1api.BackupStorageLocationSpec{
Provider: "aws",
Config: map[string]string{
ResticLocationConfigKey: "bucket/prefix",
"s3Url": "alternate-url",
},
},
ResticLocation: "bucket/prefix",
}
assert.Equal(t, "s3:alternate-url/bucket/prefix/repo-1", GetRepoIdentifier(config, "repo-1"))
assert.Equal(t, "s3:alternate-url/bucket/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1"))

config = arkv1api.ObjectStorageProviderConfig{
CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "azure"},
ResticLocation: "bucket/prefix",
backupLocation = &arkv1api.BackupStorageLocation{
Spec: arkv1api.BackupStorageLocationSpec{
Provider: "azure",
Config: map[string]string{ResticLocationConfigKey: "bucket/prefix"},
},
}
assert.Equal(t, "azure:bucket:/prefix/repo-1", GetRepoIdentifier(config, "repo-1"))
assert.Equal(t, "azure:bucket:/prefix/repo-1", GetRepoIdentifier(backupLocation, "repo-1"))

config = arkv1api.ObjectStorageProviderConfig{
CloudProviderConfig: arkv1api.CloudProviderConfig{Name: "gcp"},
ResticLocation: "bucket-2/prefix-2",
backupLocation = &arkv1api.BackupStorageLocation{
Spec: arkv1api.BackupStorageLocationSpec{
Provider: "gcp",
Config: map[string]string{ResticLocationConfigKey: "bucket-2/prefix-2"},
},
}
assert.Equal(t, "gs:bucket-2:/prefix-2/repo-2", GetRepoIdentifier(config, "repo-2"))
assert.Equal(t, "gs:bucket-2:/prefix-2/repo-2", GetRepoIdentifier(backupLocation, "repo-2"))
}

0 comments on commit 20f89fb

Please sign in to comment.