Kopia Integration Change - Storage Configuration (#5142)

* unified repo storge config

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

* add UT

Signed-off-by: Lyndon-Li <lyonghui@vmware.com>

changelogs/unreleased/5142-lyndon

@@ -0,0 +1,4 @@
+Kopia Integration: Add the Unified Repository Interface definition.
+Kopia Integration: Add the changes for Unified Repository storage config.
+
+Related Issues; #5076, #5080

pkg/controller/restic_repository_controller.go

@@ -31,6 +31,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
+	repoconfig "github.com/vmware-tanzu/velero/pkg/repository/config"
 	"github.com/vmware-tanzu/velero/pkg/restic"
 	"github.com/vmware-tanzu/velero/pkg/util/kube"
 )
@@ -127,7 +128,7 @@ func (r *ResticRepoReconciler) initializeRepo(ctx context.Context, req *velerov1
 		return r.patchResticRepository(ctx, req, repoNotReady(err.Error()))
 	}
 
-	repoIdentifier, err := restic.GetRepoIdentifier(loc, req.Spec.VolumeNamespace)
+	repoIdentifier, err := repoconfig.GetRepoIdentifier(loc, req.Spec.VolumeNamespace)
 	if err != nil {
 		return r.patchResticRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
 			rr.Status.Message = err.Error()

pkg/repository/config/aws.go

@@ -0,0 +1,99 @@
+/*
+Copyright the Velero contributors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"context"
+	"os"
+
+	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/endpoints"
+	"github.com/aws/aws-sdk-go/aws/session"
+	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/pkg/errors"
+)
+
+const (
+	// AWS specific environment variable
+	awsProfileEnvVar         = "AWS_PROFILE"
+	awsProfileKey            = "profile"
+	awsCredentialsFileEnvVar = "AWS_SHARED_CREDENTIALS_FILE"
+)
+
+// GetS3ResticEnvVars gets the environment variables that restic
+// relies on (AWS_PROFILE) based on info in the provided object
+// storage location config map.
+func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) {
+	result := make(map[string]string)
+
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
+		result[awsCredentialsFileEnvVar] = credentialsFile
+	}
+
+	if profile, ok := config[awsProfileKey]; ok {
+		result[awsProfileEnvVar] = profile
+	}
+
+	return result, nil
+}
+
+// GetS3Credentials gets the S3 credential values according to the information
+// of the provided config or the system's environment variables
+func GetS3Credentials(config map[string]string) (credentials.Value, error) {
+	credentialsFile := config[CredentialsFileKey]
+	if credentialsFile == "" {
+		credentialsFile = os.Getenv("AWS_SHARED_CREDENTIALS_FILE")
+	}
+
+	if credentialsFile == "" {
+		return credentials.Value{}, errors.New("missing credential file")
+	}
+
+	creds := credentials.NewSharedCredentials(credentialsFile, "")
+	credValue, err := creds.Get()
+	if err != nil {
+		return credValue, err
+	}
+
+	return credValue, nil
+}
+
+// GetAWSBucketRegion returns the AWS region that a bucket is in, or an error
+// if the region cannot be determined.
+func GetAWSBucketRegion(bucket string) (string, error) {
+	var region string
+
+	sess, err := session.NewSession()
+	if err != nil {
+		return "", errors.WithStack(err)
+	}
+
+	for _, partition := range endpoints.DefaultPartitions() {
+		for regionHint := range partition.Regions() {
+			region, _ = s3manager.GetBucketRegion(context.Background(), sess, bucket, regionHint)
+
+			// we only need to try a single region hint per partition, so break after the first
+			break
+		}
+
+		if region != "" {
+			return region, nil
+		}
+	}
+
+	return "", errors.New("unable to determine bucket's region")
+}

pkg/restic/aws_test.go → pkg/repository/config/aws_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package config
 
 import (
 	"testing"
@@ -55,7 +55,7 @@ func TestGetS3ResticEnvVars(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual, err := getS3ResticEnvVars(tc.config)
+			actual, err := GetS3ResticEnvVars(tc.config)
 
 			require.NoError(t, err)
 

pkg/restic/azure.go → pkg/repository/config/azure.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package config
 
 import (
 	"context"
@@ -37,6 +37,7 @@ const (
 	storageAccountConfigKey          = "storageAccount"
 	storageAccountKeyEnvVarConfigKey = "storageAccountKeyEnvVar"
 	subscriptionIDConfigKey          = "subscriptionId"
+	storageDomainConfigKey           = "storageDomain"
 )
 
 // getSubscriptionID gets the subscription ID from the 'config' map if it contains
@@ -131,10 +132,10 @@ func mapLookup(data map[string]string) func(string) string {
 	}
 }
 
-// getAzureResticEnvVars gets the environment variables that restic
+// GetAzureResticEnvVars gets the environment variables that restic
 // relies on (AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY) based
 // on info in the provided object storage location config map.
-func getAzureResticEnvVars(config map[string]string) (map[string]string, error) {
+func GetAzureResticEnvVars(config map[string]string) (map[string]string, error) {
 	storageAccountKey, _, err := getStorageAccountKey(config)
 	if err != nil {
 		return nil, err
@@ -158,7 +159,7 @@ func credentialsFileFromEnv() string {
 // selectCredentialsFile selects the Azure credentials file to use, retrieving it
 // from the given config or falling back to retrieving it from the environment.
 func selectCredentialsFile(config map[string]string) string {
-	if credentialsFile, ok := config[credentialsFileKey]; ok {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
 		return credentialsFile
 	}
 
@@ -208,3 +209,22 @@ func getRequiredValues(getValue func(string) string, keys ...string) (map[string
 
 	return results, nil
 }
+
+// GetAzureStorageDomain gets the Azure storage domain required by a Azure blob connection,
+// if the provided config doean't have the value, get it from system's environment variables
+func GetAzureStorageDomain(config map[string]string) string {
+	if domain, exist := config[storageDomainConfigKey]; exist {
+		return domain
+	} else {
+		return os.Getenv(cloudNameEnvVar)
+	}
+}
+
+func GetAzureCredentials(config map[string]string) (string, string, error) {
+	storageAccountKey, _, err := getStorageAccountKey(config)
+	if err != nil {
+		return "", "", err
+	}
+
+	return config[storageAccountConfigKey], storageAccountKey, nil
+}

pkg/restic/azure_test.go → pkg/repository/config/azure_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package config
 
 import (
 	"os"

pkg/restic/config.go → pkg/repository/config/config.go

@@ -14,17 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package config
 
 import (
-	"context"
 	"fmt"
 	"path"
 	"strings"
 
-	"github.com/aws/aws-sdk-go/aws/endpoints"
-	"github.com/aws/aws-sdk-go/aws/session"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
 	"github.com/pkg/errors"
 
 	velerov1api "github.com/vmware-tanzu/velero/pkg/apis/velero/v1"
@@ -37,11 +33,18 @@ const (
 	AWSBackend   BackendType = "velero.io/aws"
 	AzureBackend BackendType = "velero.io/azure"
 	GCPBackend   BackendType = "velero.io/gcp"
+	FSBackend    BackendType = "velero.io/fs"
+)
+
+const (
+	// CredentialsFileKey is the key within a BSL config that is checked to see if
+	// the BSL is using its own credentials, rather than those in the environment
+	CredentialsFileKey = "credentialsFile"
 )
 
 // this func is assigned to a package-level variable so it can be
 // replaced when unit-testing
-var getAWSBucketRegion = getBucketRegion
+var getAWSBucketRegion = GetAWSBucketRegion
 
 // getRepoPrefix returns the prefix of the value of the --repo flag for
 // restic commands, i.e. everything except the "/<repo-name>".
@@ -55,7 +58,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 		prefix = layout.GetResticDir()
 	}
 
-	backendType := getBackendType(location.Spec.Provider)
+	backendType := GetBackendType(location.Spec.Provider)
 
 	if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" {
 		return repoPrefix, nil
@@ -89,14 +92,18 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 	return "", errors.New("restic repository prefix (resticRepoPrefix) not specified in backup storage location's config")
 }
 
-func getBackendType(provider string) BackendType {
+func GetBackendType(provider string) BackendType {
 	if !strings.Contains(provider, "/") {
 		provider = "velero.io/" + provider
 	}
 
 	return BackendType(provider)
 }
 
+func IsBackendTypeValid(backendType BackendType) bool {
+	return (backendType == AWSBackend || backendType == AzureBackend || backendType == GCPBackend || backendType == FSBackend)
+}
+
 // GetRepoIdentifier returns the string to be used as the value of the --repo flag in
 // restic commands for the given repository.
 func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string) (string, error) {
@@ -107,29 +114,3 @@ func GetRepoIdentifier(location *velerov1api.BackupStorageLocation, name string)
 
 	return fmt.Sprintf("%s/%s", strings.TrimSuffix(prefix, "/"), name), nil
 }
-
-// getBucketRegion returns the AWS region that a bucket is in, or an error
-// if the region cannot be determined.
-func getBucketRegion(bucket string) (string, error) {
-	var region string
-
-	sess, err := session.NewSession()
-	if err != nil {
-		return "", errors.WithStack(err)
-	}
-
-	for _, partition := range endpoints.DefaultPartitions() {
-		for regionHint := range partition.Regions() {
-			region, _ = s3manager.GetBucketRegion(context.Background(), sess, bucket, regionHint)
-
-			// we only need to try a single region hint per partition, so break after the first
-			break
-		}
-
-		if region != "" {
-			return region, nil
-		}
-	}
-
-	return "", errors.New("unable to determine bucket's region")
-}

pkg/restic/config_test.go → pkg/repository/config/config_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package config
 
 import (
 	"testing"

pkg/restic/gcp.go → pkg/repository/config/gcp.go

@@ -14,21 +14,33 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package config
+
+import "os"
 
 const (
 	// GCP specific environment variable
 	gcpCredentialsFileEnvVar = "GOOGLE_APPLICATION_CREDENTIALS"
 )
 
-// getGCPResticEnvVars gets the environment variables that restic relies
+// GetGCPResticEnvVars gets the environment variables that restic relies
 // on based on info in the provided object storage location config map.
-func getGCPResticEnvVars(config map[string]string) (map[string]string, error) {
+func GetGCPResticEnvVars(config map[string]string) (map[string]string, error) {
 	result := make(map[string]string)
 
-	if credentialsFile, ok := config[credentialsFileKey]; ok {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
 		result[gcpCredentialsFileEnvVar] = credentialsFile
 	}
 
 	return result, nil
 }
+
+// GetGCPCredentials gets the credential file required by a GCP bucket connection,
+// if the provided config doean't have the value, get it from system's environment variables
+func GetGCPCredentials(config map[string]string) string {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
+		return credentialsFile
+	} else {
+		return os.Getenv("GOOGLE_APPLICATION_CREDENTIALS")
+	}
+}

pkg/restic/gcp_test.go → pkg/repository/config/gcp_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package config
 
 import (
 	"testing"
@@ -46,7 +46,7 @@ func TestGetGCPResticEnvVars(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual, err := getGCPResticEnvVars(tc.config)
+			actual, err := GetGCPResticEnvVars(tc.config)
 
 			require.NoError(t, err)