Add support for specifying "region" for S3 backend (#1050)

hossainemruz · web-flow · commit 99dcf0a5e958 · 2020-02-15T13:58:19.000-08:00
* Add support for specifying "region" for S3 backend

Signed-off-by: hossainemruz &lt;emruz@appscode.com&gt;

* Update restic version to 0.9.6

Signed-off-by: hossainemruz &lt;emruz@appscode.com&gt;

* Revendor using kubernetes-1.16.3 branch of objectstore-api

Signed-off-by: hossainemruz &lt;emruz@appscode.com&gt;
diff --git a/Makefile b/Makefile
@@ -50,7 +50,7 @@ endif
 
 RESTIC_VER       := 0.8.3
 # also update in restic wrapper library
-NEW_RESTIC_VER   := 0.9.5
+NEW_RESTIC_VER   := 0.9.6
 
 ###
 ### These variables should not need tweaking.
diff --git a/apis/constants.go b/apis/constants.go
@@ -34,6 +34,7 @@ const (
 	RepositoryBucket     = "REPOSITORY_BUCKET"
 	RepositoryPrefix     = "REPOSITORY_PREFIX"
 	RepositoryEndpoint   = "REPOSITORY_ENDPOINT"
+	RepositoryRegion     = "REPOSITORY_REGION"
 
 	Hostname       = "HOSTNAME"
 	SourceHostname = "SOURCE_HOSTNAME"
diff --git a/go.mod b/go.mod
@@ -41,8 +41,8 @@ require (
 	kmodules.xyz/constants v0.0.0-20191024095500-cd4313df4aa6
 	kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891
 	kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419
-	kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0
-	kmodules.xyz/offshoot-api v0.0.0-20200103145223-2c4f520520d2
+	kmodules.xyz/objectstore-api v0.0.0-20200213071506-f176860e67f7
+	kmodules.xyz/offshoot-api v0.0.0-20200207061254-705ac16690ec
 	kmodules.xyz/openshift v0.0.0-20191127145035-f6c48a90dbb7
 	kmodules.xyz/prober v0.0.0-20191216013129-7c91e2b0edb2
 	kmodules.xyz/webhook-runtime v0.0.0-20191127075323-d4bfdee6974d
diff --git a/go.sum b/go.sum
@@ -697,10 +697,10 @@ kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891 h1:2W/fqLbAurvup
 kmodules.xyz/crd-schema-fuzz v0.0.0-20191129174258-81f984340891/go.mod h1:9NXNZ4xhqof0WngtIuo4vl+WoCQpLJEJcyuEo3mPpiM=
 kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419 h1:o6KD8XMxdyRR3rqScTsWvcufFDT7vQBnYXpHUp6UtRg=
 kmodules.xyz/custom-resources v0.0.0-20191130062942-f41b54f62419/go.mod h1:EksiAQK3p3bVU5cNBjLT0z5V7er1tvWLBWILTuf6G/0=
-kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0 h1:ilgkGU/bteKRvH99piXxU8b2c1E9lwxb0sDpLLhy4iY=
-kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0/go.mod h1:AE1rz+T/3dlUwH5pLgCNghWeeFfeVBnBXKJVpsLHlmc=
-kmodules.xyz/offshoot-api v0.0.0-20200103145223-2c4f520520d2 h1:c/wxNy8KgaS8v/htruqtL0YgrKciUKroK1zXPVZHuqg=
-kmodules.xyz/offshoot-api v0.0.0-20200103145223-2c4f520520d2/go.mod h1:k1M+OOFHbMeS6N6OB2j9smKfMCyQOEQjZ3SLZ/KSe+w=
+kmodules.xyz/objectstore-api v0.0.0-20200213071506-f176860e67f7 h1:b2ahs7RkOilWVsXHgx/t44FhL8xdbB0r2/W1Jf62Il8=
+kmodules.xyz/objectstore-api v0.0.0-20200213071506-f176860e67f7/go.mod h1:AE1rz+T/3dlUwH5pLgCNghWeeFfeVBnBXKJVpsLHlmc=
+kmodules.xyz/offshoot-api v0.0.0-20200207061254-705ac16690ec h1:JdgvQhxyFXnxInPK42R3saSBjkhiiy8k5Uc9UdneVqE=
+kmodules.xyz/offshoot-api v0.0.0-20200207061254-705ac16690ec/go.mod h1:k1M+OOFHbMeS6N6OB2j9smKfMCyQOEQjZ3SLZ/KSe+w=
 kmodules.xyz/openshift v0.0.0-20191127054841-113eeb13704f/go.mod h1:de9+Zk2+wtGhRmGFBZ9T4+0lJZr4rHXW0dHffCaeO+g=
 kmodules.xyz/openshift v0.0.0-20191127145035-f6c48a90dbb7 h1:9PImU28r8bt+QoW7FkYbJ+prsZSNtwctLeXJj3wJK5U=
 kmodules.xyz/openshift v0.0.0-20191127145035-f6c48a90dbb7/go.mod h1:de9+Zk2+wtGhRmGFBZ9T4+0lJZr4rHXW0dHffCaeO+g=
diff --git a/pkg/cmds/backup_pvc.go b/pkg/cmds/backup_pvc.go
@@ -71,6 +71,7 @@ func NewCmdBackupPVC() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.Provider, "provider", setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
 	cmd.Flags().StringVar(&setupOpt.Bucket, "bucket", setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
 	cmd.Flags().StringVar(&setupOpt.Endpoint, "endpoint", setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+	cmd.Flags().StringVar(&setupOpt.Region, "region", setupOpt.Region, "Region for s3/s3 compatible backend")
 	cmd.Flags().StringVar(&setupOpt.Path, "path", setupOpt.Path, "Directory inside the bucket where backed up data will be stored")
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
diff --git a/pkg/cmds/restore_pvc.go b/pkg/cmds/restore_pvc.go
@@ -70,6 +70,7 @@ func NewCmdRestorePVC() *cobra.Command {
 	cmd.Flags().StringVar(&setupOpt.Provider, "provider", setupOpt.Provider, "Backend provider (i.e. gcs, s3, azure etc)")
 	cmd.Flags().StringVar(&setupOpt.Bucket, "bucket", setupOpt.Bucket, "Name of the cloud bucket/container (keep empty for local backend)")
 	cmd.Flags().StringVar(&setupOpt.Endpoint, "endpoint", setupOpt.Endpoint, "Endpoint for s3/s3 compatible backend or REST server URL")
+	cmd.Flags().StringVar(&setupOpt.Region, "region", setupOpt.Region, "Region for s3/s3 compatible backend")
 	cmd.Flags().StringVar(&setupOpt.Path, "path", setupOpt.Path, "Directory inside the bucket where backed up data has been stored")
 	cmd.Flags().StringVar(&setupOpt.SecretDir, "secret-dir", setupOpt.SecretDir, "Directory where storage secret has been mounted")
 	cmd.Flags().StringVar(&setupOpt.ScratchDir, "scratch-dir", setupOpt.ScratchDir, "Temporary directory")
diff --git a/pkg/controller/inputs.go b/pkg/controller/inputs.go
@@ -119,8 +119,13 @@ func (c *StashController) inputsForRepository(repository *apiAlpha.Repository) (
 	if repository.Spec.Backend.S3 != nil && repository.Spec.Backend.S3.Endpoint != "" {
 		inputs[apis.RepositoryEndpoint] = repository.Spec.Backend.S3.Endpoint
 	}
-	if repository.Spec.Backend.Rest != nil && repository.Spec.Backend.Rest.URL != "" {
-		inputs[apis.RepositoryEndpoint] = repository.Spec.Backend.Rest.URL
+	endpoint, found := repository.Spec.Backend.Endpoint()
+	if found {
+		inputs[apis.RepositoryEndpoint] = endpoint
+	}
+	region, found := repository.Spec.Backend.Region()
+	if found {
+		inputs[apis.RepositoryRegion] = region
 	}
 	inputs[apis.MaxConnections] = strconv.FormatInt(repository.Spec.Backend.MaxConnections(), 10)
 	return
diff --git a/pkg/restic/commands.go b/pkg/restic/commands.go
@@ -36,7 +36,7 @@ import (
 )
 
 const (
-	ResticCMD = "/bin/restic_0.9.5"
+	ResticCMD = "/bin/restic_0.9.6"
 )
 
 type Snapshot struct {
diff --git a/pkg/restic/config.go b/pkg/restic/config.go
@@ -77,6 +77,7 @@ type SetupOptions struct {
 	Provider       string
 	Bucket         string
 	Endpoint       string
+	Region         string
 	Path           string
 	SecretDir      string
 	CacertFile     string
diff --git a/pkg/restic/setup.go b/pkg/restic/setup.go
@@ -33,6 +33,7 @@ const (
 
 	AWS_ACCESS_KEY_ID     = "AWS_ACCESS_KEY_ID"
 	AWS_SECRET_ACCESS_KEY = "AWS_SECRET_ACCESS_KEY"
+	AWS_DEFAULT_REGION    = "AWS_DEFAULT_REGION"
 
 	GOOGLE_PROJECT_ID               = "GOOGLE_PROJECT_ID"
 	GOOGLE_SERVICE_ACCOUNT_JSON_KEY = "GOOGLE_SERVICE_ACCOUNT_JSON_KEY"
@@ -127,6 +128,10 @@ func (w *ResticWrapper) setupEnv() error {
 			w.sh.SetEnv(AWS_SECRET_ACCESS_KEY, string(v))
 		}
 
+		if w.config.Region != "" {
+			w.sh.SetEnv(AWS_DEFAULT_REGION, w.config.Region)
+		}
+
 	case storage.ProviderGCS:
 		r := fmt.Sprintf("gs:%s:/%s", w.config.Bucket, w.config.Path)
 		w.sh.SetEnv(RESTIC_REPOSITORY, r)
diff --git a/pkg/util/options.go b/pkg/util/options.go
@@ -93,12 +93,14 @@ func SetupOptionsForRepository(repository api_v1alpha1.Repository, extraOpt Extr
 		return restic.SetupOptions{}, err
 	}
 	endpoint, _ := repository.Spec.Backend.Endpoint()
+	region, _ := repository.Spec.Backend.Region()
 
 	return restic.SetupOptions{
 		Provider:       provider,
 		Bucket:         bucket,
 		Path:           prefix,
 		Endpoint:       endpoint,
+		Region:         region,
 		CacertFile:     extraOpt.CacertFile,
 		SecretDir:      extraOpt.SecretDir,
 		ScratchDir:     extraOpt.ScratchDir,
diff --git a/vendor/kmodules.xyz/objectstore-api/api/v1/helpers.go b/vendor/kmodules.xyz/objectstore-api/api/v1/helpers.go
@@ -155,3 +155,11 @@ func (backend Backend) Endpoint() (string, bool) {
 	}
 	return "", false
 }
+
+// Region returns region of S3/S3 compatible backend
+func (backend Backend) Region() (string, bool) {
+	if backend.S3 != nil {
+		return backend.S3.Region, true
+	}
+	return "", false
+}
diff --git a/vendor/kmodules.xyz/offshoot-api/util/runtime_settings.go b/vendor/kmodules.xyz/offshoot-api/util/runtime_settings.go
@@ -97,7 +97,7 @@ func ApplyPodRuntimeSettings(podSpec core.PodSpec, settings ofst.PodRuntimeSetti
 		podSpec.SecurityContext = settings.SecurityContext
 	}
 	if len(settings.ImagePullSecrets) > 0 {
-		podSpec.ImagePullSecrets = settings.ImagePullSecrets
+		podSpec.ImagePullSecrets = core_util.MergeLocalObjectReferences(podSpec.ImagePullSecrets, settings.ImagePullSecrets)
 	}
 	if settings.Affinity != nil {
 		podSpec.Affinity = settings.Affinity
diff --git a/vendor/modules.txt b/vendor/modules.txt
@@ -1036,10 +1036,10 @@ kmodules.xyz/custom-resources/client/informers/externalversions/appcatalog
 kmodules.xyz/custom-resources/client/informers/externalversions/appcatalog/v1alpha1
 kmodules.xyz/custom-resources/client/informers/externalversions/internalinterfaces
 kmodules.xyz/custom-resources/client/listers/appcatalog/v1alpha1
-# kmodules.xyz/objectstore-api v0.0.0-20191127144749-5881939b57f0
+# kmodules.xyz/objectstore-api v0.0.0-20200213071506-f176860e67f7
 kmodules.xyz/objectstore-api/api/v1
 kmodules.xyz/objectstore-api/osm
-# kmodules.xyz/offshoot-api v0.0.0-20200103145223-2c4f520520d2
+# kmodules.xyz/offshoot-api v0.0.0-20200207061254-705ac16690ec
 kmodules.xyz/offshoot-api/api/v1
 kmodules.xyz/offshoot-api/util
 # kmodules.xyz/openshift v0.0.0-20191127145035-f6c48a90dbb7

Original file line number	Diff line number	Diff line change
`@@ -36,7 +36,7 @@ import (`
`36`	`36`	`)`
`37`	`37`
`38`	`38`	`const (`
`39`		`- ResticCMD = "/bin/restic_0.9.5"`
	`39`	`+ ResticCMD = "/bin/restic_0.9.6"`
`40`	`40`	`)`
`41`	`41`
`42`	`42`	`type Snapshot struct {`