Skip to content

Commit

Permalink
feat(storage): add Custom Placement Config Dual Region Support (#6294)
Browse files Browse the repository at this point in the history
* feat(storage): support Custom Dual Regions with CustomPlacementConfig

* fix typo

* add comments

* address pr comments

* new sublink
  • Loading branch information
cojenco committed Jul 13, 2022
1 parent 6d1e9fe commit 5a8c607
Show file tree
Hide file tree
Showing 2 changed files with 100 additions and 40 deletions.
114 changes: 82 additions & 32 deletions storage/bucket.go
Original file line number Diff line number Diff line change
Expand Up @@ -460,8 +460,13 @@ type BucketAttrs struct {
PredefinedDefaultObjectACL string

// Location is the location of the bucket. It defaults to "US".
// If specifying a dual-region, CustomPlacementConfig should be set in conjunction.
Location string

// The bucket's custom placement configuration that holds a list of
// regional locations for custom dual regions.
CustomPlacementConfig *CustomPlacementConfig

// MetaGeneration is the metadata generation of the bucket.
// This field is read-only.
MetaGeneration int64
Expand Down Expand Up @@ -781,6 +786,15 @@ type BucketWebsite struct {
NotFoundPage string
}

// CustomPlacementConfig holds the bucket's custom placement
// configuration for Custom Dual Regions. See
// https://cloud.google.com/storage/docs/locations#location-dr for more information.
type CustomPlacementConfig struct {
// The list of regional locations in which data is placed.
// Custom Dual Regions require exactly 2 regional locations.
DataLocations []string
}

func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
if b == nil {
return nil, nil
Expand Down Expand Up @@ -814,6 +828,7 @@ func newBucket(b *raw.Bucket) (*BucketAttrs, error) {
LocationType: b.LocationType,
ProjectNumber: b.ProjectNumber,
RPO: toRPO(b),
CustomPlacementConfig: customPlacementFromRaw(b.CustomPlacementConfig),
}, nil
}

Expand Down Expand Up @@ -844,6 +859,7 @@ func newBucketFromProto(b *storagepb.Bucket) *BucketAttrs {
PublicAccessPrevention: toPublicAccessPreventionFromProto(b.GetIamConfig()),
LocationType: b.GetLocationType(),
RPO: toRPOFromProto(b),
CustomPlacementConfig: customPlacementFromProto(b.GetCustomPlacementConfig()),
}
}

Expand Down Expand Up @@ -881,22 +897,23 @@ func (b *BucketAttrs) toRawBucket() *raw.Bucket {
}
}
return &raw.Bucket{
Name: b.Name,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: toRawBucketACL(b.ACL),
DefaultObjectAcl: toRawObjectACL(b.DefaultObjectACL),
Versioning: v,
Labels: labels,
Billing: bb,
Lifecycle: toRawLifecycle(b.Lifecycle),
RetentionPolicy: b.RetentionPolicy.toRawRetentionPolicy(),
Cors: toRawCORS(b.CORS),
Encryption: b.Encryption.toRawBucketEncryption(),
Logging: b.Logging.toRawBucketLogging(),
Website: b.Website.toRawBucketWebsite(),
IamConfiguration: bktIAM,
Rpo: b.RPO.String(),
Name: b.Name,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: toRawBucketACL(b.ACL),
DefaultObjectAcl: toRawObjectACL(b.DefaultObjectACL),
Versioning: v,
Labels: labels,
Billing: bb,
Lifecycle: toRawLifecycle(b.Lifecycle),
RetentionPolicy: b.RetentionPolicy.toRawRetentionPolicy(),
Cors: toRawCORS(b.CORS),
Encryption: b.Encryption.toRawBucketEncryption(),
Logging: b.Logging.toRawBucketLogging(),
Website: b.Website.toRawBucketWebsite(),
IamConfiguration: bktIAM,
Rpo: b.RPO.String(),
CustomPlacementConfig: b.CustomPlacementConfig.toRawCustomPlacement(),
}
}

Expand Down Expand Up @@ -939,22 +956,23 @@ func (b *BucketAttrs) toProtoBucket() *storagepb.Bucket {
}

return &storagepb.Bucket{
Name: b.Name,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: toProtoBucketACL(b.ACL),
DefaultObjectAcl: toProtoObjectACL(b.DefaultObjectACL),
Versioning: v,
Labels: labels,
Billing: bb,
Lifecycle: toProtoLifecycle(b.Lifecycle),
RetentionPolicy: b.RetentionPolicy.toProtoRetentionPolicy(),
Cors: toProtoCORS(b.CORS),
Encryption: b.Encryption.toProtoBucketEncryption(),
Logging: b.Logging.toProtoBucketLogging(),
Website: b.Website.toProtoBucketWebsite(),
IamConfig: bktIAM,
Rpo: b.RPO.String(),
Name: b.Name,
Location: b.Location,
StorageClass: b.StorageClass,
Acl: toProtoBucketACL(b.ACL),
DefaultObjectAcl: toProtoObjectACL(b.DefaultObjectACL),
Versioning: v,
Labels: labels,
Billing: bb,
Lifecycle: toProtoLifecycle(b.Lifecycle),
RetentionPolicy: b.RetentionPolicy.toProtoRetentionPolicy(),
Cors: toProtoCORS(b.CORS),
Encryption: b.Encryption.toProtoBucketEncryption(),
Logging: b.Logging.toProtoBucketLogging(),
Website: b.Website.toProtoBucketWebsite(),
IamConfig: bktIAM,
Rpo: b.RPO.String(),
CustomPlacementConfig: b.CustomPlacementConfig.toProtoCustomPlacement(),
}
}

Expand Down Expand Up @@ -1932,6 +1950,38 @@ func toRPOFromProto(b *storagepb.Bucket) RPO {
}
}

func customPlacementFromRaw(c *raw.BucketCustomPlacementConfig) *CustomPlacementConfig {
if c == nil {
return nil
}
return &CustomPlacementConfig{DataLocations: c.DataLocations}
}

func (c *CustomPlacementConfig) toRawCustomPlacement() *raw.BucketCustomPlacementConfig {
if c == nil {
return nil
}
return &raw.BucketCustomPlacementConfig{
DataLocations: c.DataLocations,
}
}

func (c *CustomPlacementConfig) toProtoCustomPlacement() *storagepb.Bucket_CustomPlacementConfig {
if c == nil {
return nil
}
return &storagepb.Bucket_CustomPlacementConfig{
DataLocations: c.DataLocations,
}
}

func customPlacementFromProto(c *storagepb.Bucket_CustomPlacementConfig) *CustomPlacementConfig {
if c == nil {
return nil
}
return &CustomPlacementConfig{DataLocations: c.GetDataLocations()}
}

// Objects returns an iterator over the objects in the bucket that match the
// Query q. If q is nil, no filtering is done. Objects will be iterated over
// lexicographically by name.
Expand Down
26 changes: 18 additions & 8 deletions storage/integration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -314,12 +314,13 @@ func TestIntegration_BucketCreateDelete(t *testing.T) {

// testedAttrs are the bucket attrs directly compared in this test
type testedAttrs struct {
StorageClass string
VersioningEnabled bool
LocationType string
Labels map[string]string
Location string
Lifecycle Lifecycle
StorageClass string
VersioningEnabled bool
LocationType string
Labels map[string]string
Location string
Lifecycle Lifecycle
CustomPlacementConfig *CustomPlacementConfig
}

for _, test := range []struct {
Expand Down Expand Up @@ -358,12 +359,18 @@ func TestIntegration_BucketCreateDelete(t *testing.T) {
{
name: "dual-region",
attrs: &BucketAttrs{
Location: "US-EAST1+US-WEST1",
Location: "US",
CustomPlacementConfig: &CustomPlacementConfig{
DataLocations: []string{"US-EAST1", "US-WEST1"},
},
},
wantAttrs: testedAttrs{
Location: "US-EAST1+US-WEST1",
Location: "US",
LocationType: "dual-region",
StorageClass: "STANDARD",
CustomPlacementConfig: &CustomPlacementConfig{
DataLocations: []string{"US-EAST1", "US-WEST1"},
},
},
},
} {
Expand Down Expand Up @@ -407,6 +414,9 @@ func TestIntegration_BucketCreateDelete(t *testing.T) {
if gotAttrs.Location != test.wantAttrs.Location {
t.Errorf("location: got %s, want %s", gotAttrs.Location, test.wantAttrs.Location)
}
if got, want := gotAttrs.CustomPlacementConfig, test.wantAttrs.CustomPlacementConfig; !testutil.Equal(got, want) {
t.Errorf("customPlacementConfig: \ngot\t%v\nwant\t%v", got, want)
}

// Delete the bucket and check that the deletion was succesful
if err := b.Delete(ctx); err != nil {
Expand Down

0 comments on commit 5a8c607

Please sign in to comment.