Skip to content

Commit

Permalink
feat: limit access to S3 by VPC ID (#1829)
Browse files Browse the repository at this point in the history
* feat: changes to implement alternative number 1

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* chore: analysis to create a debate

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* chore: improve readability

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* chore: add alternative for use case number 2

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* chore: minor adjustment

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* fix: incorrect format

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* chore: small improvements

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* fix: close quotes

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* fix: format

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* fix: format when adding image

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* fix: error when adding the image

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* feat: Limit access to S3 buckets by VPC ID

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* test: new property in binding

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* test: new variable is required

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* test: change package name

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* chore: remove unexpected arn user

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* chore: be generic and let users decide what level of security is needed

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* test: change test implementation

* create a VPC endpoint
* run the test
* delete the VPC endpoint

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* test: use dms package

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* chore: change property description

[#187430076](https://www.pivotaltracker.com/story/show/187430076)

* test: follow conventions

[#187430076](https://www.pivotaltracker.com/story/show/187430076)
  • Loading branch information
zucchinidev committed May 23, 2024
1 parent 851eec8 commit b099a19
Show file tree
Hide file tree
Showing 20 changed files with 316 additions and 32 deletions.
2 changes: 1 addition & 1 deletion acceptance-tests/aurora_mysql_data_migration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package acceptance_tests_test

import (
"csbbrokerpakaws/acceptance-tests/helpers/apps"
"csbbrokerpakaws/acceptance-tests/helpers/dms"
"csbbrokerpakaws/acceptance-tests/helpers/awscli/dms"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"csbbrokerpakaws/acceptance-tests/helpers/services"

Expand Down
10 changes: 5 additions & 5 deletions acceptance-tests/dynamodb_namespace_data_migration_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package acceptance_tests_test

import (
"csbbrokerpakaws/acceptance-tests/helpers/awscli"
"fmt"
"net/http"
"time"
Expand All @@ -9,7 +10,6 @@ import (
. "github.com/onsi/gomega"

"csbbrokerpakaws/acceptance-tests/helpers/apps"
"csbbrokerpakaws/acceptance-tests/helpers/dms"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"csbbrokerpakaws/acceptance-tests/helpers/services"
)
Expand Down Expand Up @@ -66,7 +66,7 @@ var _ = Describe("DynamoDB Namespace Data Migration", Label("dynamodb-namespace-
var backupReceiver struct {
ARN string `jsonry:"BackupDetails.BackupArn"`
}
dms.AWSToJSON(
awscli.AWSToJSON(
&backupReceiver,
"dynamodb", "create-backup",
"--table-name", legacyTable,
Expand All @@ -78,7 +78,7 @@ var _ = Describe("DynamoDB Namespace Data Migration", Label("dynamodb-namespace-
var describeReceiver struct {
Status string `jsonry:"BackupDescription.BackupDetails.BackupStatus"`
}
dms.AWSToJSON(&describeReceiver, "dynamodb", "describe-backup", "--backup-arn", backupReceiver.ARN, "--region", metadata.Region)
awscli.AWSToJSON(&describeReceiver, "dynamodb", "describe-backup", "--backup-arn", backupReceiver.ARN, "--region", metadata.Region)
g.Expect(describeReceiver.Status).To(Equal("AVAILABLE"))
}).WithTimeout(time.Hour).WithPolling(time.Second).Should(Succeed())

Expand All @@ -87,14 +87,14 @@ var _ = Describe("DynamoDB Namespace Data Migration", Label("dynamodb-namespace-

By("restoring the table into the namespace of the CSB service")
csbTableName := fmt.Sprintf("csb-%s-%s", csbServiceInstance.GUID(), tableSuffix)
dms.AWS("dynamodb", "restore-table-from-backup", "--target-table-name", csbTableName, "--backup-arn", backupReceiver.ARN, "--region", metadata.Region)
awscli.AWS("dynamodb", "restore-table-from-backup", "--target-table-name", csbTableName, "--backup-arn", backupReceiver.ARN, "--region", metadata.Region)

Eventually(func(g Gomega) {
var describeReceiver struct {
RestoreInProgress bool `jsonry:"Table.RestoreSummary.RestoreInProgress"`
Status string `jsonry:"Table.TableStatus"`
}
dms.AWSToJSON(&describeReceiver, "dynamodb", "describe-table", "--table-name", csbTableName, "--region", metadata.Region)
awscli.AWSToJSON(&describeReceiver, "dynamodb", "describe-table", "--table-name", csbTableName, "--region", metadata.Region)
g.Expect(describeReceiver.Status).To(Equal("ACTIVE"))
g.Expect(describeReceiver.RestoreInProgress).To(BeFalse())
}).WithTimeout(time.Hour).WithPolling(10 * time.Second).Should(Succeed())
Expand Down
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
package dms
// Package awscli provides test helpers for setting up AWS resources
package awscli

import (
"fmt"
Expand Down
File renamed without changes.
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package dms

import (
"csbbrokerpakaws/acceptance-tests/helpers/awscli"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"fmt"
)
Expand Down Expand Up @@ -36,7 +37,7 @@ func CreateEndpoint(params CreateEndpointParams) *Endpoint {
ARN string `jsonry:"Endpoint.EndpointArn"`
}

AWSToJSON(
awscli.AWSToJSON(
&receiver,
"dms",
"create-endpoint",
Expand All @@ -58,5 +59,5 @@ func CreateEndpoint(params CreateEndpointParams) *Endpoint {
}

func (e *Endpoint) Cleanup() {
AWS("dms", "delete-endpoint", "--region", e.region, "--endpoint-arn", e.arn)
awscli.AWS("dms", "delete-endpoint", "--region", e.region, "--endpoint-arn", e.arn)
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package dms

import (
"csbbrokerpakaws/acceptance-tests/helpers/awscli"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"fmt"
"time"
Expand Down Expand Up @@ -49,7 +50,7 @@ func createReplicationInstance(replicationSubnetGroupID, envName, region string)
var receiver struct {
ARN string `jsonry:"ReplicationInstance.ReplicationInstanceArn"`
}
AWSToJSON(&receiver, "dms", "create-replication-instance", "--replication-instance-identifier", random.Name(random.WithPrefix(envName)), "--replication-instance-class", "dms.t3.micro", "--region", region, "--replication-subnet-group-identifier", replicationSubnetGroupID)
awscli.AWSToJSON(&receiver, "dms", "create-replication-instance", "--replication-instance-identifier", random.Name(random.WithPrefix(envName)), "--replication-instance-class", "dms.t3.micro", "--region", region, "--replication-subnet-group-identifier", replicationSubnetGroupID)

return receiver.ARN
}
Expand All @@ -58,7 +59,7 @@ func getReplicationInstanceState(arn, region string) string {
var receiver struct {
StatusStrings []string `jsonry:"ReplicationInstances.ReplicationInstanceStatus"`
}
AWSToJSON(&receiver, "dms", "describe-replication-instances", "--region", region, "--filters", fmt.Sprintf("Name=replication-instance-arn,Values=%s", arn))
awscli.AWSToJSON(&receiver, "dms", "describe-replication-instances", "--region", region, "--filters", fmt.Sprintf("Name=replication-instance-arn,Values=%s", arn))
switch len(receiver.StatusStrings) {
default:
ginkgo.Fail("matched more than one instance")
Expand All @@ -71,7 +72,7 @@ func getReplicationInstanceState(arn, region string) string {
}

func deleteReplicationInstance(arn, region string) {
AWS("dms", "delete-replication-instance", "--replication-instance-arn", arn, "--region", region)
awscli.AWS("dms", "delete-replication-instance", "--replication-instance-arn", arn, "--region", region)

for start := time.Now(); time.Since(start) < time.Hour; {
if !replicationInstanceExists(arn, region) {
Expand All @@ -87,7 +88,7 @@ func replicationInstanceExists(arn, region string) bool {
var receiver struct {
ARNs []string `jsonry:"ReplicationInstances.ReplicationInstanceArn"`
}
AWSToJSON(&receiver, "dms", "describe-replication-instances", "--region", region)
awscli.AWSToJSON(&receiver, "dms", "describe-replication-instances", "--region", region)

for _, a := range receiver.ARNs {
if a == arn {
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package dms

import (
"csbbrokerpakaws/acceptance-tests/helpers/awscli"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"fmt"
)
Expand All @@ -13,7 +14,7 @@ type replicationSubnetGroup struct {
func createReplicationSubnetGroup(vpc, envName, region string) *replicationSubnetGroup {
subnets := listSubnets(vpc, region)
id := random.Name(random.WithPrefix(envName))
AWS(append([]string{"dms", "create-replication-subnet-group", "--region", region, "--replication-subnet-group-identifier", id, "--replication-subnet-group-description", id, "--subnet-ids"}, subnets...)...)
awscli.AWS(append([]string{"dms", "create-replication-subnet-group", "--region", region, "--replication-subnet-group-identifier", id, "--replication-subnet-group-description", id, "--subnet-ids"}, subnets...)...)

return &replicationSubnetGroup{
id: id,
Expand All @@ -22,13 +23,13 @@ func createReplicationSubnetGroup(vpc, envName, region string) *replicationSubne
}

func (r *replicationSubnetGroup) cleanup() {
AWS("dms", "delete-replication-subnet-group", "--replication-subnet-group-identifier", r.id, "--region", r.region)
awscli.AWS("dms", "delete-replication-subnet-group", "--replication-subnet-group-identifier", r.id, "--region", r.region)
}

func listSubnets(vpc, region string) []string {
var receiver struct {
Subnets []string `jsonry:"Subnets.SubnetId"`
}
AWSToJSON(&receiver, "ec2", "describe-subnets", "--filters", fmt.Sprintf("Name=vpc-id,Values=%s", vpc), "--region", region)
awscli.AWSToJSON(&receiver, "ec2", "describe-subnets", "--filters", fmt.Sprintf("Name=vpc-id,Values=%s", vpc), "--region", region)
return receiver.Subnets
}
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package dms

import (
"csbbrokerpakaws/acceptance-tests/helpers/awscli"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"fmt"
"time"
Expand All @@ -18,7 +19,7 @@ func RunReplicationTask(replicationInstance *ReplicationInstance, source, target
ARN string `jsonry:"ReplicationTask.ReplicationTaskArn"`
}

AWSToJSON(
awscli.AWSToJSON(
&taskReceiver,
"dms",
"create-replication-task",
Expand All @@ -35,7 +36,7 @@ func RunReplicationTask(replicationInstance *ReplicationInstance, source, target

waitForReplicationTaskCreation(taskID, region)

AWS(
awscli.AWS(
"dms",
"start-replication-task",
"--replication-task-arn", taskReceiver.ARN,
Expand All @@ -53,7 +54,7 @@ func waitForReplicationTaskCompletion(taskID, region string) {
Percentages []int `jsonry:"ReplicationTasks.ReplicationTaskStats.FullLoadProgressPercent"`
}

AWSToJSON(
awscli.AWSToJSON(
&statusReceiver,
"dms",
"describe-replication-tasks",
Expand Down Expand Up @@ -85,7 +86,7 @@ func waitForReplicationTaskCreation(taskID, region string) {
Status []string `jsonry:"ReplicationTasks.Status"`
}

AWSToJSON(
awscli.AWSToJSON(
&statusReceiver,
"dms",
"describe-replication-tasks",
Expand All @@ -109,7 +110,7 @@ func waitForReplicationTaskCreation(taskID, region string) {
}

func replicationTaskDeletion(taskARN, region string) {
AWS(
awscli.AWS(
"dms",
"delete-replication-task",
"--region", region,
Expand All @@ -130,7 +131,7 @@ func replicationTaskExists(arn, region string) bool {
var receiver struct {
ARNs []string `jsonry:"ReplicationTasks.ReplicationTaskArn"`
}
AWSToJSON(&receiver, "dms", "describe-replication-tasks", "--region", region)
awscli.AWSToJSON(&receiver, "dms", "describe-replication-tasks", "--region", region)

for _, a := range receiver.ARNs {
if a == arn {
Expand Down
115 changes: 115 additions & 0 deletions acceptance-tests/helpers/awscli/vpcendpoint/vpcendpoint.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,115 @@
// Package vpcendpoint provides test helpers for setting up an AWS VPC Endpoint
package vpcendpoint

import (
"csbbrokerpakaws/acceptance-tests/helpers/awscli"
"fmt"
)

type CallerIdentity struct {
Arn string `json:"Arn"`
}

type RouteTable struct {
RouteTableID string `json:"RouteTableId"`
}

type RouteTables struct {
RouteTables []RouteTable `json:"RouteTables"`
}

type VpcEndpoint struct {
VpcEndpointID string `json:"VpcEndpointId"`
}

type VpcEndpointResponse struct {
VpcEndpoint VpcEndpoint `json:"VpcEndpoint"`
}

func CreateEndpoint(allowedVPCID, defaultRegion string) string {

// Get the ARN of the current user
getCallerIdentityCommand := []string{
"sts",
"get-caller-identity",
"--output",
"json",
}

var callerIdentity CallerIdentity
awscli.AWSToJSON(&callerIdentity, getCallerIdentityCommand...)
allowedUserARN := callerIdentity.Arn

policyDocument := fmt.Sprintf(`{
"Statement": [
{
"Action": "*",
"Effect": "Allow",
"Resource": "*",
"Principal": "*",
"Condition": {
"StringEquals": {
"aws:sourceVpc": %[1]q
},
"StringLike": {
"aws:username": "csb-*"
}
}
},
{
"Action": "*",
"Effect": "Allow",
"Resource": "*",
"Principal": {
"AWS": %[2]q
},
"Condition": {
"StringEquals": {
"aws:sourceVpc": %[1]q
}
}
}
]
}`, allowedVPCID, allowedUserARN)

describeRoutesTablesCommand := []string{
"ec2",
"describe-route-tables",
"--filters",
"Name=vpc-id,Values=" + allowedVPCID,
"--output",
"json",
}

var routesTables RouteTables
awscli.AWSToJSON(&routesTables, describeRoutesTablesCommand...)

routeTableIDs := make([]string, len(routesTables.RouteTables))
for i, rt := range routesTables.RouteTables {
routeTableIDs[i] = rt.RouteTableID
}

createEndpointCommand := []string{
"ec2", "create-vpc-endpoint",
"--vpc-id", allowedVPCID,
"--service-name", fmt.Sprintf("com.amazonaws.%s.s3", defaultRegion),
"--vpc-endpoint-type", "Gateway",
"--route-table-ids",
}

createEndpointCommand = append(createEndpointCommand, routeTableIDs...)
createEndpointCommand = append(createEndpointCommand, []string{"--policy-document", policyDocument, "--output", "json"}...)

var response VpcEndpointResponse
awscli.AWSToJSON(&response, createEndpointCommand...)
return response.VpcEndpoint.VpcEndpointID
}

func DeleteVPCEndpoint(vpcEndpointID string) {
deleteEndpointCommand := []string{
"ec2", "delete-vpc-endpoints",
"--vpc-endpoint-ids", vpcEndpointID,
}

awscli.AWS(deleteEndpointCommand...)
}
2 changes: 1 addition & 1 deletion acceptance-tests/mssql_data_migration_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package acceptance_tests_test

import (
"csbbrokerpakaws/acceptance-tests/helpers/awscli/dms"
"fmt"
"os"
"os/exec"
Expand All @@ -9,7 +10,6 @@ import (
"github.com/onsi/gomega/gexec"

"csbbrokerpakaws/acceptance-tests/helpers/apps"
"csbbrokerpakaws/acceptance-tests/helpers/dms"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"csbbrokerpakaws/acceptance-tests/helpers/services"

Expand Down
2 changes: 1 addition & 1 deletion acceptance-tests/mysql_data_migration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package acceptance_tests_test

import (
"csbbrokerpakaws/acceptance-tests/helpers/apps"
"csbbrokerpakaws/acceptance-tests/helpers/dms"
"csbbrokerpakaws/acceptance-tests/helpers/awscli/dms"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"csbbrokerpakaws/acceptance-tests/helpers/services"

Expand Down
2 changes: 1 addition & 1 deletion acceptance-tests/postgresql_data_migration_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ package acceptance_tests_test

import (
"csbbrokerpakaws/acceptance-tests/helpers/apps"
"csbbrokerpakaws/acceptance-tests/helpers/dms"
"csbbrokerpakaws/acceptance-tests/helpers/awscli/dms"
"csbbrokerpakaws/acceptance-tests/helpers/random"
"csbbrokerpakaws/acceptance-tests/helpers/services"

Expand Down
Loading

0 comments on commit b099a19

Please sign in to comment.