From 9914aad87bb07e78053ac8e3bb5691af89b21613 Mon Sep 17 00:00:00 2001 From: Shawn Kaplan Date: Thu, 21 Dec 2023 16:38:25 -0800 Subject: [PATCH 1/5] Cross-Account e2e Tests + Conflicted Gateway Status --- Makefile | 16 +- docs/contributing/developer.md | 10 +- pkg/config/controller_config.go | 5 + pkg/controllers/gateway_controller.go | 47 ++- pkg/model/core/status_condition.go | 5 + pkg/utils/common.go | 9 + test/suites/sharing/ram_share_test.go | 433 ++++++++++++++++++++++++++ test/suites/sharing/suite_test.go | 89 ++++++ 8 files changed, 597 insertions(+), 17 deletions(-) create mode 100644 pkg/model/core/status_condition.go create mode 100644 test/suites/sharing/ram_share_test.go create mode 100644 test/suites/sharing/suite_test.go diff --git a/Makefile b/Makefile index e763ba3b..bf9c25e0 100644 --- a/Makefile +++ b/Makefile @@ -121,7 +121,21 @@ e2e-test: ## Run e2e tests against cluster pointed to by ~/.kube/config -count 1 \ -timeout 90m \ -v \ - ./suites/... \ + ./suites/integration/... \ + --ginkgo.focus="${FOCUS}" \ + --ginkgo.timeout=90m \ + --ginkgo.v + +.PHONY: sharing-test +sharing-test: ## Run e2e tests against cluster pointed to by ~/.kube/config and using secondary account role. Cluster VPC must not be associated to a Service Network. + @kubectl create namespace $(e2e-test-namespace) > /dev/null 2>&1 || true # ignore already exists error + LOG_LEVEL=debug + cd test && go test \ + -p 1 \ + -count 1 \ + -timeout 90m \ + -v \ + ./suites/sharing/... \ --ginkgo.focus="${FOCUS}" \ --ginkgo.timeout=90m \ --ginkgo.v diff --git a/docs/contributing/developer.md b/docs/contributing/developer.md index edeb82c3..21228bf6 100644 --- a/docs/contributing/developer.md +++ b/docs/contributing/developer.md @@ -121,12 +121,20 @@ And use "EnvFile" GoLand plugin to read the env variables from the generated `.e For larger changes it's recommended to run e2e suites on your local cluster. E2E tests require a service network named `test-gateway` with cluster VPC associated to run. -You can either setup service network manually or use DEFAULT_SERVICE_NETWORK option when running controller locally. (e.g. `DEFAULT_SERVICE_NETWORK=test-gateway make run`) +You can either set up service network manually or use DEFAULT_SERVICE_NETWORK option when running controller locally. (e.g. `DEFAULT_SERVICE_NETWORK=test-gateway make run`) ``` REGION=us-west-2 make e2e-test ``` +For the `sharing` test suite, which runs cross-account e2e tests, you will need a secondary account with a role that +can be assumed by the primary account during test execution. +You can create an IAM Role, with a Trust Policy allowing the primary account to assume it, via the AWS IAM Console. + +``` +REGION=us-west-2 export SECONDARY_ACCOUNT_TEST_ROLE_ARN=arn:aws:iam::000000000000:role/MyRole make sharing-test +``` + You can use `FOCUS` environment variable to run some specific test cases based on filter condition. You could assign the string in the Describe("xxxxxx") or It("xxxxxx") to the FOCUS environment variable to run the specific test cases. ```go diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index 07a53b28..cd8b8449 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -23,6 +23,7 @@ const ( CLUSTER_VPC_ID = "CLUSTER_VPC_ID" CLUSTER_NAME = "CLUSTER_NAME" DEFAULT_SERVICE_NETWORK = "DEFAULT_SERVICE_NETWORK" + SECONDARY_ACCOUNT_TEST_ROLE_ARN = "SECONDARY_ACCOUNT_TEST_ROLE_ARN" ENABLE_SERVICE_NETWORK_OVERRIDE = "ENABLE_SERVICE_NETWORK_OVERRIDE" AWS_ACCOUNT_ID = "AWS_ACCOUNT_ID" ) @@ -32,6 +33,7 @@ var AccountID = "" var Region = "" var DefaultServiceNetwork = "" var ClusterName = "" +var SecondaryAccountTestRoleArn = "" var ServiceNetworkOverrideMode = false @@ -85,6 +87,9 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { return fmt.Errorf("cannot get cluster name: %s", err) } + // SECONDARY_ACCOUNT_TEST_ROLE_ARN + SecondaryAccountTestRoleArn = os.Getenv(SECONDARY_ACCOUNT_TEST_ROLE_ARN) + return nil } diff --git a/pkg/controllers/gateway_controller.go b/pkg/controllers/gateway_controller.go index e92ec758..9a435790 100644 --- a/pkg/controllers/gateway_controller.go +++ b/pkg/controllers/gateway_controller.go @@ -19,6 +19,7 @@ package controllers import ( "context" "fmt" + "github.com/aws/aws-sdk-go/service/vpclattice" anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" "github.com/aws/aws-application-networking-k8s/pkg/controllers/eventhandlers" @@ -41,7 +42,6 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - "github.com/aws/aws-application-networking-k8s/pkg/aws/services" deploy "github.com/aws/aws-application-networking-k8s/pkg/deploy/lattice" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" pkg_builder "sigs.k8s.io/controller-runtime/pkg/builder" @@ -224,18 +224,35 @@ func (r *gatewayReconciler) reconcileUpsert(ctx context.Context, gw *gwv1beta1.G return err } - snInfo, err := r.cloud.Lattice().FindServiceNetwork(ctx, gw.Name) + foundSnArns := make([]string, 0, 1) + serviceNetworks, err := r.cloud.Lattice().ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{}) if err != nil { - if services.IsNotFoundError(err) { - if err = r.updateGatewayProgrammedStatus(ctx, "", gw, false); err != nil { - return lattice_runtime.NewRetryError() - } - return nil - } return err } + for _, sn := range serviceNetworks.Items { + if *sn.Id == gw.Name { + foundSnArns = []string{*sn.Arn} + break + } else if *sn.Name == gw.Name { + foundSnArns = append(foundSnArns, *sn.Arn) + } + } + + if len(foundSnArns) == 0 { + if err = r.updateGatewayProgrammedStatus(ctx, gw, gwv1.GatewayReasonPending, "VPC Lattice Service Network not found"); err != nil { + return lattice_runtime.NewRetryError() + } + return nil + } + + if len(foundSnArns) > 1 { + if err = r.updateGatewayProgrammedStatus(ctx, gw, core.GatewayReasonConflicted, "Multiple service networks with the same name were found. Either ensure only one service network with the same name is visible in the AWS account or use the desired service network's id as the Gateway name."); err != nil { + return lattice_runtime.NewRetryError() + } + return nil + } - if err = r.updateGatewayProgrammedStatus(ctx, *snInfo.SvcNetwork.Arn, gw, true); err != nil { + if err = r.updateGatewayProgrammedStatus(ctx, gw, gwv1.GatewayReasonProgrammed, fmt.Sprintf("aws-service-network-arn: %s", foundSnArns[0])); err != nil { return err } @@ -244,27 +261,27 @@ func (r *gatewayReconciler) reconcileUpsert(ctx context.Context, gw *gwv1beta1.G func (r *gatewayReconciler) updateGatewayProgrammedStatus( ctx context.Context, - snArn string, gw *gwv1beta1.Gateway, - programmed bool, + reason gwv1.GatewayConditionReason, + message string, ) error { gwOld := gw.DeepCopy() - if programmed { + if reason == gwv1.GatewayReasonProgrammed { gw.Status.Conditions = utils.GetNewConditions(gw.Status.Conditions, metav1.Condition{ Type: string(gwv1.GatewayConditionProgrammed), Status: metav1.ConditionTrue, ObservedGeneration: gw.Generation, Reason: string(gwv1.GatewayReasonProgrammed), - Message: fmt.Sprintf("aws-gateway-arn: %s", snArn), + Message: message, }) } else { gw.Status.Conditions = utils.GetNewConditions(gw.Status.Conditions, metav1.Condition{ Type: string(gwv1.GatewayConditionProgrammed), Status: metav1.ConditionFalse, ObservedGeneration: gw.Generation, - Reason: string(gwv1.GatewayReasonPending), - Message: "VPC Lattice Gateway not found", + Reason: string(reason), + Message: message, }) } diff --git a/pkg/model/core/status_condition.go b/pkg/model/core/status_condition.go new file mode 100644 index 00000000..3fa36881 --- /dev/null +++ b/pkg/model/core/status_condition.go @@ -0,0 +1,5 @@ +package core + +import gwv1 "sigs.k8s.io/gateway-api/apis/v1" + +const GatewayReasonConflicted gwv1.GatewayConditionReason = "Conflicted" diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 341c9ca2..9d79cc07 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -2,6 +2,7 @@ package utils import ( "fmt" + "math/rand" "strings" "golang.org/x/exp/constraints" @@ -85,6 +86,14 @@ func TargetRefToLatticeResourceName( return "", fmt.Errorf("unsupported targetRef Kind: %s", targetRef.Kind) } +func RandomString(length int) string { + randomSuffix := make([]rune, length) + for i := range randomSuffix { + randomSuffix[i] = rune(rand.Intn(26) + 'a') + } + return string(randomSuffix) +} + type none struct{} type Set[T comparable] struct { diff --git a/test/suites/sharing/ram_share_test.go b/test/suites/sharing/ram_share_test.go new file mode 100644 index 00000000..5c51c389 --- /dev/null +++ b/test/suites/sharing/ram_share_test.go @@ -0,0 +1,433 @@ +package sharing + +import ( + "fmt" + "github.com/aws/aws-application-networking-k8s/pkg/config" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + "github.com/aws/aws-application-networking-k8s/pkg/utils" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ram" + "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/aws/aws-sdk-go/service/vpclattice" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strings" + "time" +) + +var _ = Describe("RAM Share", Ordered, func() { + + const ( + k8sResourceNamePrefix = "k8s-test-ram-" + ) + + var ( + secondaryAccountId string + primaryRamClient *ram.RAM + secondaryRamClient *ram.RAM + primaryVpcLatticeClient *vpclattice.VPCLattice + secondaryVpcLatticeClient *vpclattice.VPCLattice + primaryResourceGroupsClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI + secondaryResourceGroupsClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI + ) + + BeforeAll(func() { + parts := strings.Split(config.SecondaryAccountTestRoleArn, ":") + Expect(len(parts)).To(BeEquivalentTo(6), "Invalid secondary account role arn") + secondaryAccountId = parts[4] + + primarySess := session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)})) + stsClient := sts.New(primarySess) + assumeRoleInput := &sts.AssumeRoleInput{ + RoleArn: aws.String(config.SecondaryAccountTestRoleArn), + RoleSessionName: aws.String("aws-application-networking-k8s-ram-e2e-test"), + } + + assumeRoleResult, err := stsClient.AssumeRoleWithContext(ctx, assumeRoleInput) + Expect(err).NotTo(HaveOccurred()) + + creds := assumeRoleResult.Credentials + + secondarySess := session.Must(session.NewSession(&aws.Config{ + Region: aws.String(config.Region), + Credentials: credentials.NewStaticCredentials( + *creds.AccessKeyId, + *creds.SecretAccessKey, + *creds.SessionToken, + ), + })) + + primaryRamClient = ram.New(primarySess) + secondaryRamClient = ram.New(secondarySess) + primaryVpcLatticeClient = vpclattice.New(primarySess) + secondaryVpcLatticeClient = vpclattice.New(secondarySess) + primaryResourceGroupsClient = resourcegroupstaggingapi.New(primarySess) + secondaryResourceGroupsClient = resourcegroupstaggingapi.New(secondarySess) + }) + + It("makes service network from secondary account usable for gateway with matching name", func() { + randomName := k8sResourceNamePrefix + utils.RandomString(10) + + // Create secondary account's service network using randomName + createSNInput := &vpclattice.CreateServiceNetworkInput{ + Name: aws.String(randomName), + Tags: k8sTestTags, + } + createSNResult, err := secondaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, createSNInput) + Expect(err).NotTo(HaveOccurred()) + + // Share service network to primary account using randomName + createShareInput := &ram.CreateResourceShareInput{ + Name: aws.String(randomName), + ResourceArns: []*string{createSNResult.Arn}, + Principals: []*string{aws.String(config.AccountID)}, + AllowExternalPrincipals: aws.Bool(true), + Tags: k8sRamTestTags, + } + _, err = secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) + Expect(err).To(BeNil()) + + // Wait for resource share invitation to appear in primary account + var invitation *ram.ResourceShareInvitation = nil + Eventually(func(g Gomega) { + listInvitationsInput := &ram.GetResourceShareInvitationsInput{} + listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) + Expect(err).NotTo(HaveOccurred()) + + for _, inv := range listInvitationsResult.ResourceShareInvitations { + if *inv.ResourceShareName == randomName && *inv.SenderAccountId == secondaryAccountId { + invitation = inv + break + } + } + g.Expect(invitation).ToNot(BeNil()) + }).Should(Succeed()) + + // Accept resource share invitation + acceptInput := &ram.AcceptResourceShareInvitationInput{ + ResourceShareInvitationArn: invitation.ResourceShareInvitationArn, + } + _, err = primaryRamClient.AcceptResourceShareInvitation(acceptInput) + Expect(err).ToNot(HaveOccurred()) + + // Wait for service network to appear in primary account + Eventually(func(g Gomega) { + getSNInput := &vpclattice.GetServiceNetworkInput{ + ServiceNetworkIdentifier: createSNResult.Id, + } + _, err = primaryVpcLatticeClient.GetServiceNetworkWithContext(ctx, getSNInput) + g.Expect(err).ToNot(HaveOccurred()) + }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) + + // Create gateway with same name as service network + gateway := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomName, + Namespace: k8snamespace, + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "amazon-vpc-lattice", + Listeners: []gwv1.Listener{ + { + Name: gwv1.SectionName(randomName), + Port: gwv1.PortNumber(80), + Protocol: gwv1.HTTPProtocolType, + }, + }, + }, + } + err = testFramework.Create(ctx, gateway) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + // Policy status should be Accepted + gwNamespacedName := types.NamespacedName{ + Name: gateway.Name, + Namespace: gateway.Namespace, + } + gw := &gwv1.Gateway{} + err := testFramework.Client.Get(ctx, gwNamespacedName, gw) + g.Expect(err).To(BeNil()) + g.Expect(len(gw.Status.Conditions)).To(BeEquivalentTo(2)) + g.Expect(gw.Status.Conditions[0].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionAccepted))) + g.Expect(gw.Status.Conditions[0].Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(gw.Status.Conditions[0].ObservedGeneration).To(BeEquivalentTo(1)) + g.Expect(gw.Status.Conditions[0].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonAccepted))) + g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) + g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(gw.Status.Conditions[1].ObservedGeneration).To(BeEquivalentTo(1)) + g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) + }).Should(Succeed()) + }) + + It("makes service network from secondary account usable for gateway with matching id", func() { + randomName := k8sResourceNamePrefix + utils.RandomString(10) + + // Create secondary account's service network using randomName + createSNInput := &vpclattice.CreateServiceNetworkInput{ + Name: aws.String(randomName), + Tags: k8sTestTags, + } + createSNResult, err := secondaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, createSNInput) + Expect(err).NotTo(HaveOccurred()) + + // Share service network to primary account using randomName + createShareInput := &ram.CreateResourceShareInput{ + Name: aws.String(randomName), + ResourceArns: []*string{createSNResult.Arn}, + Principals: []*string{aws.String(config.AccountID)}, + AllowExternalPrincipals: aws.Bool(true), + Tags: k8sRamTestTags, + } + _, err = secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) + Expect(err).To(BeNil()) + + // Wait for resource share invitation to appear in primary account + var invitation *ram.ResourceShareInvitation = nil + Eventually(func(g Gomega) { + listInvitationsInput := &ram.GetResourceShareInvitationsInput{} + listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) + Expect(err).NotTo(HaveOccurred()) + + for _, inv := range listInvitationsResult.ResourceShareInvitations { + if *inv.ResourceShareName == randomName && *inv.SenderAccountId == secondaryAccountId { + invitation = inv + break + } + } + g.Expect(invitation).ToNot(BeNil()) + }).Should(Succeed()) + + // Accept resource share invitation + acceptInput := &ram.AcceptResourceShareInvitationInput{ + ResourceShareInvitationArn: invitation.ResourceShareInvitationArn, + } + _, err = primaryRamClient.AcceptResourceShareInvitation(acceptInput) + Expect(err).ToNot(HaveOccurred()) + + // Wait for service network to appear in primary account + Eventually(func(g Gomega) { + getSNInput := &vpclattice.GetServiceNetworkInput{ + ServiceNetworkIdentifier: createSNResult.Id, + } + _, err = primaryVpcLatticeClient.GetServiceNetworkWithContext(ctx, getSNInput) + g.Expect(err).ToNot(HaveOccurred()) + }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) + + // Create gateway with same name as service network + gateway := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: *createSNResult.Id, + Namespace: k8snamespace, + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "amazon-vpc-lattice", + Listeners: []gwv1.Listener{ + { + Name: gwv1.SectionName(randomName), + Port: gwv1.PortNumber(80), + Protocol: gwv1.HTTPProtocolType, + }, + }, + }, + } + err = testFramework.Create(ctx, gateway) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + // Policy status should be Accepted + gwNamespacedName := types.NamespacedName{ + Name: gateway.Name, + Namespace: gateway.Namespace, + } + gw := &gwv1.Gateway{} + err := testFramework.Client.Get(ctx, gwNamespacedName, gw) + g.Expect(err).To(BeNil()) + g.Expect(len(gw.Status.Conditions)).To(BeEquivalentTo(2)) + g.Expect(gw.Status.Conditions[0].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionAccepted))) + g.Expect(gw.Status.Conditions[0].Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(gw.Status.Conditions[0].ObservedGeneration).To(BeEquivalentTo(1)) + g.Expect(gw.Status.Conditions[0].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonAccepted))) + g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) + g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(gw.Status.Conditions[1].ObservedGeneration).To(BeEquivalentTo(1)) + g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) + }).Should(Succeed()) + }) + + It("results in reconciliation failure when name conflicts with another resource", func() { + randomName := k8sResourceNamePrefix + utils.RandomString(10) + + // Create primary account's service network using randomName + pCreateSNInput := &vpclattice.CreateServiceNetworkInput{ + Name: aws.String(randomName), + Tags: k8sTestTags, + } + _, err := primaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, pCreateSNInput) + Expect(err).NotTo(HaveOccurred()) + + // Create secondary account's service network using randomName + sCreateSNInput := &vpclattice.CreateServiceNetworkInput{ + Name: aws.String(randomName), + Tags: k8sTestTags, + } + sCreateSNResult, err := secondaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, sCreateSNInput) + Expect(err).NotTo(HaveOccurred()) + + // Share secondary service network to primary account using randomName + createShareInput := &ram.CreateResourceShareInput{ + Name: aws.String(randomName), + ResourceArns: []*string{sCreateSNResult.Arn}, + Principals: []*string{aws.String(config.AccountID)}, + AllowExternalPrincipals: aws.Bool(true), + Tags: k8sRamTestTags, + } + _, err = secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) + Expect(err).To(BeNil()) + + // Wait for resource share invitation to appear in primary account + var invitation *ram.ResourceShareInvitation = nil + Eventually(func(g Gomega) { + listInvitationsInput := &ram.GetResourceShareInvitationsInput{} + listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) + Expect(err).NotTo(HaveOccurred()) + + for _, inv := range listInvitationsResult.ResourceShareInvitations { + if *inv.ResourceShareName == randomName && *inv.SenderAccountId == secondaryAccountId { + invitation = inv + break + } + } + g.Expect(invitation).ToNot(BeNil()) + }).Should(Succeed()) + + // Accept resource share invitation + acceptInput := &ram.AcceptResourceShareInvitationInput{ + ResourceShareInvitationArn: invitation.ResourceShareInvitationArn, + } + _, err = primaryRamClient.AcceptResourceShareInvitation(acceptInput) + Expect(err).ToNot(HaveOccurred()) + + // Wait for service network to appear in primary account + Eventually(func(g Gomega) { + getSNInput := &vpclattice.GetServiceNetworkInput{ + ServiceNetworkIdentifier: sCreateSNResult.Id, + } + _, err = primaryVpcLatticeClient.GetServiceNetworkWithContext(ctx, getSNInput) + g.Expect(err).ToNot(HaveOccurred()) + }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) + + // Create gateway with same name as service networks + gateway := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomName, + Namespace: k8snamespace, + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "amazon-vpc-lattice", + Listeners: []gwv1.Listener{ + { + Name: gwv1.SectionName(randomName), + Port: gwv1.PortNumber(80), + Protocol: gwv1.HTTPProtocolType, + }, + }, + }, + } + err = testFramework.Create(ctx, gateway) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + // Policy status should be Accepted + gwNamespacedName := types.NamespacedName{ + Name: gateway.Name, + Namespace: gateway.Namespace, + } + gw := &gwv1.Gateway{} + err := testFramework.Client.Get(ctx, gwNamespacedName, gw) + g.Expect(err).To(BeNil()) + g.Expect(len(gw.Status.Conditions)).To(BeEquivalentTo(2)) + g.Expect(gw.Status.Conditions[0].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionAccepted))) + g.Expect(gw.Status.Conditions[0].Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(gw.Status.Conditions[0].ObservedGeneration).To(BeEquivalentTo(1)) + g.Expect(gw.Status.Conditions[0].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonAccepted))) + g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) + g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionFalse)) + g.Expect(gw.Status.Conditions[1].ObservedGeneration).To(BeEquivalentTo(1)) + g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(core.GatewayReasonConflicted)) + }).Should(Succeed()) + }) + + AfterEach(func() { + // Delete gateways in test namespace except the framework gateway + gws := &gwv1.GatewayList{} + err := testFramework.Client.List(ctx, gws, client.InNamespace(k8snamespace)) + Expect(err).To(BeNil()) + for _, gw := range gws.Items { + if gw.Name != testGateway.Name { + testFramework.ExpectDeletedThenNotFound(ctx, &gw) + } + } + }) + + AfterAll(func() { + // Find all AWS resources created in tests + var tagFilters []*resourcegroupstaggingapi.TagFilter + for key, value := range k8sTestTags { + tagFilters = append(tagFilters, &resourcegroupstaggingapi.TagFilter{ + Key: aws.String(key), + Values: []*string{value}, + }) + } + getResourcesInput := &resourcegroupstaggingapi.GetResourcesInput{ + TagFilters: tagFilters, + } + + deleteTaggedResources := func( + getResourcesResult *resourcegroupstaggingapi.GetResourcesOutput, + vpcLatticeClient *vpclattice.VPCLattice, + ramClient *ram.RAM, + ) { + for _, mapping := range getResourcesResult.ResourceTagMappingList { + arn := mapping.ResourceARN + parts := strings.Split(*arn, ":") + Expect(len(parts)).To(BeEquivalentTo(6)) + + resourceType := strings.Split(parts[5], "/")[0] + switch resourceType { + case "servicenetwork": + deleteSnInput := &vpclattice.DeleteServiceNetworkInput{ + ServiceNetworkIdentifier: arn, + } + _, err := vpcLatticeClient.DeleteServiceNetworkWithContext(ctx, deleteSnInput) + Expect(err).ToNot(HaveOccurred()) + case "resource-share": + deleteShareInput := &ram.DeleteResourceShareInput{ + ResourceShareArn: arn, + } + _, err := ramClient.DeleteResourceShareWithContext(ctx, deleteShareInput) + Expect(err).ToNot(HaveOccurred()) + default: + Fail(fmt.Sprintf("Unknown resource type %s", resourceType)) + } + } + } + + // Delete secondary account AWS resources + secondaryGetResourcesResult, err := secondaryResourceGroupsClient.GetResources(getResourcesInput) + Expect(err).ToNot(HaveOccurred()) + deleteTaggedResources(secondaryGetResourcesResult, secondaryVpcLatticeClient, secondaryRamClient) + + // Delete primary account AWS resources + primaryGetResourcesResult, err := primaryResourceGroupsClient.GetResources(getResourcesInput) + Expect(err).ToNot(HaveOccurred()) + deleteTaggedResources(primaryGetResourcesResult, primaryVpcLatticeClient, primaryRamClient) + }) +}) diff --git a/test/suites/sharing/suite_test.go b/test/suites/sharing/suite_test.go new file mode 100644 index 00000000..918586fc --- /dev/null +++ b/test/suites/sharing/suite_test.go @@ -0,0 +1,89 @@ +package sharing + +import ( + "context" + anaws "github.com/aws/aws-application-networking-k8s/pkg/aws" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ram" + "os" + + "github.com/aws/aws-sdk-go/service/vpclattice" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "go.uber.org/zap" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/aws/aws-application-networking-k8s/test/pkg/test" + + "testing" + + gwv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +const ( + k8snamespace = "e2e-test" +) + +var k8sTestTags = map[string]*string{ + anaws.TagBase + "TestSuite": aws.String("sharing"), +} +var k8sRamTestTags []*ram.Tag +var testFramework *test.Framework +var ctx context.Context +var testGateway *gwv1.Gateway +var testServiceNetwork *vpclattice.ServiceNetworkSummary +var _ = SynchronizedBeforeSuite(func() { + for key, value := range k8sTestTags { + k8sRamTestTags = append(k8sRamTestTags, &ram.Tag{ + Key: aws.String(key), + Value: value, + }) + } + + vpcId := os.Getenv("CLUSTER_VPC_ID") + if vpcId == "" { + Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests") + } + + testFramework.ExpectToBeClean(ctx) + grpcurlRunnerPod := test.NewGrpcurlRunnerPod("grpc-runner", k8snamespace) + if err := testFramework.Get(ctx, client.ObjectKeyFromObject(grpcurlRunnerPod), testFramework.GrpcurlRunner); err != nil { + if apierrors.IsNotFound(err) { + testFramework.ExpectCreated(ctx, grpcurlRunnerPod) + testFramework.GrpcurlRunner = grpcurlRunnerPod + } + } + + // provision gateway, wait for service network association + testGateway = testFramework.NewGateway("test-gateway", k8snamespace) + testFramework.ExpectCreated(ctx, testGateway) + + testServiceNetwork = testFramework.GetServiceNetwork(ctx, testGateway) + + testFramework.Log.Infof("Expecting VPC %s and service network %s association", vpcId, *testServiceNetwork.Id) + Eventually(func(g Gomega) { + associated, snva, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcId, testServiceNetwork) + g.Expect(associated).To(BeTrue()) + managed, _ := testFramework.Cloud.IsArnManaged(ctx, *snva.Arn) + g.Expect(managed).To(BeTrue()) + }).Should(Succeed()) + +}, func() { + testGateway = testFramework.NewGateway("test-gateway", k8snamespace) + testServiceNetwork = testFramework.GetServiceNetwork(ctx, testGateway) + testFramework.GrpcurlRunner = test.NewGrpcurlRunnerPod("grpc-runner", k8snamespace) +}) + +func TestIntegration(t *testing.T) { + ctx = test.NewContext(t) + logger := gwlog.NewLogger(zap.DebugLevel) + testFramework = test.NewFramework(ctx, logger, k8snamespace) + RegisterFailHandler(Fail) + RunSpecs(t, "Sharing") +} + +var _ = SynchronizedAfterSuite(func() {}, func() { + testFramework.ExpectDeletedThenNotFound(ctx, testGateway, testFramework.GrpcurlRunner) +}) From dda87ff6789c965ca8612ff71a7468c06886994b Mon Sep 17 00:00:00 2001 From: Shawn Kaplan Date: Wed, 27 Dec 2023 15:14:20 -0800 Subject: [PATCH 2/5] Add ginkgo.skip option, reduce code repetition, set Invalid state during Gateway name conflict, general code cleanliness improvements --- Makefile | 17 +- docs/contributing/developer.md | 17 +- pkg/config/controller_config.go | 5 - pkg/controllers/gateway_controller.go | 41 +- pkg/model/core/status_condition.go | 5 - pkg/model/lattice/targetgroup.go | 8 +- pkg/utils/common.go | 10 +- test/suites/integration/ram_share_test.go | 353 ++++++++++++++++++ test/suites/integration/suite_test.go | 21 +- test/suites/sharing/ram_share_test.go | 433 ---------------------- test/suites/sharing/suite_test.go | 89 ----- 11 files changed, 408 insertions(+), 591 deletions(-) delete mode 100644 pkg/model/core/status_condition.go create mode 100644 test/suites/integration/ram_share_test.go delete mode 100644 test/suites/sharing/ram_share_test.go delete mode 100644 test/suites/sharing/suite_test.go diff --git a/Makefile b/Makefile index bf9c25e0..93da0688 100644 --- a/Makefile +++ b/Makefile @@ -123,22 +123,7 @@ e2e-test: ## Run e2e tests against cluster pointed to by ~/.kube/config -v \ ./suites/integration/... \ --ginkgo.focus="${FOCUS}" \ - --ginkgo.timeout=90m \ - --ginkgo.v - -.PHONY: sharing-test -sharing-test: ## Run e2e tests against cluster pointed to by ~/.kube/config and using secondary account role. Cluster VPC must not be associated to a Service Network. - @kubectl create namespace $(e2e-test-namespace) > /dev/null 2>&1 || true # ignore already exists error - LOG_LEVEL=debug - cd test && go test \ - -p 1 \ - -count 1 \ - -timeout 90m \ - -v \ - ./suites/sharing/... \ - --ginkgo.focus="${FOCUS}" \ - --ginkgo.timeout=90m \ - --ginkgo.v + --ginkgo.skip="${SKIP}" .SILENT: .PHONY: e2e-clean diff --git a/docs/contributing/developer.md b/docs/contributing/developer.md index 21228bf6..a4713119 100644 --- a/docs/contributing/developer.md +++ b/docs/contributing/developer.md @@ -127,15 +127,16 @@ You can either set up service network manually or use DEFAULT_SERVICE_NETWORK op REGION=us-west-2 make e2e-test ``` -For the `sharing` test suite, which runs cross-account e2e tests, you will need a secondary account with a role that +For the `RAM Share` test suite, which runs cross-account e2e tests, you will need a secondary account with a role that can be assumed by the primary account during test execution. You can create an IAM Role, with a Trust Policy allowing the primary account to assume it, via the AWS IAM Console. ``` -REGION=us-west-2 export SECONDARY_ACCOUNT_TEST_ROLE_ARN=arn:aws:iam::000000000000:role/MyRole make sharing-test +export SECONDARY_ACCOUNT_TEST_ROLE_ARN=arn:aws:iam::000000000000:role/MyRole +REGION=us-west-2 make e2e-test ``` -You can use `FOCUS` environment variable to run some specific test cases based on filter condition. +You can use the `FOCUS` environment variable to run some specific test cases based on filter condition. You could assign the string in the Describe("xxxxxx") or It("xxxxxx") to the FOCUS environment variable to run the specific test cases. ```go var _ = Describe("HTTPRoute path matches", func() { @@ -144,13 +145,19 @@ var _ = Describe("HTTPRoute path matches", func() { }) ``` -``` +For example, to run the test case "HTTPRoute should support multiple path matches", you could run the following command: +```sh export FOCUS="HTTPRoute should support multiple path matches" export REGION=us-west-2 make e2e-test ``` -For example, to run the test case "HTTPRoute should support multiple path matches", you could run the following command: +Conversely, you can use the `SKIP` environment variable to skip specific test cases. + +For example, to skip the same test as above, you would run the following command: +```sh +export SKIP="HTTPRoute should support multiple path matches" +``` For more detail on filter condition for ginkgo https://onsi.github.io/ginkgo/#focused-specs diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index cd8b8449..07a53b28 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -23,7 +23,6 @@ const ( CLUSTER_VPC_ID = "CLUSTER_VPC_ID" CLUSTER_NAME = "CLUSTER_NAME" DEFAULT_SERVICE_NETWORK = "DEFAULT_SERVICE_NETWORK" - SECONDARY_ACCOUNT_TEST_ROLE_ARN = "SECONDARY_ACCOUNT_TEST_ROLE_ARN" ENABLE_SERVICE_NETWORK_OVERRIDE = "ENABLE_SERVICE_NETWORK_OVERRIDE" AWS_ACCOUNT_ID = "AWS_ACCOUNT_ID" ) @@ -33,7 +32,6 @@ var AccountID = "" var Region = "" var DefaultServiceNetwork = "" var ClusterName = "" -var SecondaryAccountTestRoleArn = "" var ServiceNetworkOverrideMode = false @@ -87,9 +85,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { return fmt.Errorf("cannot get cluster name: %s", err) } - // SECONDARY_ACCOUNT_TEST_ROLE_ARN - SecondaryAccountTestRoleArn = os.Getenv(SECONDARY_ACCOUNT_TEST_ROLE_ARN) - return nil } diff --git a/pkg/controllers/gateway_controller.go b/pkg/controllers/gateway_controller.go index 9a435790..4518a2b0 100644 --- a/pkg/controllers/gateway_controller.go +++ b/pkg/controllers/gateway_controller.go @@ -19,9 +19,8 @@ package controllers import ( "context" "fmt" - "github.com/aws/aws-sdk-go/service/vpclattice" - anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" + "github.com/aws/aws-application-networking-k8s/pkg/aws/services" "github.com/aws/aws-application-networking-k8s/pkg/controllers/eventhandlers" "github.com/aws/aws-application-networking-k8s/pkg/aws" @@ -224,35 +223,25 @@ func (r *gatewayReconciler) reconcileUpsert(ctx context.Context, gw *gwv1beta1.G return err } - foundSnArns := make([]string, 0, 1) - serviceNetworks, err := r.cloud.Lattice().ListServiceNetworksWithContext(ctx, &vpclattice.ListServiceNetworksInput{}) + snInfo, err := r.cloud.Lattice().FindServiceNetwork(ctx, gw.Name) if err != nil { - return err - } - for _, sn := range serviceNetworks.Items { - if *sn.Id == gw.Name { - foundSnArns = []string{*sn.Arn} - break - } else if *sn.Name == gw.Name { - foundSnArns = append(foundSnArns, *sn.Arn) - } - } - - if len(foundSnArns) == 0 { - if err = r.updateGatewayProgrammedStatus(ctx, gw, gwv1.GatewayReasonPending, "VPC Lattice Service Network not found"); err != nil { - return lattice_runtime.NewRetryError() + if services.IsNotFoundError(err) { + if err = r.updateGatewayProgrammedStatus(ctx, gw, gwv1.GatewayReasonPending, "VPC Lattice Service Network not found"); err != nil { + return lattice_runtime.NewRetryError() + } + return nil } - return nil - } - - if len(foundSnArns) > 1 { - if err = r.updateGatewayProgrammedStatus(ctx, gw, core.GatewayReasonConflicted, "Multiple service networks with the same name were found. Either ensure only one service network with the same name is visible in the AWS account or use the desired service network's id as the Gateway name."); err != nil { - return lattice_runtime.NewRetryError() + if errors.Is(err, services.ErrNameConflict) { + if err = r.updateGatewayProgrammedStatus(ctx, gw, gwv1.GatewayReasonInvalid, "Found multiple VPC Lattice Service Networks matching Gateway name. Either ensure only one Service Network has a matching name, or use the Service Network's id as the Gateway name."); err != nil { + return lattice_runtime.NewRetryError() + } + return nil } - return nil + return err } - if err = r.updateGatewayProgrammedStatus(ctx, gw, gwv1.GatewayReasonProgrammed, fmt.Sprintf("aws-service-network-arn: %s", foundSnArns[0])); err != nil { + err = r.updateGatewayProgrammedStatus(ctx, gw, gwv1.GatewayReasonProgrammed, fmt.Sprintf("aws-service-network-arn: %s", *snInfo.SvcNetwork.Arn)) + if err != nil { return err } diff --git a/pkg/model/core/status_condition.go b/pkg/model/core/status_condition.go deleted file mode 100644 index 3fa36881..00000000 --- a/pkg/model/core/status_condition.go +++ /dev/null @@ -1,5 +0,0 @@ -package core - -import gwv1 "sigs.k8s.io/gateway-api/apis/v1" - -const GatewayReasonConflicted gwv1.GatewayConditionReason = "Conflicted" diff --git a/pkg/model/lattice/targetgroup.go b/pkg/model/lattice/targetgroup.go index fe0186fa..fccd801c 100644 --- a/pkg/model/lattice/targetgroup.go +++ b/pkg/model/lattice/targetgroup.go @@ -7,7 +7,6 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/model/core" "github.com/aws/aws-application-networking-k8s/pkg/utils" "github.com/aws/aws-sdk-go/service/vpclattice" - "math/rand" ) const ( @@ -190,9 +189,6 @@ func TgNamePrefix(spec TargetGroupSpec) string { func GenerateTgName(spec TargetGroupSpec) string { // tg max name length 128 prefix := TgNamePrefix(spec) - randomSuffix := make([]rune, RandomSuffixLength) - for i := range randomSuffix { - randomSuffix[i] = rune(rand.Intn(26) + 'a') - } - return fmt.Sprintf("%s-%s", prefix, string(randomSuffix)) + randomSuffix := utils.RandomAlphaString(RandomSuffixLength) + return fmt.Sprintf("%s-%s", prefix, randomSuffix) } diff --git a/pkg/utils/common.go b/pkg/utils/common.go index 9d79cc07..eda54083 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -86,12 +86,12 @@ func TargetRefToLatticeResourceName( return "", fmt.Errorf("unsupported targetRef Kind: %s", targetRef.Kind) } -func RandomString(length int) string { - randomSuffix := make([]rune, length) - for i := range randomSuffix { - randomSuffix[i] = rune(rand.Intn(26) + 'a') +func RandomAlphaString(length int) string { + str := make([]rune, length) + for i := range str { + str[i] = rune(rand.Intn(26) + 'a') } - return string(randomSuffix) + return string(str) } type none struct{} diff --git a/test/suites/integration/ram_share_test.go b/test/suites/integration/ram_share_test.go new file mode 100644 index 00000000..f5309d09 --- /dev/null +++ b/test/suites/integration/ram_share_test.go @@ -0,0 +1,353 @@ +package integration + +import ( + "fmt" + "github.com/aws/aws-application-networking-k8s/pkg/config" + "github.com/aws/aws-application-networking-k8s/pkg/utils" + "github.com/aws/aws-sdk-go/aws" + arn2 "github.com/aws/aws-sdk-go/aws/arn" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ram" + "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" + "github.com/aws/aws-sdk-go/service/sts" + "github.com/aws/aws-sdk-go/service/vpclattice" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1 "sigs.k8s.io/gateway-api/apis/v1" + "strings" + "time" +) + +var _ = Describe("RAM Share", Ordered, func() { + + const ( + k8sResourceNamePrefix = "k8s-test-ram-" + ) + + var ( + secondaryAccountId string + primaryRamClient *ram.RAM + secondaryRamClient *ram.RAM + primaryVpcLatticeClient *vpclattice.VPCLattice + secondaryVpcLatticeClient *vpclattice.VPCLattice + primaryResourceGroupsClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI + secondaryResourceGroupsClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI + ) + + BeforeAll(func() { + parts := strings.Split(SecondaryAccountTestRoleArn, ":") + Expect(len(parts)).To(BeEquivalentTo(6), "Invalid secondary account role arn") + secondaryAccountId = parts[4] + + primarySess := session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)})) + stsClient := sts.New(primarySess) + assumeRoleInput := &sts.AssumeRoleInput{ + RoleArn: aws.String(SecondaryAccountTestRoleArn), + RoleSessionName: aws.String("aws-application-networking-k8s-ram-e2e-test"), + } + + assumeRoleResult, err := stsClient.AssumeRoleWithContext(ctx, assumeRoleInput) + Expect(err).NotTo(HaveOccurred()) + + creds := assumeRoleResult.Credentials + + secondarySess := session.Must(session.NewSession(&aws.Config{ + Region: aws.String(config.Region), + Credentials: credentials.NewStaticCredentials( + *creds.AccessKeyId, + *creds.SecretAccessKey, + *creds.SessionToken, + ), + })) + + primaryRamClient = ram.New(primarySess) + secondaryRamClient = ram.New(secondarySess) + primaryVpcLatticeClient = vpclattice.New(primarySess) + secondaryVpcLatticeClient = vpclattice.New(secondarySess) + primaryResourceGroupsClient = resourcegroupstaggingapi.New(primarySess) + secondaryResourceGroupsClient = resourcegroupstaggingapi.New(secondarySess) + }) + + It("makes service network from secondary account usable for gateway with matching name", func() { + randomName := k8sResourceNamePrefix + utils.RandomAlphaString(10) + createSharedServiceNetwork( + randomName, + secondaryAccountId, + primaryVpcLatticeClient, + secondaryVpcLatticeClient, + primaryRamClient, + secondaryRamClient, + ) + + // Create gateway with same name as service network + gateway := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomName, + Namespace: k8snamespace, + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "amazon-vpc-lattice", + Listeners: []gwv1.Listener{ + { + Name: gwv1.SectionName(randomName), + Port: gwv1.PortNumber(80), + Protocol: gwv1.HTTPProtocolType, + }, + }, + }, + } + err := testFramework.Create(ctx, gateway) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + // Policy status should be Accepted + gwNamespacedName := types.NamespacedName{ + Name: gateway.Name, + Namespace: gateway.Namespace, + } + gw := &gwv1.Gateway{} + err := testFramework.Client.Get(ctx, gwNamespacedName, gw) + g.Expect(err).To(BeNil()) + g.Expect(len(gw.Status.Conditions) >= 2).To(BeTrue()) + g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) + g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) + }).Should(Succeed()) + }) + + It("makes service network from secondary account usable for gateway with matching id", func() { + randomName := k8sResourceNamePrefix + utils.RandomAlphaString(10) + + serviceNetwork := createSharedServiceNetwork( + randomName, + secondaryAccountId, + primaryVpcLatticeClient, + secondaryVpcLatticeClient, + primaryRamClient, + secondaryRamClient, + ) + + // Create gateway with service network id as name + gateway := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: *serviceNetwork.Id, + Namespace: k8snamespace, + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "amazon-vpc-lattice", + Listeners: []gwv1.Listener{ + { + Name: gwv1.SectionName(randomName), + Port: gwv1.PortNumber(80), + Protocol: gwv1.HTTPProtocolType, + }, + }, + }, + } + err := testFramework.Create(ctx, gateway) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + // Policy status should be Accepted + gwNamespacedName := types.NamespacedName{ + Name: gateway.Name, + Namespace: gateway.Namespace, + } + gw := &gwv1.Gateway{} + err := testFramework.Client.Get(ctx, gwNamespacedName, gw) + g.Expect(err).To(BeNil()) + g.Expect(len(gw.Status.Conditions) >= 2).To(BeTrue()) + g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) + g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) + }).Should(Succeed()) + }) + + It("results in reconciliation failure when name conflicts with another resource", func() { + randomName := k8sResourceNamePrefix + utils.RandomAlphaString(10) + + // Create primary account's service network using randomName + createSNInput := &vpclattice.CreateServiceNetworkInput{ + Name: aws.String(randomName), + Tags: k8sTestTags, + } + _, err := primaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, createSNInput) + Expect(err).NotTo(HaveOccurred()) + + createSharedServiceNetwork( + randomName, + secondaryAccountId, + primaryVpcLatticeClient, + secondaryVpcLatticeClient, + primaryRamClient, + secondaryRamClient, + ) + + // Create gateway with same name as service networks + gateway := &gwv1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: randomName, + Namespace: k8snamespace, + }, + Spec: gwv1.GatewaySpec{ + GatewayClassName: "amazon-vpc-lattice", + Listeners: []gwv1.Listener{ + { + Name: gwv1.SectionName(randomName), + Port: gwv1.PortNumber(80), + Protocol: gwv1.HTTPProtocolType, + }, + }, + }, + } + err = testFramework.Create(ctx, gateway) + Expect(err).ToNot(HaveOccurred()) + + Eventually(func(g Gomega) { + // Policy status should be Accepted + gwNamespacedName := types.NamespacedName{ + Name: gateway.Name, + Namespace: gateway.Namespace, + } + gw := &gwv1.Gateway{} + err := testFramework.Client.Get(ctx, gwNamespacedName, gw) + g.Expect(err).To(BeNil()) + g.Expect(len(gw.Status.Conditions) >= 2).To(BeTrue()) + g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) + g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionFalse)) + g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(gwv1.GatewayReasonInvalid)) + }).Should(Succeed()) + }) + + AfterEach(func() { + // Delete gateways in test namespace except the framework gateway + gws := &gwv1.GatewayList{} + err := testFramework.Client.List(ctx, gws, client.InNamespace(k8snamespace)) + Expect(err).To(BeNil()) + for _, gw := range gws.Items { + if gw.Name != testGateway.Name { + testFramework.ExpectDeletedThenNotFound(ctx, &gw) + } + } + }) + + AfterAll(func() { + // Find all AWS resources created in tests + var tagFilters []*resourcegroupstaggingapi.TagFilter + for key, value := range k8sTestTags { + tagFilters = append(tagFilters, &resourcegroupstaggingapi.TagFilter{ + Key: aws.String(key), + Values: []*string{value}, + }) + } + getResourcesInput := &resourcegroupstaggingapi.GetResourcesInput{ + TagFilters: tagFilters, + } + + deleteTaggedResources := func( + getResourcesResult *resourcegroupstaggingapi.GetResourcesOutput, + vpcLatticeClient *vpclattice.VPCLattice, + ramClient *ram.RAM, + ) { + for _, mapping := range getResourcesResult.ResourceTagMappingList { + arn, err := arn2.Parse(*mapping.ResourceARN) + Expect(err).ToNot(HaveOccurred()) + + resourceType := strings.Split(arn.Resource, "/")[0] + switch resourceType { + case "servicenetwork": + deleteSnInput := &vpclattice.DeleteServiceNetworkInput{ + ServiceNetworkIdentifier: aws.String(arn.String()), + } + _, err := vpcLatticeClient.DeleteServiceNetworkWithContext(ctx, deleteSnInput) + Expect(err).ToNot(HaveOccurred()) + case "resource-share": + deleteShareInput := &ram.DeleteResourceShareInput{ + ResourceShareArn: aws.String(arn.String()), + } + _, err := ramClient.DeleteResourceShareWithContext(ctx, deleteShareInput) + Expect(err).ToNot(HaveOccurred()) + default: + Fail(fmt.Sprintf("Unknown resource type %s", resourceType)) + } + } + } + + // Delete secondary account AWS resources + secondaryGetResourcesResult, err := secondaryResourceGroupsClient.GetResources(getResourcesInput) + Expect(err).ToNot(HaveOccurred()) + deleteTaggedResources(secondaryGetResourcesResult, secondaryVpcLatticeClient, secondaryRamClient) + + // Delete primary account AWS resources + primaryGetResourcesResult, err := primaryResourceGroupsClient.GetResources(getResourcesInput) + Expect(err).ToNot(HaveOccurred()) + deleteTaggedResources(primaryGetResourcesResult, primaryVpcLatticeClient, primaryRamClient) + }) +}) + +func createSharedServiceNetwork( + serviceNetworkName string, + secondaryAccountId string, + primaryVpcLatticeClient *vpclattice.VPCLattice, + secondaryVpcLatticeClient *vpclattice.VPCLattice, + primaryRamClient *ram.RAM, + secondaryRamClient *ram.RAM, +) *vpclattice.GetServiceNetworkOutput { + // Create secondary account's service network using randomName + createSNInput := &vpclattice.CreateServiceNetworkInput{ + Name: aws.String(serviceNetworkName), + Tags: k8sTestTags, + } + createSNResult, err := secondaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, createSNInput) + Expect(err).NotTo(HaveOccurred()) + + // Share service network to primary account using randomName + createShareInput := &ram.CreateResourceShareInput{ + Name: aws.String(serviceNetworkName), + ResourceArns: []*string{createSNResult.Arn}, + Principals: []*string{aws.String(config.AccountID)}, + AllowExternalPrincipals: aws.Bool(true), + Tags: k8sRamTestTags, + } + _, err = secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) + Expect(err).To(BeNil()) + + // Wait for resource share invitation to appear in primary account + var invitation *ram.ResourceShareInvitation = nil + Eventually(func(g Gomega) { + listInvitationsInput := &ram.GetResourceShareInvitationsInput{} + listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) + Expect(err).NotTo(HaveOccurred()) + + for _, inv := range listInvitationsResult.ResourceShareInvitations { + if *inv.ResourceShareName == serviceNetworkName && *inv.SenderAccountId == secondaryAccountId { + invitation = inv + break + } + } + g.Expect(invitation).ToNot(BeNil()) + }).Should(Succeed()) + + // Accept resource share invitation + acceptInput := &ram.AcceptResourceShareInvitationInput{ + ResourceShareInvitationArn: invitation.ResourceShareInvitationArn, + } + _, err = primaryRamClient.AcceptResourceShareInvitation(acceptInput) + Expect(err).ToNot(HaveOccurred()) + + // Wait for service network to appear in primary account + var sn *vpclattice.GetServiceNetworkOutput + Eventually(func(g Gomega) { + getSNInput := &vpclattice.GetServiceNetworkInput{ + ServiceNetworkIdentifier: createSNResult.Id, + } + sn, err = primaryVpcLatticeClient.GetServiceNetworkWithContext(ctx, getSNInput) + g.Expect(err).ToNot(HaveOccurred()) + }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) + + return sn +} diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go index d39f6c30..83fb8e93 100644 --- a/test/suites/integration/suite_test.go +++ b/test/suites/integration/suite_test.go @@ -2,6 +2,9 @@ package integration import ( "context" + anaws "github.com/aws/aws-application-networking-k8s/pkg/aws" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/ram" "os" "github.com/aws/aws-sdk-go/service/vpclattice" @@ -20,14 +23,30 @@ import ( ) const ( - k8snamespace = "e2e-test" + k8snamespace = "e2e-test" + secondaryAccountTestRoleArn = "SECONDARY_ACCOUNT_TEST_ROLE_ARN" ) +var k8sTestTags = map[string]*string{ + anaws.TagBase + "TestSuite": aws.String("sharing"), +} +var k8sRamTestTags []*ram.Tag var testFramework *test.Framework var ctx context.Context var testGateway *gwv1.Gateway var testServiceNetwork *vpclattice.ServiceNetworkSummary +var SecondaryAccountTestRoleArn string + var _ = SynchronizedBeforeSuite(func() { + for key, value := range k8sTestTags { + k8sRamTestTags = append(k8sRamTestTags, &ram.Tag{ + Key: aws.String(key), + Value: value, + }) + } + + SecondaryAccountTestRoleArn = os.Getenv(secondaryAccountTestRoleArn) + vpcId := os.Getenv("CLUSTER_VPC_ID") if vpcId == "" { Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests") diff --git a/test/suites/sharing/ram_share_test.go b/test/suites/sharing/ram_share_test.go deleted file mode 100644 index 5c51c389..00000000 --- a/test/suites/sharing/ram_share_test.go +++ /dev/null @@ -1,433 +0,0 @@ -package sharing - -import ( - "fmt" - "github.com/aws/aws-application-networking-k8s/pkg/config" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - "github.com/aws/aws-application-networking-k8s/pkg/utils" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/ram" - "github.com/aws/aws-sdk-go/service/resourcegroupstaggingapi" - "github.com/aws/aws-sdk-go/service/sts" - "github.com/aws/aws-sdk-go/service/vpclattice" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - gwv1 "sigs.k8s.io/gateway-api/apis/v1" - "strings" - "time" -) - -var _ = Describe("RAM Share", Ordered, func() { - - const ( - k8sResourceNamePrefix = "k8s-test-ram-" - ) - - var ( - secondaryAccountId string - primaryRamClient *ram.RAM - secondaryRamClient *ram.RAM - primaryVpcLatticeClient *vpclattice.VPCLattice - secondaryVpcLatticeClient *vpclattice.VPCLattice - primaryResourceGroupsClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI - secondaryResourceGroupsClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI - ) - - BeforeAll(func() { - parts := strings.Split(config.SecondaryAccountTestRoleArn, ":") - Expect(len(parts)).To(BeEquivalentTo(6), "Invalid secondary account role arn") - secondaryAccountId = parts[4] - - primarySess := session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)})) - stsClient := sts.New(primarySess) - assumeRoleInput := &sts.AssumeRoleInput{ - RoleArn: aws.String(config.SecondaryAccountTestRoleArn), - RoleSessionName: aws.String("aws-application-networking-k8s-ram-e2e-test"), - } - - assumeRoleResult, err := stsClient.AssumeRoleWithContext(ctx, assumeRoleInput) - Expect(err).NotTo(HaveOccurred()) - - creds := assumeRoleResult.Credentials - - secondarySess := session.Must(session.NewSession(&aws.Config{ - Region: aws.String(config.Region), - Credentials: credentials.NewStaticCredentials( - *creds.AccessKeyId, - *creds.SecretAccessKey, - *creds.SessionToken, - ), - })) - - primaryRamClient = ram.New(primarySess) - secondaryRamClient = ram.New(secondarySess) - primaryVpcLatticeClient = vpclattice.New(primarySess) - secondaryVpcLatticeClient = vpclattice.New(secondarySess) - primaryResourceGroupsClient = resourcegroupstaggingapi.New(primarySess) - secondaryResourceGroupsClient = resourcegroupstaggingapi.New(secondarySess) - }) - - It("makes service network from secondary account usable for gateway with matching name", func() { - randomName := k8sResourceNamePrefix + utils.RandomString(10) - - // Create secondary account's service network using randomName - createSNInput := &vpclattice.CreateServiceNetworkInput{ - Name: aws.String(randomName), - Tags: k8sTestTags, - } - createSNResult, err := secondaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, createSNInput) - Expect(err).NotTo(HaveOccurred()) - - // Share service network to primary account using randomName - createShareInput := &ram.CreateResourceShareInput{ - Name: aws.String(randomName), - ResourceArns: []*string{createSNResult.Arn}, - Principals: []*string{aws.String(config.AccountID)}, - AllowExternalPrincipals: aws.Bool(true), - Tags: k8sRamTestTags, - } - _, err = secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) - Expect(err).To(BeNil()) - - // Wait for resource share invitation to appear in primary account - var invitation *ram.ResourceShareInvitation = nil - Eventually(func(g Gomega) { - listInvitationsInput := &ram.GetResourceShareInvitationsInput{} - listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) - Expect(err).NotTo(HaveOccurred()) - - for _, inv := range listInvitationsResult.ResourceShareInvitations { - if *inv.ResourceShareName == randomName && *inv.SenderAccountId == secondaryAccountId { - invitation = inv - break - } - } - g.Expect(invitation).ToNot(BeNil()) - }).Should(Succeed()) - - // Accept resource share invitation - acceptInput := &ram.AcceptResourceShareInvitationInput{ - ResourceShareInvitationArn: invitation.ResourceShareInvitationArn, - } - _, err = primaryRamClient.AcceptResourceShareInvitation(acceptInput) - Expect(err).ToNot(HaveOccurred()) - - // Wait for service network to appear in primary account - Eventually(func(g Gomega) { - getSNInput := &vpclattice.GetServiceNetworkInput{ - ServiceNetworkIdentifier: createSNResult.Id, - } - _, err = primaryVpcLatticeClient.GetServiceNetworkWithContext(ctx, getSNInput) - g.Expect(err).ToNot(HaveOccurred()) - }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) - - // Create gateway with same name as service network - gateway := &gwv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: randomName, - Namespace: k8snamespace, - }, - Spec: gwv1.GatewaySpec{ - GatewayClassName: "amazon-vpc-lattice", - Listeners: []gwv1.Listener{ - { - Name: gwv1.SectionName(randomName), - Port: gwv1.PortNumber(80), - Protocol: gwv1.HTTPProtocolType, - }, - }, - }, - } - err = testFramework.Create(ctx, gateway) - Expect(err).ToNot(HaveOccurred()) - - Eventually(func(g Gomega) { - // Policy status should be Accepted - gwNamespacedName := types.NamespacedName{ - Name: gateway.Name, - Namespace: gateway.Namespace, - } - gw := &gwv1.Gateway{} - err := testFramework.Client.Get(ctx, gwNamespacedName, gw) - g.Expect(err).To(BeNil()) - g.Expect(len(gw.Status.Conditions)).To(BeEquivalentTo(2)) - g.Expect(gw.Status.Conditions[0].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionAccepted))) - g.Expect(gw.Status.Conditions[0].Status).To(BeEquivalentTo(metav1.ConditionTrue)) - g.Expect(gw.Status.Conditions[0].ObservedGeneration).To(BeEquivalentTo(1)) - g.Expect(gw.Status.Conditions[0].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonAccepted))) - g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) - g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionTrue)) - g.Expect(gw.Status.Conditions[1].ObservedGeneration).To(BeEquivalentTo(1)) - g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) - }).Should(Succeed()) - }) - - It("makes service network from secondary account usable for gateway with matching id", func() { - randomName := k8sResourceNamePrefix + utils.RandomString(10) - - // Create secondary account's service network using randomName - createSNInput := &vpclattice.CreateServiceNetworkInput{ - Name: aws.String(randomName), - Tags: k8sTestTags, - } - createSNResult, err := secondaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, createSNInput) - Expect(err).NotTo(HaveOccurred()) - - // Share service network to primary account using randomName - createShareInput := &ram.CreateResourceShareInput{ - Name: aws.String(randomName), - ResourceArns: []*string{createSNResult.Arn}, - Principals: []*string{aws.String(config.AccountID)}, - AllowExternalPrincipals: aws.Bool(true), - Tags: k8sRamTestTags, - } - _, err = secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) - Expect(err).To(BeNil()) - - // Wait for resource share invitation to appear in primary account - var invitation *ram.ResourceShareInvitation = nil - Eventually(func(g Gomega) { - listInvitationsInput := &ram.GetResourceShareInvitationsInput{} - listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) - Expect(err).NotTo(HaveOccurred()) - - for _, inv := range listInvitationsResult.ResourceShareInvitations { - if *inv.ResourceShareName == randomName && *inv.SenderAccountId == secondaryAccountId { - invitation = inv - break - } - } - g.Expect(invitation).ToNot(BeNil()) - }).Should(Succeed()) - - // Accept resource share invitation - acceptInput := &ram.AcceptResourceShareInvitationInput{ - ResourceShareInvitationArn: invitation.ResourceShareInvitationArn, - } - _, err = primaryRamClient.AcceptResourceShareInvitation(acceptInput) - Expect(err).ToNot(HaveOccurred()) - - // Wait for service network to appear in primary account - Eventually(func(g Gomega) { - getSNInput := &vpclattice.GetServiceNetworkInput{ - ServiceNetworkIdentifier: createSNResult.Id, - } - _, err = primaryVpcLatticeClient.GetServiceNetworkWithContext(ctx, getSNInput) - g.Expect(err).ToNot(HaveOccurred()) - }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) - - // Create gateway with same name as service network - gateway := &gwv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: *createSNResult.Id, - Namespace: k8snamespace, - }, - Spec: gwv1.GatewaySpec{ - GatewayClassName: "amazon-vpc-lattice", - Listeners: []gwv1.Listener{ - { - Name: gwv1.SectionName(randomName), - Port: gwv1.PortNumber(80), - Protocol: gwv1.HTTPProtocolType, - }, - }, - }, - } - err = testFramework.Create(ctx, gateway) - Expect(err).ToNot(HaveOccurred()) - - Eventually(func(g Gomega) { - // Policy status should be Accepted - gwNamespacedName := types.NamespacedName{ - Name: gateway.Name, - Namespace: gateway.Namespace, - } - gw := &gwv1.Gateway{} - err := testFramework.Client.Get(ctx, gwNamespacedName, gw) - g.Expect(err).To(BeNil()) - g.Expect(len(gw.Status.Conditions)).To(BeEquivalentTo(2)) - g.Expect(gw.Status.Conditions[0].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionAccepted))) - g.Expect(gw.Status.Conditions[0].Status).To(BeEquivalentTo(metav1.ConditionTrue)) - g.Expect(gw.Status.Conditions[0].ObservedGeneration).To(BeEquivalentTo(1)) - g.Expect(gw.Status.Conditions[0].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonAccepted))) - g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) - g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionTrue)) - g.Expect(gw.Status.Conditions[1].ObservedGeneration).To(BeEquivalentTo(1)) - g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) - }).Should(Succeed()) - }) - - It("results in reconciliation failure when name conflicts with another resource", func() { - randomName := k8sResourceNamePrefix + utils.RandomString(10) - - // Create primary account's service network using randomName - pCreateSNInput := &vpclattice.CreateServiceNetworkInput{ - Name: aws.String(randomName), - Tags: k8sTestTags, - } - _, err := primaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, pCreateSNInput) - Expect(err).NotTo(HaveOccurred()) - - // Create secondary account's service network using randomName - sCreateSNInput := &vpclattice.CreateServiceNetworkInput{ - Name: aws.String(randomName), - Tags: k8sTestTags, - } - sCreateSNResult, err := secondaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, sCreateSNInput) - Expect(err).NotTo(HaveOccurred()) - - // Share secondary service network to primary account using randomName - createShareInput := &ram.CreateResourceShareInput{ - Name: aws.String(randomName), - ResourceArns: []*string{sCreateSNResult.Arn}, - Principals: []*string{aws.String(config.AccountID)}, - AllowExternalPrincipals: aws.Bool(true), - Tags: k8sRamTestTags, - } - _, err = secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) - Expect(err).To(BeNil()) - - // Wait for resource share invitation to appear in primary account - var invitation *ram.ResourceShareInvitation = nil - Eventually(func(g Gomega) { - listInvitationsInput := &ram.GetResourceShareInvitationsInput{} - listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) - Expect(err).NotTo(HaveOccurred()) - - for _, inv := range listInvitationsResult.ResourceShareInvitations { - if *inv.ResourceShareName == randomName && *inv.SenderAccountId == secondaryAccountId { - invitation = inv - break - } - } - g.Expect(invitation).ToNot(BeNil()) - }).Should(Succeed()) - - // Accept resource share invitation - acceptInput := &ram.AcceptResourceShareInvitationInput{ - ResourceShareInvitationArn: invitation.ResourceShareInvitationArn, - } - _, err = primaryRamClient.AcceptResourceShareInvitation(acceptInput) - Expect(err).ToNot(HaveOccurred()) - - // Wait for service network to appear in primary account - Eventually(func(g Gomega) { - getSNInput := &vpclattice.GetServiceNetworkInput{ - ServiceNetworkIdentifier: sCreateSNResult.Id, - } - _, err = primaryVpcLatticeClient.GetServiceNetworkWithContext(ctx, getSNInput) - g.Expect(err).ToNot(HaveOccurred()) - }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) - - // Create gateway with same name as service networks - gateway := &gwv1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: randomName, - Namespace: k8snamespace, - }, - Spec: gwv1.GatewaySpec{ - GatewayClassName: "amazon-vpc-lattice", - Listeners: []gwv1.Listener{ - { - Name: gwv1.SectionName(randomName), - Port: gwv1.PortNumber(80), - Protocol: gwv1.HTTPProtocolType, - }, - }, - }, - } - err = testFramework.Create(ctx, gateway) - Expect(err).ToNot(HaveOccurred()) - - Eventually(func(g Gomega) { - // Policy status should be Accepted - gwNamespacedName := types.NamespacedName{ - Name: gateway.Name, - Namespace: gateway.Namespace, - } - gw := &gwv1.Gateway{} - err := testFramework.Client.Get(ctx, gwNamespacedName, gw) - g.Expect(err).To(BeNil()) - g.Expect(len(gw.Status.Conditions)).To(BeEquivalentTo(2)) - g.Expect(gw.Status.Conditions[0].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionAccepted))) - g.Expect(gw.Status.Conditions[0].Status).To(BeEquivalentTo(metav1.ConditionTrue)) - g.Expect(gw.Status.Conditions[0].ObservedGeneration).To(BeEquivalentTo(1)) - g.Expect(gw.Status.Conditions[0].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonAccepted))) - g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) - g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionFalse)) - g.Expect(gw.Status.Conditions[1].ObservedGeneration).To(BeEquivalentTo(1)) - g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(core.GatewayReasonConflicted)) - }).Should(Succeed()) - }) - - AfterEach(func() { - // Delete gateways in test namespace except the framework gateway - gws := &gwv1.GatewayList{} - err := testFramework.Client.List(ctx, gws, client.InNamespace(k8snamespace)) - Expect(err).To(BeNil()) - for _, gw := range gws.Items { - if gw.Name != testGateway.Name { - testFramework.ExpectDeletedThenNotFound(ctx, &gw) - } - } - }) - - AfterAll(func() { - // Find all AWS resources created in tests - var tagFilters []*resourcegroupstaggingapi.TagFilter - for key, value := range k8sTestTags { - tagFilters = append(tagFilters, &resourcegroupstaggingapi.TagFilter{ - Key: aws.String(key), - Values: []*string{value}, - }) - } - getResourcesInput := &resourcegroupstaggingapi.GetResourcesInput{ - TagFilters: tagFilters, - } - - deleteTaggedResources := func( - getResourcesResult *resourcegroupstaggingapi.GetResourcesOutput, - vpcLatticeClient *vpclattice.VPCLattice, - ramClient *ram.RAM, - ) { - for _, mapping := range getResourcesResult.ResourceTagMappingList { - arn := mapping.ResourceARN - parts := strings.Split(*arn, ":") - Expect(len(parts)).To(BeEquivalentTo(6)) - - resourceType := strings.Split(parts[5], "/")[0] - switch resourceType { - case "servicenetwork": - deleteSnInput := &vpclattice.DeleteServiceNetworkInput{ - ServiceNetworkIdentifier: arn, - } - _, err := vpcLatticeClient.DeleteServiceNetworkWithContext(ctx, deleteSnInput) - Expect(err).ToNot(HaveOccurred()) - case "resource-share": - deleteShareInput := &ram.DeleteResourceShareInput{ - ResourceShareArn: arn, - } - _, err := ramClient.DeleteResourceShareWithContext(ctx, deleteShareInput) - Expect(err).ToNot(HaveOccurred()) - default: - Fail(fmt.Sprintf("Unknown resource type %s", resourceType)) - } - } - } - - // Delete secondary account AWS resources - secondaryGetResourcesResult, err := secondaryResourceGroupsClient.GetResources(getResourcesInput) - Expect(err).ToNot(HaveOccurred()) - deleteTaggedResources(secondaryGetResourcesResult, secondaryVpcLatticeClient, secondaryRamClient) - - // Delete primary account AWS resources - primaryGetResourcesResult, err := primaryResourceGroupsClient.GetResources(getResourcesInput) - Expect(err).ToNot(HaveOccurred()) - deleteTaggedResources(primaryGetResourcesResult, primaryVpcLatticeClient, primaryRamClient) - }) -}) diff --git a/test/suites/sharing/suite_test.go b/test/suites/sharing/suite_test.go deleted file mode 100644 index 918586fc..00000000 --- a/test/suites/sharing/suite_test.go +++ /dev/null @@ -1,89 +0,0 @@ -package sharing - -import ( - "context" - anaws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ram" - "os" - - "github.com/aws/aws-sdk-go/service/vpclattice" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" - "go.uber.org/zap" - apierrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" - - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - "github.com/aws/aws-application-networking-k8s/test/pkg/test" - - "testing" - - gwv1 "sigs.k8s.io/gateway-api/apis/v1" -) - -const ( - k8snamespace = "e2e-test" -) - -var k8sTestTags = map[string]*string{ - anaws.TagBase + "TestSuite": aws.String("sharing"), -} -var k8sRamTestTags []*ram.Tag -var testFramework *test.Framework -var ctx context.Context -var testGateway *gwv1.Gateway -var testServiceNetwork *vpclattice.ServiceNetworkSummary -var _ = SynchronizedBeforeSuite(func() { - for key, value := range k8sTestTags { - k8sRamTestTags = append(k8sRamTestTags, &ram.Tag{ - Key: aws.String(key), - Value: value, - }) - } - - vpcId := os.Getenv("CLUSTER_VPC_ID") - if vpcId == "" { - Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests") - } - - testFramework.ExpectToBeClean(ctx) - grpcurlRunnerPod := test.NewGrpcurlRunnerPod("grpc-runner", k8snamespace) - if err := testFramework.Get(ctx, client.ObjectKeyFromObject(grpcurlRunnerPod), testFramework.GrpcurlRunner); err != nil { - if apierrors.IsNotFound(err) { - testFramework.ExpectCreated(ctx, grpcurlRunnerPod) - testFramework.GrpcurlRunner = grpcurlRunnerPod - } - } - - // provision gateway, wait for service network association - testGateway = testFramework.NewGateway("test-gateway", k8snamespace) - testFramework.ExpectCreated(ctx, testGateway) - - testServiceNetwork = testFramework.GetServiceNetwork(ctx, testGateway) - - testFramework.Log.Infof("Expecting VPC %s and service network %s association", vpcId, *testServiceNetwork.Id) - Eventually(func(g Gomega) { - associated, snva, _ := testFramework.IsVpcAssociatedWithServiceNetwork(ctx, vpcId, testServiceNetwork) - g.Expect(associated).To(BeTrue()) - managed, _ := testFramework.Cloud.IsArnManaged(ctx, *snva.Arn) - g.Expect(managed).To(BeTrue()) - }).Should(Succeed()) - -}, func() { - testGateway = testFramework.NewGateway("test-gateway", k8snamespace) - testServiceNetwork = testFramework.GetServiceNetwork(ctx, testGateway) - testFramework.GrpcurlRunner = test.NewGrpcurlRunnerPod("grpc-runner", k8snamespace) -}) - -func TestIntegration(t *testing.T) { - ctx = test.NewContext(t) - logger := gwlog.NewLogger(zap.DebugLevel) - testFramework = test.NewFramework(ctx, logger, k8snamespace) - RegisterFailHandler(Fail) - RunSpecs(t, "Sharing") -} - -var _ = SynchronizedAfterSuite(func() {}, func() { - testFramework.ExpectDeletedThenNotFound(ctx, testGateway, testFramework.GrpcurlRunner) -}) From b907de6b8a648fb02ce94d8e2827fb3f06ceff99 Mon Sep 17 00:00:00 2001 From: Shawn Kaplan Date: Thu, 28 Dec 2023 10:42:05 -0800 Subject: [PATCH 3/5] Change test suite tag value and optimize ram share cleanup --- test/suites/integration/ram_share_test.go | 13 ++++++------- test/suites/integration/suite_test.go | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/test/suites/integration/ram_share_test.go b/test/suites/integration/ram_share_test.go index f5309d09..6cd1fdd7 100644 --- a/test/suites/integration/ram_share_test.go +++ b/test/suites/integration/ram_share_test.go @@ -313,21 +313,20 @@ func createSharedServiceNetwork( AllowExternalPrincipals: aws.Bool(true), Tags: k8sRamTestTags, } - _, err = secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) + createShareResult, err := secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) Expect(err).To(BeNil()) // Wait for resource share invitation to appear in primary account var invitation *ram.ResourceShareInvitation = nil Eventually(func(g Gomega) { - listInvitationsInput := &ram.GetResourceShareInvitationsInput{} + listInvitationsInput := &ram.GetResourceShareInvitationsInput{ + ResourceShareArns: []*string{createShareResult.ResourceShare.ResourceShareArn}, + } listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) Expect(err).NotTo(HaveOccurred()) - for _, inv := range listInvitationsResult.ResourceShareInvitations { - if *inv.ResourceShareName == serviceNetworkName && *inv.SenderAccountId == secondaryAccountId { - invitation = inv - break - } + if len(listInvitationsResult.ResourceShareInvitations) > 0 { + invitation = listInvitationsResult.ResourceShareInvitations[0] } g.Expect(invitation).ToNot(BeNil()) }).Should(Succeed()) diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go index 83fb8e93..d137034a 100644 --- a/test/suites/integration/suite_test.go +++ b/test/suites/integration/suite_test.go @@ -28,7 +28,7 @@ const ( ) var k8sTestTags = map[string]*string{ - anaws.TagBase + "TestSuite": aws.String("sharing"), + anaws.TagBase + "TestSuite": aws.String(k8snamespace), } var k8sRamTestTags []*ram.Tag var testFramework *test.Framework From 101fe25120d5ebbe5c749dd279e43f2f0babfc82 Mon Sep 17 00:00:00 2001 From: Shawn Kaplan Date: Thu, 28 Dec 2023 19:26:59 -0800 Subject: [PATCH 4/5] Address PR comments --- test/suites/integration/ram_share_test.go | 122 +++++++++------------- test/suites/integration/suite_test.go | 6 +- 2 files changed, 51 insertions(+), 77 deletions(-) diff --git a/test/suites/integration/ram_share_test.go b/test/suites/integration/ram_share_test.go index 6cd1fdd7..c050f3fe 100644 --- a/test/suites/integration/ram_share_test.go +++ b/test/suites/integration/ram_share_test.go @@ -14,14 +14,25 @@ import ( "github.com/aws/aws-sdk-go/service/vpclattice" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "os" "sigs.k8s.io/controller-runtime/pkg/client" gwv1 "sigs.k8s.io/gateway-api/apis/v1" "strings" "time" ) +type testClients struct { + ramClient1 *ram.RAM + ramClient2 *ram.RAM + latticeClient1 *vpclattice.VPCLattice + latticeClient2 *vpclattice.VPCLattice + rgClient1 *resourcegroupstaggingapi.ResourceGroupsTaggingAPI + rgClient2 *resourcegroupstaggingapi.ResourceGroupsTaggingAPI +} + var _ = Describe("RAM Share", Ordered, func() { const ( @@ -29,24 +40,17 @@ var _ = Describe("RAM Share", Ordered, func() { ) var ( - secondaryAccountId string - primaryRamClient *ram.RAM - secondaryRamClient *ram.RAM - primaryVpcLatticeClient *vpclattice.VPCLattice - secondaryVpcLatticeClient *vpclattice.VPCLattice - primaryResourceGroupsClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI - secondaryResourceGroupsClient *resourcegroupstaggingapi.ResourceGroupsTaggingAPI + clients *testClients + secondaryTestRoleArn string ) BeforeAll(func() { - parts := strings.Split(SecondaryAccountTestRoleArn, ":") - Expect(len(parts)).To(BeEquivalentTo(6), "Invalid secondary account role arn") - secondaryAccountId = parts[4] + secondaryTestRoleArn = os.Getenv("SECONDARY_ACCOUNT_TEST_ROLE_ARN") primarySess := session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)})) stsClient := sts.New(primarySess) assumeRoleInput := &sts.AssumeRoleInput{ - RoleArn: aws.String(SecondaryAccountTestRoleArn), + RoleArn: aws.String(secondaryTestRoleArn), RoleSessionName: aws.String("aws-application-networking-k8s-ram-e2e-test"), } @@ -64,24 +68,19 @@ var _ = Describe("RAM Share", Ordered, func() { ), })) - primaryRamClient = ram.New(primarySess) - secondaryRamClient = ram.New(secondarySess) - primaryVpcLatticeClient = vpclattice.New(primarySess) - secondaryVpcLatticeClient = vpclattice.New(secondarySess) - primaryResourceGroupsClient = resourcegroupstaggingapi.New(primarySess) - secondaryResourceGroupsClient = resourcegroupstaggingapi.New(secondarySess) + clients = &testClients{ + ramClient1: ram.New(primarySess), + ramClient2: ram.New(secondarySess), + latticeClient1: vpclattice.New(primarySess), + latticeClient2: vpclattice.New(secondarySess), + rgClient1: resourcegroupstaggingapi.New(primarySess), + rgClient2: resourcegroupstaggingapi.New(secondarySess), + } }) It("makes service network from secondary account usable for gateway with matching name", func() { randomName := k8sResourceNamePrefix + utils.RandomAlphaString(10) - createSharedServiceNetwork( - randomName, - secondaryAccountId, - primaryVpcLatticeClient, - secondaryVpcLatticeClient, - primaryRamClient, - secondaryRamClient, - ) + createSharedServiceNetwork(randomName, clients) // Create gateway with same name as service network gateway := &gwv1.Gateway{ @@ -112,24 +111,17 @@ var _ = Describe("RAM Share", Ordered, func() { gw := &gwv1.Gateway{} err := testFramework.Client.Get(ctx, gwNamespacedName, gw) g.Expect(err).To(BeNil()) - g.Expect(len(gw.Status.Conditions) >= 2).To(BeTrue()) - g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) - g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionTrue)) - g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) + programmedCondition := meta.FindStatusCondition(gw.Status.Conditions, string(gwv1.GatewayConditionProgrammed)) + g.Expect(programmedCondition).ToNot(BeNil()) + g.Expect(programmedCondition.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(programmedCondition.Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) }).Should(Succeed()) }) It("makes service network from secondary account usable for gateway with matching id", func() { randomName := k8sResourceNamePrefix + utils.RandomAlphaString(10) - serviceNetwork := createSharedServiceNetwork( - randomName, - secondaryAccountId, - primaryVpcLatticeClient, - secondaryVpcLatticeClient, - primaryRamClient, - secondaryRamClient, - ) + serviceNetwork := createSharedServiceNetwork(randomName, clients) // Create gateway with service network id as name gateway := &gwv1.Gateway{ @@ -160,10 +152,10 @@ var _ = Describe("RAM Share", Ordered, func() { gw := &gwv1.Gateway{} err := testFramework.Client.Get(ctx, gwNamespacedName, gw) g.Expect(err).To(BeNil()) - g.Expect(len(gw.Status.Conditions) >= 2).To(BeTrue()) - g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) - g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionTrue)) - g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) + programmedCondition := meta.FindStatusCondition(gw.Status.Conditions, string(gwv1.GatewayConditionProgrammed)) + g.Expect(programmedCondition).ToNot(BeNil()) + g.Expect(programmedCondition.Status).To(BeEquivalentTo(metav1.ConditionTrue)) + g.Expect(programmedCondition.Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonProgrammed))) }).Should(Succeed()) }) @@ -175,17 +167,10 @@ var _ = Describe("RAM Share", Ordered, func() { Name: aws.String(randomName), Tags: k8sTestTags, } - _, err := primaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, createSNInput) + _, err := clients.latticeClient1.CreateServiceNetworkWithContext(ctx, createSNInput) Expect(err).NotTo(HaveOccurred()) - createSharedServiceNetwork( - randomName, - secondaryAccountId, - primaryVpcLatticeClient, - secondaryVpcLatticeClient, - primaryRamClient, - secondaryRamClient, - ) + createSharedServiceNetwork(randomName, clients) // Create gateway with same name as service networks gateway := &gwv1.Gateway{ @@ -216,10 +201,10 @@ var _ = Describe("RAM Share", Ordered, func() { gw := &gwv1.Gateway{} err := testFramework.Client.Get(ctx, gwNamespacedName, gw) g.Expect(err).To(BeNil()) - g.Expect(len(gw.Status.Conditions) >= 2).To(BeTrue()) - g.Expect(gw.Status.Conditions[1].Type).To(BeEquivalentTo(string(gwv1.GatewayConditionProgrammed))) - g.Expect(gw.Status.Conditions[1].Status).To(BeEquivalentTo(metav1.ConditionFalse)) - g.Expect(gw.Status.Conditions[1].Reason).To(BeEquivalentTo(gwv1.GatewayReasonInvalid)) + programmedCondition := meta.FindStatusCondition(gw.Status.Conditions, string(gwv1.GatewayConditionProgrammed)) + g.Expect(programmedCondition).ToNot(BeNil()) + g.Expect(programmedCondition.Status).To(BeEquivalentTo(metav1.ConditionFalse)) + g.Expect(programmedCondition.Reason).To(BeEquivalentTo(string(gwv1.GatewayReasonInvalid))) }).Should(Succeed()) }) @@ -278,31 +263,24 @@ var _ = Describe("RAM Share", Ordered, func() { } // Delete secondary account AWS resources - secondaryGetResourcesResult, err := secondaryResourceGroupsClient.GetResources(getResourcesInput) + secondaryGetResourcesResult, err := clients.rgClient2.GetResources(getResourcesInput) Expect(err).ToNot(HaveOccurred()) - deleteTaggedResources(secondaryGetResourcesResult, secondaryVpcLatticeClient, secondaryRamClient) + deleteTaggedResources(secondaryGetResourcesResult, clients.latticeClient2, clients.ramClient2) // Delete primary account AWS resources - primaryGetResourcesResult, err := primaryResourceGroupsClient.GetResources(getResourcesInput) + primaryGetResourcesResult, err := clients.rgClient1.GetResources(getResourcesInput) Expect(err).ToNot(HaveOccurred()) - deleteTaggedResources(primaryGetResourcesResult, primaryVpcLatticeClient, primaryRamClient) + deleteTaggedResources(primaryGetResourcesResult, clients.latticeClient1, clients.ramClient1) }) }) -func createSharedServiceNetwork( - serviceNetworkName string, - secondaryAccountId string, - primaryVpcLatticeClient *vpclattice.VPCLattice, - secondaryVpcLatticeClient *vpclattice.VPCLattice, - primaryRamClient *ram.RAM, - secondaryRamClient *ram.RAM, -) *vpclattice.GetServiceNetworkOutput { +func createSharedServiceNetwork(serviceNetworkName string, clients *testClients) *vpclattice.GetServiceNetworkOutput { // Create secondary account's service network using randomName createSNInput := &vpclattice.CreateServiceNetworkInput{ Name: aws.String(serviceNetworkName), Tags: k8sTestTags, } - createSNResult, err := secondaryVpcLatticeClient.CreateServiceNetworkWithContext(ctx, createSNInput) + createSNResult, err := clients.latticeClient2.CreateServiceNetworkWithContext(ctx, createSNInput) Expect(err).NotTo(HaveOccurred()) // Share service network to primary account using randomName @@ -313,7 +291,7 @@ func createSharedServiceNetwork( AllowExternalPrincipals: aws.Bool(true), Tags: k8sRamTestTags, } - createShareResult, err := secondaryRamClient.CreateResourceShareWithContext(ctx, createShareInput) + createShareResult, err := clients.ramClient2.CreateResourceShareWithContext(ctx, createShareInput) Expect(err).To(BeNil()) // Wait for resource share invitation to appear in primary account @@ -322,20 +300,20 @@ func createSharedServiceNetwork( listInvitationsInput := &ram.GetResourceShareInvitationsInput{ ResourceShareArns: []*string{createShareResult.ResourceShare.ResourceShareArn}, } - listInvitationsResult, err := primaryRamClient.GetResourceShareInvitations(listInvitationsInput) + listInvitationsResult, err := clients.ramClient1.GetResourceShareInvitations(listInvitationsInput) Expect(err).NotTo(HaveOccurred()) if len(listInvitationsResult.ResourceShareInvitations) > 0 { invitation = listInvitationsResult.ResourceShareInvitations[0] } g.Expect(invitation).ToNot(BeNil()) - }).Should(Succeed()) + }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) // Accept resource share invitation acceptInput := &ram.AcceptResourceShareInvitationInput{ ResourceShareInvitationArn: invitation.ResourceShareInvitationArn, } - _, err = primaryRamClient.AcceptResourceShareInvitation(acceptInput) + _, err = clients.ramClient1.AcceptResourceShareInvitation(acceptInput) Expect(err).ToNot(HaveOccurred()) // Wait for service network to appear in primary account @@ -344,7 +322,7 @@ func createSharedServiceNetwork( getSNInput := &vpclattice.GetServiceNetworkInput{ ServiceNetworkIdentifier: createSNResult.Id, } - sn, err = primaryVpcLatticeClient.GetServiceNetworkWithContext(ctx, getSNInput) + sn, err = clients.latticeClient1.GetServiceNetworkWithContext(ctx, getSNInput) g.Expect(err).ToNot(HaveOccurred()) }).WithTimeout(5 * time.Minute).WithPolling(5 * time.Second).Should(Succeed()) diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go index d137034a..1037ba55 100644 --- a/test/suites/integration/suite_test.go +++ b/test/suites/integration/suite_test.go @@ -23,8 +23,7 @@ import ( ) const ( - k8snamespace = "e2e-test" - secondaryAccountTestRoleArn = "SECONDARY_ACCOUNT_TEST_ROLE_ARN" + k8snamespace = "e2e-test" ) var k8sTestTags = map[string]*string{ @@ -35,7 +34,6 @@ var testFramework *test.Framework var ctx context.Context var testGateway *gwv1.Gateway var testServiceNetwork *vpclattice.ServiceNetworkSummary -var SecondaryAccountTestRoleArn string var _ = SynchronizedBeforeSuite(func() { for key, value := range k8sTestTags { @@ -45,8 +43,6 @@ var _ = SynchronizedBeforeSuite(func() { }) } - SecondaryAccountTestRoleArn = os.Getenv(secondaryAccountTestRoleArn) - vpcId := os.Getenv("CLUSTER_VPC_ID") if vpcId == "" { Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests") From 293e6375e8c91d78e54fa92891c150ef85f4a431 Mon Sep 17 00:00:00 2001 From: Shawn Kaplan Date: Fri, 29 Dec 2023 11:20:37 -0800 Subject: [PATCH 5/5] Removed global mutable test tags --- test/pkg/test/framework.go | 18 ++++++++++++++---- test/suites/integration/ram_share_test.go | 18 ++++++++++++++---- test/suites/integration/suite_test.go | 16 +--------------- 3 files changed, 29 insertions(+), 23 deletions(-) diff --git a/test/pkg/test/framework.go b/test/pkg/test/framework.go index 7e3cb750..9c2bbf94 100644 --- a/test/pkg/test/framework.go +++ b/test/pkg/test/framework.go @@ -8,7 +8,7 @@ import ( "strings" "time" - an_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" + anaws "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-sdk-go/service/ec2" "github.com/onsi/gomega/format" @@ -50,6 +50,10 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/aws/services" ) +const ( + K8sNamespace = "e2e-test" +) + type TestObject struct { Type client.Object ListType client.ObjectList @@ -111,14 +115,14 @@ type Framework struct { Ec2Client *ec2.EC2 GrpcurlRunner *corev1.Pod DefaultTags services.Tags - Cloud an_aws.Cloud + Cloud anaws.Cloud } func NewFramework(ctx context.Context, log gwlog.Logger, testNamespace string) *Framework { addOptionalCRDs(testScheme) config.ConfigInit() controllerRuntimeConfig := controllerruntime.GetConfigOrDie() - cloudConfig := an_aws.CloudConfig{ + cloudConfig := anaws.CloudConfig{ VpcId: config.VpcID, AccountId: config.AccountID, Region: config.Region, @@ -137,7 +141,7 @@ func NewFramework(ctx context.Context, log gwlog.Logger, testNamespace string) * namespace: testNamespace, controllerRuntimeConfig: controllerRuntimeConfig, } - framework.Cloud = an_aws.NewDefaultCloud(framework.LatticeClient, cloudConfig) + framework.Cloud = anaws.NewDefaultCloud(framework.LatticeClient, cloudConfig) framework.DefaultTags = framework.Cloud.DefaultTags() SetDefaultEventuallyTimeout(3 * time.Minute) SetDefaultEventuallyPollingInterval(10 * time.Second) @@ -611,3 +615,9 @@ func (env *Framework) RunGrpcurlCmd(opts RunGrpcurlCmdOptions) (string, string, func (env *Framework) SleepForRouteUpdate() { time.Sleep(10 * time.Second) } + +func (env *Framework) NewTestTags() map[string]*string { + return env.Cloud.DefaultTagsMergedWith(map[string]*string{ + anaws.TagBase + "TestSuite": aws.String(K8sNamespace), + }) +} diff --git a/test/suites/integration/ram_share_test.go b/test/suites/integration/ram_share_test.go index c050f3fe..5c24466e 100644 --- a/test/suites/integration/ram_share_test.go +++ b/test/suites/integration/ram_share_test.go @@ -165,7 +165,7 @@ var _ = Describe("RAM Share", Ordered, func() { // Create primary account's service network using randomName createSNInput := &vpclattice.CreateServiceNetworkInput{ Name: aws.String(randomName), - Tags: k8sTestTags, + Tags: testFramework.NewTestTags(), } _, err := clients.latticeClient1.CreateServiceNetworkWithContext(ctx, createSNInput) Expect(err).NotTo(HaveOccurred()) @@ -223,7 +223,7 @@ var _ = Describe("RAM Share", Ordered, func() { AfterAll(func() { // Find all AWS resources created in tests var tagFilters []*resourcegroupstaggingapi.TagFilter - for key, value := range k8sTestTags { + for key, value := range testFramework.NewTestTags() { tagFilters = append(tagFilters, &resourcegroupstaggingapi.TagFilter{ Key: aws.String(key), Values: []*string{value}, @@ -275,14 +275,24 @@ var _ = Describe("RAM Share", Ordered, func() { }) func createSharedServiceNetwork(serviceNetworkName string, clients *testClients) *vpclattice.GetServiceNetworkOutput { - // Create secondary account's service network using randomName + tags := testFramework.NewTestTags() + + // Create secondary account's service network createSNInput := &vpclattice.CreateServiceNetworkInput{ Name: aws.String(serviceNetworkName), - Tags: k8sTestTags, + Tags: tags, } createSNResult, err := clients.latticeClient2.CreateServiceNetworkWithContext(ctx, createSNInput) Expect(err).NotTo(HaveOccurred()) + var k8sRamTestTags []*ram.Tag + for key, value := range tags { + k8sRamTestTags = append(k8sRamTestTags, &ram.Tag{ + Key: aws.String(key), + Value: value, + }) + } + // Share service network to primary account using randomName createShareInput := &ram.CreateResourceShareInput{ Name: aws.String(serviceNetworkName), diff --git a/test/suites/integration/suite_test.go b/test/suites/integration/suite_test.go index 1037ba55..d18980b5 100644 --- a/test/suites/integration/suite_test.go +++ b/test/suites/integration/suite_test.go @@ -2,9 +2,6 @@ package integration import ( "context" - anaws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/ram" "os" "github.com/aws/aws-sdk-go/service/vpclattice" @@ -23,26 +20,15 @@ import ( ) const ( - k8snamespace = "e2e-test" + k8snamespace = test.K8sNamespace ) -var k8sTestTags = map[string]*string{ - anaws.TagBase + "TestSuite": aws.String(k8snamespace), -} -var k8sRamTestTags []*ram.Tag var testFramework *test.Framework var ctx context.Context var testGateway *gwv1.Gateway var testServiceNetwork *vpclattice.ServiceNetworkSummary var _ = SynchronizedBeforeSuite(func() { - for key, value := range k8sTestTags { - k8sRamTestTags = append(k8sRamTestTags, &ram.Tag{ - Key: aws.String(key), - Value: value, - }) - } - vpcId := os.Getenv("CLUSTER_VPC_ID") if vpcId == "" { Fail("CLUSTER_VPC_ID environment variable must be set to run integration tests")