diff --git a/pkg/aws/services/vpclattice.go b/pkg/aws/services/vpclattice.go index 2fe4252c..1acba627 100644 --- a/pkg/aws/services/vpclattice.go +++ b/pkg/aws/services/vpclattice.go @@ -2,13 +2,12 @@ package services import ( "context" - "github.com/golang/glog" - "os" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/vpclattice" "github.com/aws/aws-sdk-go/service/vpclattice/vpclatticeiface" + "github.com/golang/glog" + "os" ) type Lattice interface { @@ -35,7 +34,7 @@ func NewDefaultLattice(sess *session.Session, region string) *defaultLattice { endpoint = latticeEndpoint } - latticeSess = vpclattice.New(sess, aws.NewConfig().WithRegion(region).WithEndpoint(endpoint)) + latticeSess = vpclattice.New(sess, aws.NewConfig().WithRegion(region).WithEndpoint(endpoint).WithMaxRetries(20)) glog.V(2).Infoln("Lattice Service EndPoint:", endpoint) diff --git a/test/pkg/test/elasticsearch.go b/test/pkg/test/elasticsearch.go index 43cb2174..15b4841c 100644 --- a/test/pkg/test/elasticsearch.go +++ b/test/pkg/test/elasticsearch.go @@ -19,7 +19,7 @@ type ElasticSearchOptions struct { MergeFromService []*v1.Service } -func (env *Framework) NewElasticeApp(options ElasticSearchOptions) (*appsv1.Deployment, *v1.Service) { +func (env *Framework) NewElasticApp(options ElasticSearchOptions) (*appsv1.Deployment, *v1.Service) { if options.Port == 0 { options.Port = 80 } diff --git a/test/pkg/test/framework.go b/test/pkg/test/framework.go index b7f2b5c6..8cd31526 100644 --- a/test/pkg/test/framework.go +++ b/test/pkg/test/framework.go @@ -144,8 +144,9 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { retrievedTargetGroups, _ := env.LatticeClient.ListTargetGroupsAsList(ctx, &vpclattice.ListTargetGroupsInput{}) for _, tg := range retrievedTargetGroups { - Logger(ctx).Infof("Found TargetGroup: %v, checking it whether it's created by current EKS Cluster", tg) + Logger(ctx).Infof("Found TargetGroup: %s, checking it whether it's created by current EKS Cluster", *tg.Id) if currentClusterVpcId != *tg.VpcIdentifier { + Logger(ctx).Infof("Target group VPC Id: %s, does not match current EKS Cluster VPC Id: %s", *tg.VpcIdentifier, currentClusterVpcId) //This tg is not created by current EKS Cluster, skip it continue } @@ -156,6 +157,7 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { Logger(ctx).Infof("Found Tags for tg %v tags: %v", *tg.Name, retrievedTags) tagValue, ok := retrievedTags.Tags[lattice.K8SParentRefTypeKey] if ok && *tagValue == lattice.K8SServiceExportType { + Logger(ctx).Infof("TargetGroup: %s was created by k8s controller, by a ServiceExport", *tg.Id) //This tg is created by k8s controller, by a ServiceExport, //ServiceExport still have a known targetGroup leaking issue, //so we temporarily skip to verify whether ServiceExport created TargetGroup is deleted or not @@ -220,9 +222,9 @@ func (env *Framework) EventuallyExpectNotFound(ctx context.Context, objects ...c Logger(ctx).Infof("Checking whether %s %s %s is not found", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) g.Expect(errors.IsNotFound(env.Get(ctx, client.ObjectKeyFromObject(object), object))).To(BeTrue()) } - // Wait for 6 minutes at maximum just in case the k8sService deletion triggered targets draining time + // Wait for 7 minutes at maximum just in case the k8sService deletion triggered targets draining time // and httproute deletion need to wait for that targets draining time finish then it can return - }).WithTimeout(6 * time.Minute).WithOffset(1).Should(Succeed()) + }).WithTimeout(7 * time.Minute).WithOffset(1).Should(Succeed()) } func (env *Framework) EventuallyExpectNoneFound(ctx context.Context, objectList client.ObjectList) { @@ -459,7 +461,9 @@ func (env *Framework) DeleteAllFrameworkTracedVpcLatticeServices(ctx aws.Context _, err := env.LatticeClient.DeleteServiceNetworkServiceAssociationWithContext(ctx, &vpclattice.DeleteServiceNetworkServiceAssociationInput{ ServiceNetworkServiceAssociationIdentifier: snsaId, }) - Expect(err).ToNot(HaveOccurred()) + if err != nil { + Expect(err.(awserr.Error).Code()).To(Equal(vpclattice.ErrCodeResourceNotFoundException)) + } } Eventually(func(g Gomega) { @@ -493,6 +497,9 @@ func (env *Framework) DeleteAllFrameworkTracedTargetGroups(ctx aws.Context) { tgIds := lo.Map(filteredTgs, func(targetGroup *vpclattice.TargetGroupSummary, _ int) *string { return targetGroup.Id }) + + log.Println("Number of traced target groups to delete is:", len(tgIds)) + for _, tgId := range tgIds { targetSummaries, err := env.LatticeClient.ListTargetsAsList(ctx, &vpclattice.ListTargetsInput{ TargetGroupIdentifier: tgId, @@ -510,6 +517,18 @@ func (env *Framework) DeleteAllFrameworkTracedTargetGroups(ctx aws.Context) { TargetGroupIdentifier: tgId, Targets: targets, }) + } else { + Logger(ctx).Infof("Target group %s no longer has targets registered. Deleting now.", *tgId) + Eventually(func() bool { + _, err := env.LatticeClient.DeleteTargetGroup(&vpclattice.DeleteTargetGroupInput{ + TargetGroupIdentifier: tgId, + }) + if err != nil { + // Allow time for related service to be deleted prior + return err.(awserr.Error).Code() == vpclattice.ErrCodeResourceNotFoundException + } + return true + }).WithPolling(15 * time.Second).WithTimeout(2 * time.Minute).Should(BeTrue()) } } @@ -518,7 +537,7 @@ func (env *Framework) DeleteAllFrameworkTracedTargetGroups(ctx aws.Context) { //After initiating the DeregisterTargets call, the Targets will be in `draining` status for the next 5 minutes, //And VPC lattice backend will run a background job to completely delete the targets within 6 minutes at maximum in total. Eventually(func(g Gomega) { - log.Println("Trying to clear Target group", tgIdsThatNeedToWaitForDrainingTargetsToBeDeleted, " need to wait for draining targets to be deregistered") + log.Println("Trying to clear Target group", tgIdsThatNeedToWaitForDrainingTargetsToBeDeleted, "need to wait for draining targets to be deregistered") for _, tgId := range tgIdsThatNeedToWaitForDrainingTargetsToBeDeleted { _, err := env.LatticeClient.DeleteTargetGroupWithContext(ctx, &vpclattice.DeleteTargetGroupInput{ @@ -528,7 +547,7 @@ func (env *Framework) DeleteAllFrameworkTracedTargetGroups(ctx aws.Context) { g.Expect(err.(awserr.Error).Code()).To(Equal(vpclattice.ErrCodeResourceNotFoundException)) } } - }).WithTimeout(360 * time.Second).Should(Succeed()) + }).WithPolling(time.Minute).WithTimeout(7 * time.Minute).Should(Succeed()) } env.TestCasesCreatedServiceNames = make(map[string]bool) diff --git a/test/suites/integration/srvexport_port_annotation_targets_test.go b/test/suites/integration/srvexport_port_annotation_targets_test.go index 2750f21f..f563bc61 100644 --- a/test/suites/integration/srvexport_port_annotation_targets_test.go +++ b/test/suites/integration/srvexport_port_annotation_targets_test.go @@ -31,7 +31,7 @@ var _ = Describe("Port Annotations Targets", func() { BeforeEach(func() { gateway = testFramework.NewGateway("test-gateway", k8snamespace) - deployment, service = testFramework.NewElasticeApp(test.ElasticSearchOptions{ + deployment, service = testFramework.NewElasticApp(test.ElasticSearchOptions{ Name: "port-test", Namespace: k8snamespace, }) @@ -61,22 +61,12 @@ var _ = Describe("Port Annotations Targets", func() { AfterEach(func() { testFramework.CleanTestEnvironment(ctx) - testFramework.EventuallyExpectNotFound( - ctx, - gateway, - serviceExport, - serviceImport, - service, - deployment, - httpRoute, - ) }) - It("Port Annotaion on Service Export", func() { - + It("Port Annotation on Service Export", func() { targets := testFramework.GetTargets(ctx, targetGroup, deployment) Expect(*targetGroup.Port).To(BeEquivalentTo(80)) - log.Println("Verifying Targets are only craeted for the port defined in Port Annotaion in ServiceExport") + log.Println("Verifying Targets are only created for the port defined in Port Annotation in ServiceExport") for _, target := range targets { Expect(*target.Port).To(BeEquivalentTo(service.Spec.Ports[0].Port)) Expect(*target.Status).To(Or( @@ -85,12 +75,5 @@ var _ = Describe("Port Annotations Targets", func() { )) log.Println("Target:", target) } - - testFramework.ExpectDeleted(ctx, service) - Eventually(func(g Gomega) { - log.Println("Verifying Targets are only craeted for the port defined in Port Annotaion in ServiceExport") - targets := testFramework.GetTargets(ctx, targetGroup, deployment) - Expect(len(targets) == 0) - }).WithTimeout(5*time.Minute + 30*time.Second) }) })