From 5be20c35c4090e5ac4e5c3370eff97e9167a9e0c Mon Sep 17 00:00:00 2001 From: Erik F <16261515+erikfuller@users.noreply.github.com> Date: Thu, 12 Oct 2023 19:37:55 -0700 Subject: [PATCH 1/6] Fix for #425. More robust target group naming and conflict resolution. Various refactoring and improvements. --- cmd/aws-application-networking-k8s/main.go | 16 +- controllers/iamauthpolicy_controller.go | 4 +- controllers/route_controller.go | 48 +- controllers/route_controller_test.go | 294 +++ controllers/service_controller.go | 45 +- controllers/serviceexport_controller.go | 18 +- controllers/serviceimport_controller.go | 4 - go.mod | 2 +- go.sum | 8 +- .../controller-runtime/client/record_mock.go | 81 + pkg/aws/services/vpclattice.go | 98 +- pkg/aws/services/vpclattice_mocks.go | 62 +- pkg/aws/services/vpclattice_test.go | 20 +- pkg/config/controller_config.go | 24 +- pkg/config/controller_config_test.go | 5 - .../access_log_subscription_manager.go | 3 +- .../access_log_subscription_manager_test.go | 9 +- pkg/deploy/lattice/listener_manager.go | 152 +- pkg/deploy/lattice/listener_manager_mock.go | 53 +- pkg/deploy/lattice/listener_manager_test.go | 335 +--- pkg/deploy/lattice/listener_synthesizer.go | 149 +- .../lattice/listener_synthesizer_test.go | 292 ++- pkg/deploy/lattice/rule_manager.go | 624 +++---- pkg/deploy/lattice/rule_manager_mock.go | 69 +- pkg/deploy/lattice/rule_manager_test.go | 1624 ++--------------- pkg/deploy/lattice/rule_synthesizer.go | 296 ++- pkg/deploy/lattice/rule_synthesizer_test.go | 518 ++---- pkg/deploy/lattice/service_manager.go | 43 +- pkg/deploy/lattice/service_manager_mock.go | 30 +- pkg/deploy/lattice/service_manager_test.go | 15 +- pkg/deploy/lattice/service_network_manager.go | 15 +- .../lattice/service_network_manager_test.go | 6 +- pkg/deploy/lattice/service_synthesizer.go | 51 +- .../lattice/service_synthesizer_test.go | 44 +- pkg/deploy/lattice/target_group_manager.go | 463 +++-- .../lattice/target_group_manager_mock.go | 57 +- .../lattice/target_group_manager_test.go | 1142 +++++------- .../lattice/target_group_synthesizer.go | 589 +++--- .../lattice/target_group_synthesizer_test.go | 1619 +++++----------- pkg/deploy/lattice/targets_manager.go | 145 +- pkg/deploy/lattice/targets_manager_mock.go | 12 +- pkg/deploy/lattice/targets_manager_test.go | 458 ++--- pkg/deploy/lattice/targets_synthesizer.go | 50 +- .../lattice/targets_synthesizer_test.go | 143 +- pkg/deploy/stack_deployer.go | 114 +- pkg/deploy/stack_deployer_test.go | 255 --- pkg/gateway/model_build_lattice_service.go | 154 +- .../model_build_lattice_service_mock.go | 51 + .../model_build_lattice_service_test.go | 361 +++- pkg/gateway/model_build_listener.go | 112 +- pkg/gateway/model_build_listener_test.go | 269 +-- pkg/gateway/model_build_rule.go | 164 +- pkg/gateway/model_build_rule_test.go | 1120 +++++------- pkg/gateway/model_build_targetgroup.go | 493 +++-- pkg/gateway/model_build_targetgroup_mock.go | 107 ++ pkg/gateway/model_build_targetgroup_test.go | 423 ++--- pkg/gateway/model_build_targets.go | 239 +-- pkg/gateway/model_build_targets_test.go | 262 +-- pkg/k8s/finalizer_mock.go | 74 + pkg/latticestore/introspect.go | 115 -- pkg/latticestore/latticestore.go | 392 ---- pkg/latticestore/latticestore_test.go | 191 -- pkg/model/core/httproute.go | 4 + pkg/model/core/stack.go | 42 +- pkg/model/core/stack_id.go | 2 +- pkg/model/core/stack_mock.go | 15 + pkg/model/core/stack_test.go | 16 + pkg/model/lattice/listener.go | 42 +- pkg/model/lattice/rule.go | 82 +- pkg/model/lattice/service.go | 38 +- pkg/model/lattice/targetgroup.go | 173 +- pkg/model/lattice/targets.go | 21 +- pkg/utils/common.go | 4 +- scripts/gen_mocks.sh | 4 + test/pkg/test/framework.go | 215 ++- test/pkg/test/pod_manager.go | 5 + .../integration/access_log_policy_test.go | 8 - test/suites/integration/byoc_test.go | 2 - .../integration/defined_target_ports_test.go | 11 +- test/suites/integration/grpcroute_test.go | 4 +- .../integration/httproute_creation_test.go | 4 +- .../httproute_header_match_test.go | 4 +- .../httproute_method_match_test.go | 2 - ..._mutation_do_not_leak_target_group_test.go | 55 +- .../integration/httproute_path_match_test.go | 4 +- .../integration/httproute_update_test.go | 143 +- ...ed_rule_with_service_export_import_test.go | 5 +- 87 files changed, 6065 insertions(+), 9471 deletions(-) create mode 100644 controllers/route_controller_test.go create mode 100644 mocks/controller-runtime/client/record_mock.go delete mode 100644 pkg/deploy/stack_deployer_test.go create mode 100644 pkg/gateway/model_build_lattice_service_mock.go create mode 100644 pkg/gateway/model_build_targetgroup_mock.go create mode 100644 pkg/k8s/finalizer_mock.go delete mode 100644 pkg/latticestore/introspect.go delete mode 100644 pkg/latticestore/latticestore.go delete mode 100644 pkg/latticestore/latticestore_test.go diff --git a/cmd/aws-application-networking-k8s/main.go b/cmd/aws-application-networking-k8s/main.go index 30d38c5f..aa8ba518 100644 --- a/cmd/aws-application-networking-k8s/main.go +++ b/cmd/aws-application-networking-k8s/main.go @@ -48,7 +48,6 @@ import ( anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" "github.com/aws/aws-application-networking-k8s/pkg/config" "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" ) var ( @@ -112,9 +111,8 @@ func main() { setupLog.Infow("init config", "VpcId", config.VpcID, "Region", config.Region, - "AccoundId", config.AccountID, + "AccountId", config.AccountID, "DefaultServiceNetwork", config.DefaultServiceNetwork, - "UseLongTgName", config.UseLongTGName, "ClusterName", config.ClusterName, ) @@ -141,7 +139,6 @@ func main() { } finalizerManager := k8s.NewDefaultFinalizerManager(mgr.GetClient()) - latticeDataStore := latticestore.NewLatticeDataStoreWithLog(log.Named("datastore")) // parent logging scope for all controllers ctrlLog := log.Named("controller") @@ -151,7 +148,7 @@ func main() { setupLog.Fatalf("pod controller setup failed: %s", err) } - err = controllers.RegisterServiceController(ctrlLog.Named("service"), cloud, latticeDataStore, finalizerManager, mgr) + err = controllers.RegisterServiceController(ctrlLog.Named("service"), cloud, finalizerManager, mgr) if err != nil { setupLog.Fatalf("service controller setup failed: %s", err) } @@ -166,17 +163,17 @@ func main() { setupLog.Fatalf("gateway controller setup failed: %s", err) } - err = controllers.RegisterAllRouteControllers(ctrlLog.Named("route"), cloud, latticeDataStore, finalizerManager, mgr) + err = controllers.RegisterAllRouteControllers(ctrlLog.Named("route"), cloud, finalizerManager, mgr) if err != nil { setupLog.Fatalf("route controller setup failed: %s", err) } - err = controllers.RegisterServiceImportController(ctrlLog.Named("service-import"), mgr, latticeDataStore, finalizerManager) + err = controllers.RegisterServiceImportController(ctrlLog.Named("service-import"), mgr, finalizerManager) if err != nil { setupLog.Fatalf("serviceimport controller setup failed: %s", err) } - err = controllers.RegisterServiceExportController(ctrlLog.Named("service-export"), cloud, latticeDataStore, finalizerManager, mgr) + err = controllers.RegisterServiceExportController(ctrlLog.Named("service-export"), cloud, finalizerManager, mgr) if err != nil { setupLog.Fatalf("serviceexport controller setup failed: %s", err) } @@ -190,9 +187,6 @@ func main() { if err != nil { setupLog.Fatalf("iam auth policy controller setup failed: %s", err) } - - go latticestore.GetDefaultLatticeDataStore().ServeIntrospection() - //+kubebuilder:scaffold:builder if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil { diff --git a/controllers/iamauthpolicy_controller.go b/controllers/iamauthpolicy_controller.go index bfeac066..2a71d001 100644 --- a/controllers/iamauthpolicy_controller.go +++ b/controllers/iamauthpolicy_controller.go @@ -122,7 +122,7 @@ func (c *IAMAuthPolicyController) deleteGatewayPolicy(ctx context.Context, k8sPo func (c *IAMAuthPolicyController) findSnId(ctx context.Context, k8sPolicy *anv1alpha1.IAMAuthPolicy) (string, error) { tr := k8sPolicy.Spec.TargetRef - snInfo, err := c.policyMgr.Cloud.Lattice().FindServiceNetworkByK8sName(ctx, string(tr.Name)) + snInfo, err := c.policyMgr.Cloud.Lattice().FindServiceNetwork(ctx, string(tr.Name), "") if err != nil { return "", err } @@ -149,7 +149,7 @@ func (c *IAMAuthPolicyController) upsertGatewayPolicy(ctx context.Context, k8sPo func (c *IAMAuthPolicyController) findSvcId(ctx context.Context, k8sPolicy *anv1alpha1.IAMAuthPolicy) (string, error) { tr := k8sPolicy.Spec.TargetRef svcName := utils.LatticeServiceName(string(tr.Name), k8sPolicy.Namespace) - svcInfo, err := c.policyMgr.Cloud.Lattice().FindServiceByK8sName(ctx, svcName) + svcInfo, err := c.policyMgr.Cloud.Lattice().FindService(ctx, svcName) if err != nil { return "", err } diff --git a/controllers/route_controller.go b/controllers/route_controller.go index 1630e9b7..c5743a2a 100644 --- a/controllers/route_controller.go +++ b/controllers/route_controller.go @@ -19,10 +19,8 @@ package controllers import ( "context" "fmt" - "github.com/aws/aws-application-networking-k8s/pkg/aws/services" "github.com/aws/aws-application-networking-k8s/pkg/utils" - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" "github.com/pkg/errors" @@ -49,9 +47,7 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/deploy/lattice" "github.com/aws/aws-application-networking-k8s/pkg/gateway" "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" lattice_runtime "github.com/aws/aws-application-networking-k8s/pkg/runtime" ) @@ -69,7 +65,6 @@ type routeReconciler struct { eventRecorder record.EventRecorder modelBuilder gateway.LatticeServiceBuilder stackDeployer deploy.StackDeployer - latticeDataStore *latticestore.LatticeDataStore stackMarshaller deploy.StackMarshaller cloud aws.Cloud } @@ -78,18 +73,9 @@ const ( LatticeAssignedDomainName = "application-networking.k8s.aws/lattice-assigned-domain-name" ) -type RouteLSNProvider struct { - Route core.Route -} - -func (r *RouteLSNProvider) LatticeServiceName() string { - return utils.LatticeServiceName(r.Route.Name(), r.Route.Namespace()) -} - func RegisterAllRouteControllers( log gwlog.Logger, cloud aws.Cloud, - datastore *latticestore.LatticeDataStore, finalizerManager k8s.FinalizerManager, mgr ctrl.Manager, ) error { @@ -113,9 +99,8 @@ func RegisterAllRouteControllers( scheme: mgr.GetScheme(), finalizerManager: finalizerManager, eventRecorder: mgr.GetEventRecorderFor(string(routeInfo.routeType) + "route"), - latticeDataStore: datastore, - modelBuilder: gateway.NewLatticeServiceBuilder(log, mgrClient, datastore, cloud), - stackDeployer: deploy.NewLatticeServiceStackDeploy(log, cloud, mgrClient, datastore), + modelBuilder: gateway.NewLatticeServiceBuilder(log, mgrClient), + stackDeployer: deploy.NewLatticeServiceStackDeploy(log, cloud, mgrClient), stackMarshaller: deploy.NewDefaultStackMarshaller(), cloud: cloud, } @@ -195,8 +180,8 @@ func (r *routeReconciler) reconcileDelete(ctx context.Context, req ctrl.Request, r.eventRecorder.Event(route.K8sObject(), corev1.EventTypeNormal, k8s.RouteEventReasonReconcile, "Deleting Reconcile") - if err := r.cleanupRouteResources(ctx, route); err != nil { - return fmt.Errorf("failed to cleanup GRPCRoute %s, %s: %w", route.Name(), route.Namespace(), err) + if _, err := r.buildAndDeployModel(ctx, route); err != nil { + return fmt.Errorf("failed to cleanup route %s, %s: %w", route.Name(), route.Namespace(), err) } if err := updateRouteListenerStatus(ctx, r.client, route); err != nil { @@ -238,11 +223,6 @@ func updateRouteListenerStatus(ctx context.Context, k8sClient client.Client, rou return UpdateGWListenerStatus(ctx, k8sClient, gw) } -func (r *routeReconciler) cleanupRouteResources(ctx context.Context, route core.Route) error { - _, _, err := r.buildAndDeployModel(ctx, route) - return err -} - func (r *routeReconciler) isRouteRelevant(ctx context.Context, route core.Route) bool { if len(route.Spec().ParentRefs()) == 0 { r.log.Infof("Ignore Route which has no ParentRefs gateway %s ", route.Name()) @@ -290,8 +270,8 @@ func (r *routeReconciler) isRouteRelevant(ctx context.Context, route core.Route) func (r *routeReconciler) buildAndDeployModel( ctx context.Context, route core.Route, -) (core.Stack, *model.Service, error) { - stack, latticeService, err := r.modelBuilder.Build(ctx, route) +) (core.Stack, error) { + stack, err := r.modelBuilder.Build(ctx, route) if err != nil { r.eventRecorder.Event(route.K8sObject(), corev1.EventTypeWarning, @@ -300,14 +280,16 @@ func (r *routeReconciler) buildAndDeployModel( // Build failed // TODO continue deploy to trigger reconcile of stale Route and policy - return nil, nil, err + return nil, err } - _, err = r.stackMarshaller.Marshal(stack) + json, err := r.stackMarshaller.Marshal(stack) if err != nil { r.log.Errorf("error on r.stackMarshaller.Marshal error %s", err) } + r.log.Debugf("stack: %s", json) + if err := r.stackDeployer.Deploy(ctx, stack); err != nil { if errors.As(err, &lattice.RetryErr) { r.eventRecorder.Event(route.K8sObject(), corev1.EventTypeNormal, @@ -316,10 +298,10 @@ func (r *routeReconciler) buildAndDeployModel( r.eventRecorder.Event(route.K8sObject(), corev1.EventTypeWarning, k8s.RouteEventReasonFailedDeployModel, fmt.Sprintf("Failed deploy model due to %s", err)) } - return nil, nil, err + return nil, err } - return stack, latticeService, err + return stack, err } func (r *routeReconciler) reconcileUpsert(ctx context.Context, req ctrl.Request, route core.Route) error { @@ -353,19 +335,21 @@ func (r *routeReconciler) reconcileUpsert(ctx context.Context, req ctrl.Request, return backendRefIPFamiliesErr } - if _, _, err := r.buildAndDeployModel(ctx, route); err != nil { + if _, err := r.buildAndDeployModel(ctx, route); err != nil { return err } r.eventRecorder.Event(route.K8sObject(), corev1.EventTypeNormal, k8s.RouteEventReasonDeploySucceed, "Adding/Updating reconcile Done!") - svc, err := r.cloud.Lattice().FindService(ctx, &RouteLSNProvider{route}) + svcName := utils.LatticeServiceName(route.Name(), route.Namespace()) + svc, err := r.cloud.Lattice().FindService(ctx, svcName) if err != nil && !services.IsNotFoundError(err) { return err } if svc == nil || svc.DnsEntry == nil || svc.DnsEntry.DomainName == nil { + r.log.Infof("Either service, dns entry, or domain name is not available. Will Retry") return errors.New(lattice.LATTICE_RETRY) } diff --git a/controllers/route_controller_test.go b/controllers/route_controller_test.go new file mode 100644 index 00000000..9ababf70 --- /dev/null +++ b/controllers/route_controller_test.go @@ -0,0 +1,294 @@ +package controllers + +import ( + "context" + mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" + anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" + aws2 "github.com/aws/aws-application-networking-k8s/pkg/aws" + mocks "github.com/aws/aws-application-networking-k8s/pkg/aws/services" + "github.com/aws/aws-application-networking-k8s/pkg/config" + "github.com/aws/aws-application-networking-k8s/pkg/deploy" + "github.com/aws/aws-application-networking-k8s/pkg/gateway" + "github.com/aws/aws-application-networking-k8s/pkg/k8s" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/vpclattice" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + "sigs.k8s.io/external-dns/endpoint" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "testing" +) + +func TestRouteReconciler_ReconcileCreates(t *testing.T) { + config.VpcID = "my-vpc" + config.ClusterName = "my-cluster" + + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + + k8sScheme := runtime.NewScheme() + clientgoscheme.AddToScheme(k8sScheme) + gwv1beta1.AddToScheme(k8sScheme) + addOptionalCRDs(k8sScheme) + + k8sClient := testclient.NewFakeClientWithScheme(k8sScheme) + + gwClass := &gwv1beta1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "amazon-vpc-lattice", + Namespace: defaultNameSpace, + }, + Spec: gwv1beta1.GatewayClassSpec{ + ControllerName: config.LatticeGatewayControllerName, + }, + Status: gwv1beta1.GatewayClassStatus{}, + } + k8sClient.Create(ctx, gwClass.DeepCopy()) + + // here we have a gateway, service, and route + gw := &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-gateway", + Namespace: "ns1", + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: "amazon-vpc-lattice", + Listeners: []gwv1beta1.Listener{ + { + Name: "http", + Protocol: "HTTP", + Port: 80, + }, + }, + }, + } + k8sClient.Create(ctx, gw.DeepCopy()) + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-service", + Namespace: "ns1", + }, + Spec: corev1.ServiceSpec{ + IPFamilies: []corev1.IPFamily{ + "IPv4", + }, + Ports: []corev1.ServicePort{ + { + Protocol: "TCP", + Port: 80, + TargetPort: intstr.FromInt(8090), + }, + }, + }, + } + k8sClient.Create(ctx, svc.DeepCopy()) + + endPoints := corev1.Endpoints{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "my-service", + Name: "ns1", + }, + Subsets: []corev1.EndpointSubset{ + { + Addresses: []corev1.EndpointAddress{ + { + IP: "192.0.2.22", + }, + { + IP: "192.0.2.33", + }, + }, + Ports: []corev1.EndpointPort{ + { + Port: 8090, + }, + }, + }, + }, + } + k8sClient.Create(ctx, endPoints.DeepCopy()) + + kind := gwv1beta1.Kind("Service") + port := gwv1beta1.PortNumber(80) + route := gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-route", + Namespace: "ns1", + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Name: "my-gateway", + }, + }, + }, + Rules: []gwv1beta1.HTTPRouteRule{ + { + BackendRefs: []gwv1beta1.HTTPBackendRef{ + { + BackendRef: gwv1beta1.BackendRef{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Kind: &kind, + Name: "my-service", + Port: &port, + }, + Weight: aws.Int32(10), + }, + }, + }, + }, + }, + }, + } + k8sClient.Create(ctx, route.DeepCopy()) + + mockCloud := aws2.NewMockCloud(c) + mockLattice := mocks.NewMockLattice(c) + mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() + mockCloud.EXPECT().Config().Return( + aws2.CloudConfig{ + VpcId: config.VpcID, + AccountId: "account-id", + Region: "ep-imagine-1", + ClusterName: config.ClusterName, + }).AnyTimes() + mockCloud.EXPECT().DefaultTags().Return(mocks.Tags{}).AnyTimes() + + // we expect a fair number of lattice calls + mockLattice.EXPECT().FindServiceNetwork(ctx, gomock.Any(), gomock.Any()).Return( + &mocks.ServiceNetworkInfo{ + SvcNetwork: vpclattice.ServiceNetworkSummary{ + Arn: aws.String("sn-arn"), + Id: aws.String("sn-id"), + Name: aws.String("sn-name"), + }, + }, nil) + mockLattice.EXPECT().FindService(ctx, gomock.Any()).Return( + nil, mocks.NewNotFoundError("Service", "svc-name")) // will trigger create + mockLattice.EXPECT().CreateServiceWithContext(ctx, gomock.Any()).Return( + &vpclattice.CreateServiceOutput{ + Arn: aws.String("svc-arn"), + Id: aws.String("svc-id"), + Name: aws.String("svc-name"), + Status: aws.String(vpclattice.ServiceStatusActive), + }, nil) + mockLattice.EXPECT().CreateServiceNetworkServiceAssociationWithContext(ctx, gomock.Any()).Return( + &vpclattice.CreateServiceNetworkServiceAssociationOutput{ + Arn: aws.String("sns-assoc-arn"), + Id: aws.String("sns-assoc-id"), + Status: aws.String(vpclattice.ServiceNetworkServiceAssociationStatusActive), + }, nil) + mockLattice.EXPECT().FindService(ctx, gomock.Any()).Return( + &vpclattice.ServiceSummary{ + Arn: aws.String("svc-arn"), + Id: aws.String("svc-id"), + Name: aws.String("svc-name"), + Status: aws.String(vpclattice.ServiceStatusActive), + DnsEntry: &vpclattice.DnsEntry{ + DomainName: aws.String("my-fqdn.lattice.on.aws"), + HostedZoneId: aws.String("my-hosted-zone"), + }, + }, nil) // will trigger DNS Update + + mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return( + []*vpclattice.TargetGroupSummary{}, nil).AnyTimes() // this will cause us to skip "unused delete" step + mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).Return( + &vpclattice.CreateTargetGroupOutput{ + Arn: aws.String("tg-arn"), + Id: aws.String("tg-id"), + Name: aws.String("tg-name"), + Status: aws.String(vpclattice.TargetGroupStatusActive), + }, nil) + + mockLattice.EXPECT().ListListenersWithContext(ctx, gomock.Any()).Return( + &vpclattice.ListListenersOutput{ + Items: []*vpclattice.ListenerSummary{}, + NextToken: nil, + }, nil).AnyTimes() + mockLattice.EXPECT().CreateListenerWithContext(ctx, gomock.Any()).Return( + &vpclattice.CreateListenerOutput{ + Arn: aws.String("listener-arn"), + Id: aws.String("listener-id"), + Name: aws.String("listener-name"), + ServiceArn: aws.String("svc-arn"), + ServiceId: aws.String("svc-id"), + }, nil) + + mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( + []*vpclattice.GetRuleOutput{}, nil) + mockLattice.EXPECT().CreateRuleWithContext(ctx, gomock.Any()).Return( + &vpclattice.CreateRuleOutput{ + Arn: aws.String("rule-arn"), + Id: aws.String("rule-id"), + Name: aws.String("rule-name"), + Priority: aws.Int64(1), + }, nil) + // List is called after create, so we'll return what we have + mockLattice.EXPECT().ListRulesAsList(ctx, gomock.Any()).Return( + []*vpclattice.RuleSummary{ + { + Arn: aws.String("rule-arn"), + Id: aws.String("rule-id"), + IsDefault: aws.Bool(false), + Name: aws.String("rule-name"), + Priority: aws.Int64(1), + }, + }, nil) + + mockEventRecorder := mock_client.NewMockEventRecorder(c) + mockEventRecorder.EXPECT().Event(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + mockFinalizer := k8s.NewMockFinalizerManager(c) + mockFinalizer.EXPECT().AddFinalizers(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + mockFinalizer.EXPECT().RemoveFinalizers(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + + rc := routeReconciler{ + routeType: core.HttpRouteType, + log: gwlog.FallbackLogger, + client: k8sClient, + scheme: k8sScheme, + finalizerManager: mockFinalizer, + eventRecorder: mockEventRecorder, + modelBuilder: gateway.NewLatticeServiceBuilder(gwlog.FallbackLogger, k8sClient), + stackDeployer: deploy.NewLatticeServiceStackDeploy(gwlog.FallbackLogger, mockCloud, k8sClient), + stackMarshaller: deploy.NewDefaultStackMarshaller(), + cloud: mockCloud, + } + + routeName := k8s.NamespacedName(&route) + result, err := rc.Reconcile(ctx, reconcile.Request{NamespacedName: routeName}) + assert.Nil(t, err) + assert.False(t, result.Requeue) + +} + +func addOptionalCRDs(scheme *runtime.Scheme) { + dnsEndpoint := schema.GroupVersion{ + Group: "externaldns.k8s.io", + Version: "v1alpha1", + } + scheme.AddKnownTypes(dnsEndpoint, &endpoint.DNSEndpoint{}, &endpoint.DNSEndpointList{}) + metav1.AddToGroupVersion(scheme, dnsEndpoint) + + awsGatewayControllerCRDGroupVersion := schema.GroupVersion{ + Group: anv1alpha1.GroupName, + Version: "v1alpha1", + } + scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &anv1alpha1.TargetGroupPolicy{}, &anv1alpha1.TargetGroupPolicyList{}) + metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion) + + scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &anv1alpha1.VpcAssociationPolicy{}, &anv1alpha1.VpcAssociationPolicyList{}) + metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion) +} diff --git a/controllers/service_controller.go b/controllers/service_controller.go index c007a076..be3c2cf4 100644 --- a/controllers/service_controller.go +++ b/controllers/service_controller.go @@ -30,9 +30,6 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/deploy" "github.com/aws/aws-application-networking-k8s/pkg/gateway" "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" lattice_runtime "github.com/aws/aws-application-networking-k8s/pkg/runtime" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" ) @@ -49,33 +46,24 @@ type serviceReconciler struct { eventRecorder record.EventRecorder targetsBuilder gateway.LatticeTargetsBuilder stackDeployer deploy.StackDeployer - datastore *latticestore.LatticeDataStore stackMashaller deploy.StackMarshaller } func RegisterServiceController( log gwlog.Logger, cloud aws.Cloud, - datastore *latticestore.LatticeDataStore, finalizerManager k8s.FinalizerManager, mgr ctrl.Manager, ) error { client := mgr.GetClient() scheme := mgr.GetScheme() evtRec := mgr.GetEventRecorderFor("service") - modelBuilder := gateway.NewTargetsBuilder(log, client, cloud, datastore) - stackDeployer := deploy.NewTargetsStackDeployer(log, cloud, client, datastore) - stackMarshaller := deploy.NewDefaultStackMarshaller() sr := &serviceReconciler{ log: log, client: client, scheme: scheme, finalizerManager: finalizerManager, - targetsBuilder: modelBuilder, - stackDeployer: stackDeployer, eventRecorder: evtRec, - datastore: datastore, - stackMashaller: stackMarshaller, } err := ctrl.NewControllerManagedBy(mgr). For(&corev1.Service{}). @@ -100,19 +88,15 @@ func (r *serviceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } func (r *serviceReconciler) reconcile(ctx context.Context, req ctrl.Request) error { + // today, target registration is handled through the route controller + // if that proves not to be fast enough, we can look to do quicker + // target registration with the service controller + svc := &corev1.Service{} if err := r.client.Get(ctx, req.NamespacedName, svc); err != nil { return client.IgnoreNotFound(err) } if !svc.DeletionTimestamp.IsZero() { - tgName := latticestore.TargetGroupName(svc.Name, svc.Namespace) - tgs := r.datastore.GetTargetGroupsByName(tgName) - for _, tg := range tgs { - r.log.Debugf("deletion request for tgName: %s: at timestamp: %s", tg.TargetGroupKey.Name, svc.DeletionTimestamp) - if _, _, err := r.buildAndDeployModel(ctx, svc, tg.TargetGroupKey.RouteName); err != nil { - return err - } - } r.finalizerManager.RemoveFinalizers(ctx, svc, serviceFinalizer) } else { if err := r.finalizerManager.AddFinalizers(ctx, svc, serviceFinalizer); err != nil { @@ -123,24 +107,3 @@ func (r *serviceReconciler) reconcile(ctx context.Context, req ctrl.Request) err r.log.Infow("reconciled", "name", req.Name) return nil } - -func (r *serviceReconciler) buildAndDeployModel(ctx context.Context, svc *corev1.Service, routename string) (core.Stack, *model.Targets, error) { - stack, latticeTargets, err := r.targetsBuilder.Build(ctx, svc, routename) - if err != nil { - r.eventRecorder.Event(svc, corev1.EventTypeWarning, - k8s.ServiceEventReasonFailedBuildModel, fmt.Sprintf("failed build model: %s", err)) - return nil, nil, err - } - - jsonStack, _ := r.stackMashaller.Marshal(stack) - r.log.Debugw("successfully built model", "stack", jsonStack) - - if err := r.stackDeployer.Deploy(ctx, stack); err != nil { - r.eventRecorder.Event(svc, corev1.EventTypeWarning, - k8s.ServiceEventReasonFailedDeployModel, fmt.Sprintf("failed deploy model: %s", err)) - return nil, nil, err - } - - r.log.Debugw("successfully deployed model", "service", svc.Name) - return stack, latticeTargets, err -} diff --git a/controllers/serviceexport_controller.go b/controllers/serviceexport_controller.go index 880893f2..e88d2a14 100644 --- a/controllers/serviceexport_controller.go +++ b/controllers/serviceexport_controller.go @@ -36,7 +36,6 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/deploy" "github.com/aws/aws-application-networking-k8s/pkg/gateway" "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" lattice_runtime "github.com/aws/aws-application-networking-k8s/pkg/runtime" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" ) @@ -49,7 +48,6 @@ type serviceExportReconciler struct { eventRecorder record.EventRecorder modelBuilder gateway.SvcExportTargetGroupModelBuilder stackDeployer deploy.StackDeployer - latticeDataStore *latticestore.LatticeDataStore stackMarshaller deploy.StackMarshaller } @@ -60,7 +58,6 @@ const ( func RegisterServiceExportController( log gwlog.Logger, cloud aws.Cloud, - latticeDataStore *latticestore.LatticeDataStore, finalizerManager k8s.FinalizerManager, mgr ctrl.Manager, ) error { @@ -68,8 +65,8 @@ func RegisterServiceExportController( scheme := mgr.GetScheme() eventRecorder := mgr.GetEventRecorderFor("serviceExport") - modelBuilder := gateway.NewSvcExportTargetGroupBuilder(log, mgrClient, latticeDataStore, cloud) - stackDeploy := deploy.NewTargetGroupStackDeploy(log, cloud, mgrClient, latticeDataStore) + modelBuilder := gateway.NewSvcExportTargetGroupBuilder(log, mgrClient) + stackDeploy := deploy.NewTargetGroupStackDeploy(log, cloud, mgrClient) stackMarshaller := deploy.NewDefaultStackMarshaller() r := &serviceExportReconciler{ @@ -80,7 +77,6 @@ func RegisterServiceExportController( modelBuilder: modelBuilder, stackDeployer: stackDeploy, eventRecorder: eventRecorder, - latticeDataStore: latticeDataStore, stackMarshaller: stackMarshaller, } @@ -152,7 +148,7 @@ func (r *serviceExportReconciler) buildAndDeployModel( ctx context.Context, srvExport *mcsv1alpha1.ServiceExport, ) error { - stack, _, err := r.modelBuilder.Build(ctx, srvExport) + stack, err := r.modelBuilder.Build(ctx, srvExport) if err != nil { r.log.Debugf("Failed to buildAndDeployModel for service export %s-%s due to %s", @@ -162,16 +158,14 @@ func (r *serviceExportReconciler) buildAndDeployModel( k8s.GatewayEventReasonFailedBuildModel, fmt.Sprintf("Failed BuildModel due to %s", err)) - // Build failed means the K8S serviceexport, service are NOT ready to be deployed to lattice - // return nil to complete controller loop for current change. - // TODO continue deploy to trigger reconcile of stale SDK objects - //return stack, targetGroup, nil + return err } - _, err = r.stackMarshaller.Marshal(stack) + json, err := r.stackMarshaller.Marshal(stack) if err != nil { r.log.Errorf("Error on marshalling model for service export %s-%s", srvExport.Name, srvExport.Namespace) } + r.log.Debugf("stack: %s", json) if err := r.stackDeployer.Deploy(ctx, stack); err != nil { r.eventRecorder.Event(srvExport, corev1.EventTypeWarning, diff --git a/controllers/serviceimport_controller.go b/controllers/serviceimport_controller.go index 99ed0fa7..8c6c8b5c 100644 --- a/controllers/serviceimport_controller.go +++ b/controllers/serviceimport_controller.go @@ -30,7 +30,6 @@ import ( mcs_api "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" ) type serviceImportReconciler struct { @@ -39,7 +38,6 @@ type serviceImportReconciler struct { Scheme *runtime.Scheme finalizerManager k8s.FinalizerManager eventRecorder record.EventRecorder - latticeDataStore *latticestore.LatticeDataStore } const ( @@ -49,7 +47,6 @@ const ( func RegisterServiceImportController( log gwlog.Logger, mgr ctrl.Manager, - dataStore *latticestore.LatticeDataStore, finalizerManager k8s.FinalizerManager, ) error { mgrClient := mgr.GetClient() @@ -62,7 +59,6 @@ func RegisterServiceImportController( Scheme: scheme, finalizerManager: finalizerManager, eventRecorder: eventRecorder, - latticeDataStore: dataStore, } return ctrl.NewControllerManagedBy(mgr). diff --git a/go.mod b/go.mod index 080d432e..43afd7bf 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.24.1 github.com/pkg/errors v0.9.1 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.4 go.uber.org/zap v1.24.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 k8s.io/api v0.26.1 diff --git a/go.sum b/go.sum index 869b504c..0d3aa598 100644 --- a/go.sum +++ b/go.sum @@ -472,18 +472,14 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= diff --git a/mocks/controller-runtime/client/record_mock.go b/mocks/controller-runtime/client/record_mock.go new file mode 100644 index 00000000..399f2c11 --- /dev/null +++ b/mocks/controller-runtime/client/record_mock.go @@ -0,0 +1,81 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: k8s.io/client-go/tools/record (interfaces: EventRecorder) + +// Package mock_client is a generated GoMock package. +package mock_client + +import ( + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// MockEventRecorder is a mock of EventRecorder interface. +type MockEventRecorder struct { + ctrl *gomock.Controller + recorder *MockEventRecorderMockRecorder +} + +// MockEventRecorderMockRecorder is the mock recorder for MockEventRecorder. +type MockEventRecorderMockRecorder struct { + mock *MockEventRecorder +} + +// NewMockEventRecorder creates a new mock instance. +func NewMockEventRecorder(ctrl *gomock.Controller) *MockEventRecorder { + mock := &MockEventRecorder{ctrl: ctrl} + mock.recorder = &MockEventRecorderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockEventRecorder) EXPECT() *MockEventRecorderMockRecorder { + return m.recorder +} + +// AnnotatedEventf mocks base method. +func (m *MockEventRecorder) AnnotatedEventf(arg0 runtime.Object, arg1 map[string]string, arg2, arg3, arg4 string, arg5 ...interface{}) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3, arg4} + for _, a := range arg5 { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "AnnotatedEventf", varargs...) +} + +// AnnotatedEventf indicates an expected call of AnnotatedEventf. +func (mr *MockEventRecorderMockRecorder) AnnotatedEventf(arg0, arg1, arg2, arg3, arg4 interface{}, arg5 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3, arg4}, arg5...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AnnotatedEventf", reflect.TypeOf((*MockEventRecorder)(nil).AnnotatedEventf), varargs...) +} + +// Event mocks base method. +func (m *MockEventRecorder) Event(arg0 runtime.Object, arg1, arg2, arg3 string) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Event", arg0, arg1, arg2, arg3) +} + +// Event indicates an expected call of Event. +func (mr *MockEventRecorderMockRecorder) Event(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Event", reflect.TypeOf((*MockEventRecorder)(nil).Event), arg0, arg1, arg2, arg3) +} + +// Eventf mocks base method. +func (m *MockEventRecorder) Eventf(arg0 runtime.Object, arg1, arg2, arg3 string, arg4 ...interface{}) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1, arg2, arg3} + for _, a := range arg4 { + varargs = append(varargs, a) + } + m.ctrl.Call(m, "Eventf", varargs...) +} + +// Eventf indicates an expected call of Eventf. +func (mr *MockEventRecorderMockRecorder) Eventf(arg0, arg1, arg2, arg3 interface{}, arg4 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1, arg2, arg3}, arg4...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Eventf", reflect.TypeOf((*MockEventRecorder)(nil).Eventf), varargs...) +} diff --git a/pkg/aws/services/vpclattice.go b/pkg/aws/services/vpclattice.go index 16a34c83..34ece7c7 100644 --- a/pkg/aws/services/vpclattice.go +++ b/pkg/aws/services/vpclattice.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/aws/aws-sdk-go/aws/awserr" "os" "github.com/aws/aws-sdk-go/aws" @@ -22,24 +23,6 @@ type ServiceNetworkInfo struct { Tags Tags } -type LatticeServiceNameProvider interface { - LatticeServiceName() string -} - -type defaultLatticeServiceNameProvider struct { - name string -} - -func NewDefaultLatticeServiceNameProvider(name string) *defaultLatticeServiceNameProvider { - return &defaultLatticeServiceNameProvider{ - name: name, - } -} - -func (p *defaultLatticeServiceNameProvider) LatticeServiceName() string { - return p.name -} - type NotFoundError struct { ResourceType string Name string @@ -96,6 +79,8 @@ func IsInvalidError(err error) bool { type Lattice interface { vpclatticeiface.VPCLatticeAPI + GetRulesAsList(ctx context.Context, input *vpclattice.ListRulesInput) ([]*vpclattice.GetRuleOutput, error) + ListRulesAsList(ctx context.Context, input *vpclattice.ListRulesInput) ([]*vpclattice.RuleSummary, error) ListServiceNetworksAsList(ctx context.Context, input *vpclattice.ListServiceNetworksInput) ([]*vpclattice.ServiceNetworkSummary, error) ListServicesAsList(ctx context.Context, input *vpclattice.ListServicesInput) ([]*vpclattice.ServiceSummary, error) ListTargetGroupsAsList(ctx context.Context, input *vpclattice.ListTargetGroupsInput) ([]*vpclattice.TargetGroupSummary, error) @@ -103,9 +88,7 @@ type Lattice interface { ListServiceNetworkVpcAssociationsAsList(ctx context.Context, input *vpclattice.ListServiceNetworkVpcAssociationsInput) ([]*vpclattice.ServiceNetworkVpcAssociationSummary, error) ListServiceNetworkServiceAssociationsAsList(ctx context.Context, input *vpclattice.ListServiceNetworkServiceAssociationsInput) ([]*vpclattice.ServiceNetworkServiceAssociationSummary, error) FindServiceNetwork(ctx context.Context, name string, accountId string) (*ServiceNetworkInfo, error) - FindService(ctx context.Context, nameProvider LatticeServiceNameProvider) (*vpclattice.ServiceSummary, error) - FindServiceByK8sName(ctx context.Context, k8sname string) (*vpclattice.ServiceSummary, error) - FindServiceNetworkByK8sName(ctx context.Context, k8sname string) (*ServiceNetworkInfo, error) + FindService(ctx context.Context, latticeServiceName string) (*vpclattice.ServiceSummary, error) } type defaultLattice struct { @@ -127,6 +110,56 @@ func NewDefaultLattice(sess *session.Session, region string) *defaultLattice { return &defaultLattice{latticeSess} } +func (d *defaultLattice) GetRulesAsList(ctx context.Context, input *vpclattice.ListRulesInput) ([]*vpclattice.GetRuleOutput, error) { + var result []*vpclattice.GetRuleOutput + + var innerErr error + err := d.ListRulesPagesWithContext(ctx, input, func(page *vpclattice.ListRulesOutput, lastPage bool) bool { + for _, r := range page.Items { + grInput := vpclattice.GetRuleInput{ + ServiceIdentifier: input.ServiceIdentifier, + ListenerIdentifier: input.ListenerIdentifier, + RuleIdentifier: r.Id, + } + + var gro *vpclattice.GetRuleOutput + gro, innerErr = d.GetRuleWithContext(ctx, &grInput) + if innerErr != nil { + return false + } + result = append(result, gro) + } + return true + }) + + if innerErr != nil { + return nil, innerErr + } + + if err != nil { + return nil, err + } + + return result, nil +} + +func (d *defaultLattice) ListRulesAsList(ctx context.Context, input *vpclattice.ListRulesInput) ([]*vpclattice.RuleSummary, error) { + var result []*vpclattice.RuleSummary + + err := d.ListRulesPagesWithContext(ctx, input, func(page *vpclattice.ListRulesOutput, lastPage bool) bool { + for _, r := range page.Items { + result = append(result, r) + } + return true + }) + + if err != nil { + return nil, err + } + + return result, nil +} + func (d *defaultLattice) ListServiceNetworksAsList(ctx context.Context, input *vpclattice.ListServiceNetworksInput) ([]*vpclattice.ServiceNetworkSummary, error) { result := []*vpclattice.ServiceNetworkSummary{} @@ -280,14 +313,15 @@ func (d *defaultLattice) FindServiceNetwork(ctx context.Context, name string, op return snMatch, nil } -func (d *defaultLattice) FindService(ctx context.Context, nameProvider LatticeServiceNameProvider) (*vpclattice.ServiceSummary, error) { - serviceName := nameProvider.LatticeServiceName() + +// see utils.LatticeServiceName +func (d *defaultLattice) FindService(ctx context.Context, latticeServiceName string) (*vpclattice.ServiceSummary, error) { input := vpclattice.ListServicesInput{} var svcMatch *vpclattice.ServiceSummary err := d.ListServicesPagesWithContext(ctx, &input, func(page *vpclattice.ListServicesOutput, lastPage bool) bool { for _, svc := range page.Items { - if *svc.Name == serviceName { + if *svc.Name == latticeServiceName { svcMatch = svc return false } @@ -299,7 +333,7 @@ func (d *defaultLattice) FindService(ctx context.Context, nameProvider LatticeSe return nil, err } if svcMatch == nil { - return nil, NewNotFoundError("Service", serviceName) + return nil, NewNotFoundError("Service", latticeServiceName) } return svcMatch, nil @@ -318,10 +352,14 @@ func accountIdMatches(accountId string, itemArn string) (bool, error) { return accountId == parsedArn.AccountID, nil } -func (d *defaultLattice) FindServiceByK8sName(ctx context.Context, k8sname string) (*vpclattice.ServiceSummary, error) { - return d.FindService(ctx, NewDefaultLatticeServiceNameProvider(k8sname)) -} +func IsLatticeAPINotFoundErr(err error) bool { + if err == nil { + return false + } -func (d *defaultLattice) FindServiceNetworkByK8sName(ctx context.Context, k8sname string) (*ServiceNetworkInfo, error) { - return d.FindServiceNetwork(ctx, k8sname, "") + var aErr awserr.Error + if errors.As(err, &aErr) { + return aErr.Code() == vpclattice.ErrCodeResourceNotFoundException + } + return false } diff --git a/pkg/aws/services/vpclattice_mocks.go b/pkg/aws/services/vpclattice_mocks.go index 13f43d0e..febbc415 100644 --- a/pkg/aws/services/vpclattice_mocks.go +++ b/pkg/aws/services/vpclattice_mocks.go @@ -1037,7 +1037,7 @@ func (mr *MockLatticeMockRecorder) DeregisterTargetsWithContext(arg0, arg1 inter } // FindService mocks base method. -func (m *MockLattice) FindService(arg0 context.Context, arg1 LatticeServiceNameProvider) (*vpclattice.ServiceSummary, error) { +func (m *MockLattice) FindService(arg0 context.Context, arg1 string) (*vpclattice.ServiceSummary, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FindService", arg0, arg1) ret0, _ := ret[0].(*vpclattice.ServiceSummary) @@ -1051,21 +1051,6 @@ func (mr *MockLatticeMockRecorder) FindService(arg0, arg1 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindService", reflect.TypeOf((*MockLattice)(nil).FindService), arg0, arg1) } -// FindServiceByK8sName mocks base method. -func (m *MockLattice) FindServiceByK8sName(arg0 context.Context, arg1 string) (*vpclattice.ServiceSummary, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FindServiceByK8sName", arg0, arg1) - ret0, _ := ret[0].(*vpclattice.ServiceSummary) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// FindServiceByK8sName indicates an expected call of FindServiceByK8sName. -func (mr *MockLatticeMockRecorder) FindServiceByK8sName(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindServiceByK8sName", reflect.TypeOf((*MockLattice)(nil).FindServiceByK8sName), arg0, arg1) -} - // FindServiceNetwork mocks base method. func (m *MockLattice) FindServiceNetwork(arg0 context.Context, arg1, arg2 string) (*ServiceNetworkInfo, error) { m.ctrl.T.Helper() @@ -1081,21 +1066,6 @@ func (mr *MockLatticeMockRecorder) FindServiceNetwork(arg0, arg1, arg2 interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindServiceNetwork", reflect.TypeOf((*MockLattice)(nil).FindServiceNetwork), arg0, arg1, arg2) } -// FindServiceNetworkByK8sName mocks base method. -func (m *MockLattice) FindServiceNetworkByK8sName(arg0 context.Context, arg1 string) (*ServiceNetworkInfo, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "FindServiceNetworkByK8sName", arg0, arg1) - ret0, _ := ret[0].(*ServiceNetworkInfo) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// FindServiceNetworkByK8sName indicates an expected call of FindServiceNetworkByK8sName. -func (mr *MockLatticeMockRecorder) FindServiceNetworkByK8sName(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FindServiceNetworkByK8sName", reflect.TypeOf((*MockLattice)(nil).FindServiceNetworkByK8sName), arg0, arg1) -} - // GetAccessLogSubscription mocks base method. func (m *MockLattice) GetAccessLogSubscription(arg0 *vpclattice.GetAccessLogSubscriptionInput) (*vpclattice.GetAccessLogSubscriptionOutput, error) { m.ctrl.T.Helper() @@ -1346,6 +1316,21 @@ func (mr *MockLatticeMockRecorder) GetRuleWithContext(arg0, arg1 interface{}, ar return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRuleWithContext", reflect.TypeOf((*MockLattice)(nil).GetRuleWithContext), varargs...) } +// GetRulesAsList mocks base method. +func (m *MockLattice) GetRulesAsList(arg0 context.Context, arg1 *vpclattice.ListRulesInput) ([]*vpclattice.GetRuleOutput, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRulesAsList", arg0, arg1) + ret0, _ := ret[0].([]*vpclattice.GetRuleOutput) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRulesAsList indicates an expected call of GetRulesAsList. +func (mr *MockLatticeMockRecorder) GetRulesAsList(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRulesAsList", reflect.TypeOf((*MockLattice)(nil).GetRulesAsList), arg0, arg1) +} + // GetService mocks base method. func (m *MockLattice) GetService(arg0 *vpclattice.GetServiceInput) (*vpclattice.GetServiceOutput, error) { m.ctrl.T.Helper() @@ -1777,6 +1762,21 @@ func (mr *MockLatticeMockRecorder) ListRules(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRules", reflect.TypeOf((*MockLattice)(nil).ListRules), arg0) } +// ListRulesAsList mocks base method. +func (m *MockLattice) ListRulesAsList(arg0 context.Context, arg1 *vpclattice.ListRulesInput) ([]*vpclattice.RuleSummary, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRulesAsList", arg0, arg1) + ret0, _ := ret[0].([]*vpclattice.RuleSummary) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRulesAsList indicates an expected call of ListRulesAsList. +func (mr *MockLatticeMockRecorder) ListRulesAsList(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRulesAsList", reflect.TypeOf((*MockLattice)(nil).ListRulesAsList), arg0, arg1) +} + // ListRulesPages mocks base method. func (m *MockLattice) ListRulesPages(arg0 *vpclattice.ListRulesInput, arg1 func(*vpclattice.ListRulesOutput, bool) bool) error { m.ctrl.T.Helper() diff --git a/pkg/aws/services/vpclattice_test.go b/pkg/aws/services/vpclattice_test.go index 3b8ba243..16425ca9 100644 --- a/pkg/aws/services/vpclattice_test.go +++ b/pkg/aws/services/vpclattice_test.go @@ -570,14 +570,6 @@ func Test_defaultLattice_FindServiceNetwork_errorsRaised(t *testing.T) { assert.False(t, IsNotFoundError(tagErr)) } -type StringLSNProvider struct { - name string -} - -func (p *StringLSNProvider) LatticeServiceName() string { - return p.name -} - func Test_defaultLattice_FindService_happyPath(t *testing.T) { ctx := context.TODO() c := gomock.NewController(t) @@ -603,12 +595,12 @@ func Test_defaultLattice_FindService_happyPath(t *testing.T) { return nil }).AnyTimes() - itemFound, err1 := d.FindService(ctx, &StringLSNProvider{name}) + itemFound, err1 := d.FindService(ctx, name) assert.Nil(t, err1) assert.NotNil(t, itemFound) assert.Equal(t, name, *itemFound.Name) - itemNotFound, err2 := d.FindService(ctx, &StringLSNProvider{"no-name"}) + itemNotFound, err2 := d.FindService(ctx, "no-name") assert.True(t, IsNotFoundError(err2)) assert.Nil(t, itemNotFound) } @@ -648,17 +640,17 @@ func Test_defaultLattice_FindService_pagedResults(t *testing.T) { return nil }).AnyTimes() - itemFound1, err1 := d.FindService(ctx, &StringLSNProvider{"name1"}) + itemFound1, err1 := d.FindService(ctx, "name1") assert.Nil(t, err1) assert.NotNil(t, itemFound1) assert.Equal(t, "name1", *itemFound1.Name) - itemFound2, err2 := d.FindService(ctx, &StringLSNProvider{"name2"}) + itemFound2, err2 := d.FindService(ctx, "name2") assert.Nil(t, err2) assert.NotNil(t, itemFound2) assert.Equal(t, "name2", *itemFound2.Name) - itemNotFound, err3 := d.FindService(ctx, &StringLSNProvider{"no-name"}) + itemNotFound, err3 := d.FindService(ctx, "no-name") assert.True(t, IsNotFoundError(err3)) assert.Nil(t, itemNotFound) } @@ -672,7 +664,7 @@ func Test_defaultLattice_FindService_errorsRaised(t *testing.T) { mockLattice.EXPECT().ListServicesPagesWithContext(gomock.Any(), gomock.Any(), gomock.Any()). Return(errors.New("LIST_ERR")).Times(1) - _, listErr := d.FindService(ctx, &StringLSNProvider{"foo"}) + _, listErr := d.FindService(ctx, "foo") assert.NotNil(t, listErr) assert.False(t, IsNotFoundError(listErr)) } diff --git a/pkg/config/controller_config.go b/pkg/config/controller_config.go index c55e9f0a..d7193212 100644 --- a/pkg/config/controller_config.go +++ b/pkg/config/controller_config.go @@ -18,14 +18,12 @@ const ( ) const ( - NO_DEFAULT_SERVICE_NETWORK = "NO_DEFAULT_SERVICE_NETWORK" - REGION = "REGION" - CLUSTER_VPC_ID = "CLUSTER_VPC_ID" - CLUSTER_NAME = "CLUSTER_NAME" - CLUSTER_LOCAL_GATEWAY = "CLUSTER_LOCAL_GATEWAY" - AWS_ACCOUNT_ID = "AWS_ACCOUNT_ID" - TARGET_GROUP_NAME_LEN_MODE = "TARGET_GROUP_NAME_LEN_MODE" - GATEWAY_API_CONTROLLER_LOGLEVEL = "GATEWAY_API_CONTROLLER_LOGLEVEL" + NO_DEFAULT_SERVICE_NETWORK = "NO_DEFAULT_SERVICE_NETWORK" + REGION = "REGION" + CLUSTER_VPC_ID = "CLUSTER_VPC_ID" + CLUSTER_NAME = "CLUSTER_NAME" + CLUSTER_LOCAL_GATEWAY = "CLUSTER_LOCAL_GATEWAY" + AWS_ACCOUNT_ID = "AWS_ACCOUNT_ID" ) var VpcID = "" @@ -33,7 +31,6 @@ var AccountID = "" var Region = "" var logLevel = defaultLogLevel var DefaultServiceNetwork = "" -var UseLongTGName = false var ClusterName = "" func GetClusterLocalGateway() (string, error) { @@ -88,15 +85,6 @@ func configInit(sess *session.Session, metadata EC2Metadata) error { return fmt.Errorf("cannot get cluster name: %s", err) } - // TARGET_GROUP_NAME_LEN_MODE - tgNameLengthMode := os.Getenv(TARGET_GROUP_NAME_LEN_MODE) - - if tgNameLengthMode == "long" { - UseLongTGName = true - } else { - UseLongTGName = false - } - return nil } diff --git a/pkg/config/controller_config_test.go b/pkg/config/controller_config_test.go index c7955ded..7e5dc735 100644 --- a/pkg/config/controller_config_test.go +++ b/pkg/config/controller_config_test.go @@ -37,7 +37,6 @@ func Test_config_init_with_partial_env_var(t *testing.T) { os.Setenv(CLUSTER_VPC_ID, testClusterVpcId) os.Setenv(CLUSTER_LOCAL_GATEWAY, testClusterLocalGateway) os.Unsetenv(AWS_ACCOUNT_ID) - os.Unsetenv(TARGET_GROUP_NAME_LEN_MODE) err := configInit(nil, ec2MetadataUnavailable()) assert.NotNil(t, err) } @@ -47,7 +46,6 @@ func Test_config_init_no_env_var(t *testing.T) { os.Unsetenv(CLUSTER_VPC_ID) os.Unsetenv(CLUSTER_LOCAL_GATEWAY) os.Unsetenv(AWS_ACCOUNT_ID) - os.Unsetenv(TARGET_GROUP_NAME_LEN_MODE) err := configInit(nil, ec2MetadataUnavailable()) assert.NotNil(t, err) @@ -58,7 +56,6 @@ func Test_config_init_with_all_env_var(t *testing.T) { testRegion := "us-west-2" testClusterVpcId := "vpc-123456" testClusterLocalGateway := "default" - testTargetGroupNameLenMode := "long" testAwsAccountId := "12345678" testClusterName := "cluster-name" @@ -66,13 +63,11 @@ func Test_config_init_with_all_env_var(t *testing.T) { os.Setenv(CLUSTER_VPC_ID, testClusterVpcId) os.Setenv(CLUSTER_LOCAL_GATEWAY, testClusterLocalGateway) os.Setenv(AWS_ACCOUNT_ID, testAwsAccountId) - os.Setenv(TARGET_GROUP_NAME_LEN_MODE, testTargetGroupNameLenMode) os.Setenv(CLUSTER_NAME, testClusterName) configInit(nil, ec2MetadataUnavailable()) assert.Equal(t, Region, testRegion) assert.Equal(t, VpcID, testClusterVpcId) assert.Equal(t, AccountID, testAwsAccountId) assert.Equal(t, DefaultServiceNetwork, testClusterLocalGateway) - assert.Equal(t, UseLongTGName, true) assert.Equal(t, testClusterName, ClusterName) } diff --git a/pkg/deploy/lattice/access_log_subscription_manager.go b/pkg/deploy/lattice/access_log_subscription_manager.go index d0a713d2..e1764099 100644 --- a/pkg/deploy/lattice/access_log_subscription_manager.go +++ b/pkg/deploy/lattice/access_log_subscription_manager.go @@ -205,8 +205,7 @@ func (m *defaultAccessLogSubscriptionManager) getSourceArn( } return serviceNetwork.SvcNetwork.Arn, nil case lattice.ServiceSourceType: - serviceNameProvider := services.NewDefaultLatticeServiceNameProvider(sourceName) - service, err := vpcLatticeSess.FindService(ctx, serviceNameProvider) + service, err := vpcLatticeSess.FindService(ctx, sourceName) if err != nil { return nil, err } diff --git a/pkg/deploy/lattice/access_log_subscription_manager_test.go b/pkg/deploy/lattice/access_log_subscription_manager_test.go index 89aa4b0b..c0420802 100644 --- a/pkg/deploy/lattice/access_log_subscription_manager_test.go +++ b/pkg/deploy/lattice/access_log_subscription_manager_test.go @@ -108,13 +108,12 @@ func TestAccessLogSubscriptionManager(t *testing.T) { t.Run("Create_NewALSForService_ReturnsNewALSStatus", func(t *testing.T) { accessLogSubscription := simpleAccessLogSubscription(core.CreateEvent) accessLogSubscription.Spec.SourceType = lattice.ServiceSourceType - serviceNameProvider := services.NewDefaultLatticeServiceNameProvider(sourceName) findServiceOutput := &vpclattice.ServiceSummary{ Arn: aws.String(serviceArn), Name: aws.String(sourceName), } - mockLattice.EXPECT().FindService(ctx, serviceNameProvider).Return(findServiceOutput, nil) + mockLattice.EXPECT().FindService(ctx, sourceName).Return(findServiceOutput, nil) mockLattice.EXPECT().CreateAccessLogSubscriptionWithContext(ctx, createALSForSvcInput).Return(createALSOutput, nil) mgr := NewAccessLogSubscriptionManager(gwlog.FallbackLogger, cloud) @@ -142,7 +141,6 @@ func TestAccessLogSubscriptionManager(t *testing.T) { t.Run("Create_NewALSForDeletedService_ReturnsNotFoundError", func(t *testing.T) { accessLogSubscription := simpleAccessLogSubscription(core.CreateEvent) accessLogSubscription.Spec.SourceType = lattice.ServiceSourceType - serviceNameProvider := services.NewDefaultLatticeServiceNameProvider(sourceName) findServiceOutput := &vpclattice.ServiceSummary{ Arn: aws.String(serviceArn), Name: aws.String(sourceName), @@ -152,7 +150,7 @@ func TestAccessLogSubscriptionManager(t *testing.T) { ResourceId: aws.String(serviceArn), } - mockLattice.EXPECT().FindService(ctx, serviceNameProvider).Return(findServiceOutput, nil) + mockLattice.EXPECT().FindService(ctx, sourceName).Return(findServiceOutput, nil) mockLattice.EXPECT().CreateAccessLogSubscriptionWithContext(ctx, createALSForSvcInput).Return(nil, createALSErr) mgr := NewAccessLogSubscriptionManager(gwlog.FallbackLogger, cloud) @@ -309,9 +307,8 @@ func TestAccessLogSubscriptionManager(t *testing.T) { accessLogSubscription := simpleAccessLogSubscription(core.CreateEvent) accessLogSubscription.Spec.SourceType = lattice.ServiceSourceType notFoundErr := services.NewNotFoundError("", "") - serviceNameProvider := services.NewDefaultLatticeServiceNameProvider(sourceName) - mockLattice.EXPECT().FindService(ctx, serviceNameProvider).Return(nil, notFoundErr) + mockLattice.EXPECT().FindService(ctx, sourceName).Return(nil, notFoundErr) mgr := NewAccessLogSubscriptionManager(gwlog.FallbackLogger, cloud) resp, err := mgr.Create(ctx, accessLogSubscription) diff --git a/pkg/deploy/lattice/listener_manager.go b/pkg/deploy/lattice/listener_manager.go index 1295d228..91361366 100644 --- a/pkg/deploy/lattice/listener_manager.go +++ b/pkg/deploy/lattice/listener_manager.go @@ -2,8 +2,8 @@ package lattice import ( "context" + "errors" "fmt" - "github.com/aws/aws-application-networking-k8s/pkg/aws/services" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -15,82 +15,60 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/utils" pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) type ListenerManager interface { - Cloud() pkg_aws.Cloud - Create(ctx context.Context, service *model.Listener) (model.ListenerStatus, error) - Delete(ctx context.Context, listenerID string, serviceID string) error + Upsert(ctx context.Context, modelListener *model.Listener, modelSvc *model.Service) (model.ListenerStatus, error) + Delete(ctx context.Context, modelListener *model.Listener) error List(ctx context.Context, serviceID string) ([]*vpclattice.ListenerSummary, error) } type defaultListenerManager struct { - log gwlog.Logger - cloud pkg_aws.Cloud - latticeDataStore *latticestore.LatticeDataStore + log gwlog.Logger + cloud pkg_aws.Cloud } func NewListenerManager( log gwlog.Logger, cloud pkg_aws.Cloud, - latticeDataStore *latticestore.LatticeDataStore, ) *defaultListenerManager { return &defaultListenerManager{ - log: log, - cloud: cloud, - latticeDataStore: latticeDataStore, + log: log, + cloud: cloud, } } -func (d *defaultListenerManager) Cloud() pkg_aws.Cloud { - return d.cloud -} - -type ListenerLSNProvider struct { - l *model.Listener -} - -func (r *ListenerLSNProvider) LatticeServiceName() string { - return utils.LatticeServiceName(r.l.Spec.Name, r.l.Spec.Namespace) -} - -func (d *defaultListenerManager) Create( +func (d *defaultListenerManager) Upsert( ctx context.Context, - listener *model.Listener, + modelListener *model.Listener, + modelSvc *model.Service, ) (model.ListenerStatus, error) { - listenerSpec := listener.Spec - d.log.Infof("Creating listener %s-%s", listenerSpec.Name, listenerSpec.Namespace) - - svc, err1 := d.cloud.Lattice().FindService(ctx, &ListenerLSNProvider{listener}) - if err1 != nil { - if services.IsNotFoundError(err1) { - errMsg := fmt.Sprintf("Service not found during creation of Listener %s-%s", - listenerSpec.Name, listenerSpec.Namespace) - return model.ListenerStatus{}, fmt.Errorf(errMsg) - } else { - return model.ListenerStatus{}, err1 - } + if modelSvc.Status == nil || modelSvc.Status.Id == "" { + return model.ListenerStatus{}, errors.New("model service is missing id") } - lis, err2 := d.findListenerByNamePort(ctx, *svc.Id, listener.Spec.Port) - if err2 == nil { - // update Listener - k8sName, k8sNamespace := latticeName2k8s(aws.StringValue(lis.Name)) + d.log.Infof("Upsert listener %s-%s", modelListener.Spec.K8SRouteName, modelListener.Spec.K8SRouteNamespace) + + latticeListener, err := d.findListenerByPort(ctx, modelSvc.Status.Id, modelListener.Spec.Port) + if err != nil { + return model.ListenerStatus{}, err + } + if latticeListener != nil { + // we do not support listener updates as the only mutable property + // is the default action, which we set to 404 as required by the gw spec + // so here we just return the existing one + d.log.Debugf("Found existing listener %s, nothing to update", aws.StringValue(latticeListener.Id)) return model.ListenerStatus{ - Name: k8sName, - Namespace: k8sNamespace, - Port: aws.Int64Value(lis.Port), - Protocol: aws.StringValue(lis.Protocol), - ListenerARN: aws.StringValue(lis.Arn), - ListenerID: aws.StringValue(lis.Id), - ServiceID: aws.StringValue(svc.Id), + Name: aws.StringValue(latticeListener.Name), + ListenerArn: aws.StringValue(latticeListener.Arn), + Id: aws.StringValue(latticeListener.Id), + ServiceId: modelSvc.Status.Id, }, nil } + // no listener currently exists, create defaultStatus := aws.Int64(404) - defaultResp := vpclattice.FixedResponseAction{ StatusCode: defaultStatus, } @@ -100,37 +78,36 @@ func (d *defaultListenerManager) Create( DefaultAction: &vpclattice.RuleAction{ FixedResponse: &defaultResp, }, - Name: aws.String(k8sLatticeListenerName(listener.Spec.Name, listener.Spec.Namespace, int(listener.Spec.Port), listener.Spec.Protocol)), - Port: aws.Int64(listener.Spec.Port), - Protocol: aws.String(listener.Spec.Protocol), - ServiceIdentifier: aws.String(*svc.Id), + Name: aws.String(k8sLatticeListenerName(modelListener)), + Port: aws.Int64(modelListener.Spec.Port), + Protocol: aws.String(modelListener.Spec.Protocol), + ServiceIdentifier: aws.String(modelSvc.Status.Id), Tags: d.cloud.DefaultTags(), } - resp, err := d.cloud.Lattice().CreateListener(&listenerInput) + resp, err := d.cloud.Lattice().CreateListenerWithContext(ctx, &listenerInput) if err != nil { - return model.ListenerStatus{}, err + return model.ListenerStatus{}, + fmt.Errorf("Failed CreateListener %s due to %s", aws.StringValue(listenerInput.Name), err) } + d.log.Infof("Success CreateListener %s, %s", aws.StringValue(resp.Name), aws.StringValue(resp.Id)) return model.ListenerStatus{ - Name: listener.Spec.Name, - Namespace: listener.Spec.Namespace, - ListenerARN: aws.StringValue(resp.Arn), - ListenerID: aws.StringValue(resp.Id), - ServiceID: aws.StringValue(svc.Id), - Port: listener.Spec.Port, - Protocol: listener.Spec.Protocol, + Name: aws.StringValue(resp.Name), + ListenerArn: aws.StringValue(resp.Arn), + Id: aws.StringValue(resp.Id), + ServiceId: modelSvc.Status.Id, }, nil } -func k8sLatticeListenerName(name string, namespace string, port int, protocol string) string { - listenerName := fmt.Sprintf("%s-%s-%d-%s", utils.Truncate(name, 20), utils.Truncate(namespace, 18), port, strings.ToLower(protocol)) +func k8sLatticeListenerName(modelListener *model.Listener) string { + listenerName := fmt.Sprintf("%s-%s-%d-%s", + utils.Truncate(modelListener.Spec.K8SRouteName, 20), + utils.Truncate(modelListener.Spec.K8SRouteNamespace, 18), + modelListener.Spec.Port, + strings.ToLower(modelListener.Spec.Protocol)) return listenerName } -func latticeName2k8s(name string) (string, string) { - // TODO handle namespace - return name, "default" -} func (d *defaultListenerManager) List(ctx context.Context, serviceID string) ([]*vpclattice.ListenerSummary, error) { var sdkListeners []*vpclattice.ListenerSummary @@ -140,7 +117,7 @@ func (d *defaultListenerManager) List(ctx context.Context, serviceID string) ([] ServiceIdentifier: aws.String(serviceID), } - resp, err := d.cloud.Lattice().ListListeners(&listenerListInput) + resp, err := d.cloud.Lattice().ListListenersWithContext(ctx, &listenerListInput) if err != nil { return sdkListeners, err } @@ -159,13 +136,13 @@ func (d *defaultListenerManager) List(ctx context.Context, serviceID string) ([] return sdkListeners, nil } -func (d *defaultListenerManager) findListenerByNamePort( +func (d *defaultListenerManager) findListenerByPort( ctx context.Context, - serviceId string, + latticeSvcId string, port int64, ) (*vpclattice.ListenerSummary, error) { listenerListInput := vpclattice.ListListenersInput{ - ServiceIdentifier: aws.String(serviceId), + ServiceIdentifier: aws.String(latticeSvcId), } resp, err := d.cloud.Lattice().ListListenersWithContext(ctx, &listenerListInput) @@ -175,21 +152,34 @@ func (d *defaultListenerManager) findListenerByNamePort( for _, r := range resp.Items { if aws.Int64Value(r.Port) == port { - d.log.Debugf("Port %d already in use by listener %s for service %s", port, *r.Arn, serviceId) + d.log.Debugf("Port %d already in use by listener %s for service %s", port, *r.Arn, latticeSvcId) return r, nil } } - return nil, fmt.Errorf("listener for service %s and port %d does not exist", serviceId, port) + return nil, nil } -func (d *defaultListenerManager) Delete(ctx context.Context, listenerId string, serviceId string) error { - d.log.Debugf("Deleting listener %s in service %s", listenerId, serviceId) +func (d *defaultListenerManager) Delete(ctx context.Context, modelListener *model.Listener) error { + if modelListener == nil || modelListener.Status == nil { + return errors.New("model listener and model listener status cannot be nil") + } + + d.log.Debugf("Deleting listener %s in service %s", modelListener.Status.Id, modelListener.Status.ServiceId) listenerDeleteInput := vpclattice.DeleteListenerInput{ - ServiceIdentifier: aws.String(serviceId), - ListenerIdentifier: aws.String(listenerId), + ServiceIdentifier: aws.String(modelListener.Status.ServiceId), + ListenerIdentifier: aws.String(modelListener.Status.Id), + } + + _, err := d.cloud.Lattice().DeleteListenerWithContext(ctx, &listenerDeleteInput) + if err != nil { + if services.IsLatticeAPINotFoundErr(err) { + d.log.Debugf("Listener already deleted") + return nil + } + return fmt.Errorf("Failed DeleteListener %s, %s due to %s", modelListener.Status.Id, modelListener.Status.ServiceId, err) } - _, err := d.cloud.Lattice().DeleteListener(&listenerDeleteInput) - return err + d.log.Infof("Success DeleteListener %s, %s", modelListener.Status.Id, modelListener.Status.ServiceId) + return nil } diff --git a/pkg/deploy/lattice/listener_manager_mock.go b/pkg/deploy/lattice/listener_manager_mock.go index 1fb8664e..ba8d5126 100644 --- a/pkg/deploy/lattice/listener_manager_mock.go +++ b/pkg/deploy/lattice/listener_manager_mock.go @@ -8,7 +8,6 @@ import ( context "context" reflect "reflect" - aws "github.com/aws/aws-application-networking-k8s/pkg/aws" lattice "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" vpclattice "github.com/aws/aws-sdk-go/service/vpclattice" gomock "github.com/golang/mock/gomock" @@ -37,47 +36,18 @@ func (m *MockListenerManager) EXPECT() *MockListenerManagerMockRecorder { return m.recorder } -// Cloud mocks base method. -func (m *MockListenerManager) Cloud() aws.Cloud { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Cloud") - ret0, _ := ret[0].(aws.Cloud) - return ret0 -} - -// Cloud indicates an expected call of Cloud. -func (mr *MockListenerManagerMockRecorder) Cloud() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cloud", reflect.TypeOf((*MockListenerManager)(nil).Cloud)) -} - -// Create mocks base method. -func (m *MockListenerManager) Create(ctx context.Context, service *lattice.Listener) (lattice.ListenerStatus, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", ctx, service) - ret0, _ := ret[0].(lattice.ListenerStatus) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Create indicates an expected call of Create. -func (mr *MockListenerManagerMockRecorder) Create(ctx, service interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockListenerManager)(nil).Create), ctx, service) -} - // Delete mocks base method. -func (m *MockListenerManager) Delete(ctx context.Context, listenerID, serviceID string) error { +func (m *MockListenerManager) Delete(ctx context.Context, modelListener *lattice.Listener) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", ctx, listenerID, serviceID) + ret := m.ctrl.Call(m, "Delete", ctx, modelListener) ret0, _ := ret[0].(error) return ret0 } // Delete indicates an expected call of Delete. -func (mr *MockListenerManagerMockRecorder) Delete(ctx, listenerID, serviceID interface{}) *gomock.Call { +func (mr *MockListenerManagerMockRecorder) Delete(ctx, modelListener interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockListenerManager)(nil).Delete), ctx, listenerID, serviceID) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockListenerManager)(nil).Delete), ctx, modelListener) } // List mocks base method. @@ -94,3 +64,18 @@ func (mr *MockListenerManagerMockRecorder) List(ctx, serviceID interface{}) *gom mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockListenerManager)(nil).List), ctx, serviceID) } + +// Upsert mocks base method. +func (m *MockListenerManager) Upsert(ctx context.Context, modelListener *lattice.Listener, modelSvc *lattice.Service) (lattice.ListenerStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Upsert", ctx, modelListener, modelSvc) + ret0, _ := ret[0].(lattice.ListenerStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Upsert indicates an expected call of Upsert. +func (mr *MockListenerManagerMockRecorder) Upsert(ctx, modelListener, modelSvc interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockListenerManager)(nil).Upsert), ctx, modelListener, modelSvc) +} diff --git a/pkg/deploy/lattice/listener_manager_test.go b/pkg/deploy/lattice/listener_manager_test.go index 760eb865..ccce81cf 100644 --- a/pkg/deploy/lattice/listener_manager_test.go +++ b/pkg/deploy/lattice/listener_manager_test.go @@ -2,288 +2,109 @@ package lattice import ( "context" - "errors" - "fmt" - - "github.com/aws/aws-sdk-go/aws" - "k8s.io/apimachinery/pkg/types" - - "testing" - - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - - "github.com/aws/aws-sdk-go/service/vpclattice" - - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" mocks "github.com/aws/aws-application-networking-k8s/pkg/aws/services" - + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/vpclattice" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + "testing" ) -var namespaceName = types.NamespacedName{ - Namespace: "default", - Name: "test", -} -var listenerSummaries = []struct { - Arn string - Id string - Name string - Port int64 - Protocol string -}{ - { - Arn: "arn-1", - Id: "id-1", - Name: namespaceName.Name, - Port: 80, - Protocol: "HTTP", - }, - { - Arn: "arn-2", - Id: "Id-2", - Name: namespaceName.Name, - Port: 443, - Protocol: "HTTPS", - }, -} -var summaries = []vpclattice.ListenerSummary{ - { - Arn: &listenerSummaries[0].Arn, - Id: &listenerSummaries[0].Id, - Name: &listenerSummaries[0].Name, - Port: &listenerSummaries[0].Port, - Protocol: &listenerSummaries[0].Protocol, - }, - { - Arn: &listenerSummaries[1].Arn, - Id: &listenerSummaries[1].Id, - Name: &listenerSummaries[1].Name, - Port: &listenerSummaries[1].Port, - Protocol: &listenerSummaries[1].Protocol, - }, -} -var listenerList = vpclattice.ListListenersOutput{ - Items: []*vpclattice.ListenerSummary{ - &summaries[0], - &summaries[1], - }, -} - -func Test_AddListener(t *testing.T) { - - tests := []struct { - name string - isUpdate bool - noServiceID bool - }{ - { - name: "add listener", - isUpdate: false, - noServiceID: false, - }, - - { - name: "update listener", - isUpdate: true, - noServiceID: false, - }, +func Test_CreateListenerNew(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - { - name: "add listener, no service ID", - isUpdate: false, - noServiceID: true, - }, + ml := &model.Listener{} + ms := &model.Service{ + Status: &model.ServiceStatus{Id: "svc-id"}, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - latticeDataStore := latticestore.NewLatticeDataStore() - listenerManager := NewListenerManager(gwlog.FallbackLogger, cloud, latticeDataStore) - - var serviceID = "serviceID" - var serviceARN = "serviceARN" - var serviceDNS = "DNS-test" - - stack := core.NewDefaultStack(core.StackID(namespaceName)) - - action := model.DefaultAction{ - BackendServiceName: "tg-test", - BackendServiceNamespace: "tg-default", - } + mockLattice.EXPECT().ListListenersWithContext(ctx, gomock.Any()).Return( + &vpclattice.ListListenersOutput{Items: []*vpclattice.ListenerSummary{}}, nil) - listenerResourceName := fmt.Sprintf("%s-%s-%d-%s", namespaceName.Name, namespaceName.Namespace, - listenerSummaries[0].Port, "HTTP") + mockLattice.EXPECT().CreateListenerWithContext(ctx, gomock.Any()).DoAndReturn( + func(ctx aws.Context, input *vpclattice.CreateListenerInput, opts ...request.Option) (*vpclattice.CreateListenerOutput, error) { + // part of the gw spec is to default to 404, so assert that here + assert.Equal(t, int64(404), *input.DefaultAction.FixedResponse.StatusCode) + assert.Equal(t, "svc-id", *input.ServiceIdentifier) - listener := model.NewListener(stack, listenerResourceName, listenerSummaries[0].Port, "HTTP", - namespaceName.Name, namespaceName.Namespace, action) - - if !tt.noServiceID { - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String((&ListenerLSNProvider{listener}).LatticeServiceName()), - Arn: aws.String(serviceARN), - Id: aws.String(serviceID), - DnsEntry: &vpclattice.DnsEntry{ - DomainName: aws.String(serviceDNS), - HostedZoneId: aws.String("my-favourite-zone"), - }, - }, nil).Times(1) - } else { - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return(nil, &mocks.NotFoundError{}).Times(1) - } - - listenerOutput := vpclattice.CreateListenerOutput{} - listenerInput := vpclattice.CreateListenerInput{} - - defaultStatus := aws.Int64(404) - - defaultResp := vpclattice.FixedResponseAction{ - StatusCode: defaultStatus, - } - defaultAction := vpclattice.RuleAction{ - FixedResponse: &defaultResp, - } - - if !tt.noServiceID && !tt.isUpdate { - listenerName := k8sLatticeListenerName(namespaceName.Name, namespaceName.Namespace, - int(listenerSummaries[0].Port), listenerSummaries[0].Protocol) - listenerInput = vpclattice.CreateListenerInput{ - DefaultAction: &defaultAction, - Name: &listenerName, - ServiceIdentifier: &serviceID, - Protocol: aws.String("HTTP"), - Port: aws.Int64(listenerSummaries[0].Port), - Tags: cloud.DefaultTags(), - } - listenerOutput = vpclattice.CreateListenerOutput{ - Arn: &listenerSummaries[0].Arn, - DefaultAction: &defaultAction, - Id: &listenerSummaries[0].Id, - } - mockLattice.EXPECT().CreateListener(&listenerInput).Return(&listenerOutput, nil) - } - - if !tt.noServiceID { - listenerListInput := vpclattice.ListListenersInput{ - ServiceIdentifier: aws.String(serviceID), - } - - listenerOutput := vpclattice.ListListenersOutput{} - - if tt.isUpdate { - listenerOutput = listenerList - } - - mockLattice.EXPECT().ListListenersWithContext(ctx, &listenerListInput).Return(&listenerOutput, nil) - } - resp, err := listenerManager.Create(ctx, listener) - - if !tt.noServiceID { - assert.NoError(t, err) - - assert.Equal(t, resp.ListenerARN, listenerSummaries[0].Arn) - assert.Equal(t, resp.ListenerID, listenerSummaries[0].Id) - assert.Equal(t, resp.Name, namespaceName.Name) - assert.Equal(t, resp.Namespace, namespaceName.Namespace) - assert.Equal(t, resp.Port, listenerSummaries[0].Port) - assert.Equal(t, resp.Protocol, "HTTP") - } - - fmt.Printf("listener create : resp %v, err %v, listenerOutput %v\n", resp, err, listenerOutput) + return &vpclattice.CreateListenerOutput{Id: aws.String("new-lid")}, nil + }, + ) - if tt.noServiceID { - assert.NotNil(t, err) - } - }) - } + lm := NewListenerManager(gwlog.FallbackLogger, cloud) + status, err := lm.Upsert(ctx, ml, ms) + assert.Nil(t, err) + assert.Equal(t, "new-lid", status.Id) + assert.Equal(t, "svc-id", status.ServiceId) } -func Test_ListListener(t *testing.T) { +func Test_CreateListenerExisting(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - tests := []struct { - Name string - mgrErr error - }{ - { - Name: "listener LIST API call ok", - mgrErr: nil, - }, - { - Name: "listener List API call return NOK", - mgrErr: errors.New("call failed"), + ml := &model.Listener{ + Spec: model.ListenerSpec{ + Port: 8181, }, } - - for _, tt := range tests { - t.Run(tt.Name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, pkg_aws.CloudConfig{}) - - latticeDataStore := latticestore.NewLatticeDataStore() - listenerManager := NewListenerManager(gwlog.FallbackLogger, cloud, latticeDataStore) - - serviceID := "service1-ID" - listenerListInput := vpclattice.ListListenersInput{ - ServiceIdentifier: aws.String(serviceID), - } - mockLattice.EXPECT().ListListeners(&listenerListInput).Return(&listenerList, tt.mgrErr) - - resp, err := listenerManager.List(ctx, serviceID) - fmt.Printf("listener list :%v, err: %v \n", resp, err) - - if err == nil { - var i = 0 - for _, rsp := range resp { - assert.Equal(t, *rsp.Arn, *listenerList.Items[i].Arn) - i++ - } - } else { - assert.Equal(t, err, tt.mgrErr) - } - }) + ms := &model.Service{ + Status: &model.ServiceStatus{Id: "svc-id"}, } + + mockLattice.EXPECT().ListListenersWithContext(ctx, gomock.Any()).Return( + &vpclattice.ListListenersOutput{Items: []*vpclattice.ListenerSummary{ + { + Arn: aws.String("existing-arn"), + Id: aws.String("existing-lid"), + Name: aws.String("existing-name"), + Port: aws.Int64(8181), + }, + }}, nil) + + lm := NewListenerManager(gwlog.FallbackLogger, cloud) + status, err := lm.Upsert(ctx, ml, ms) + assert.Nil(t, err) + assert.Equal(t, "existing-lid", status.Id) + assert.Equal(t, "svc-id", status.ServiceId) + assert.Equal(t, "existing-name", status.Name) } func Test_DeleteListener(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - serviceID := "service1-ID" - listenerID := "listener-ID" - - listenerDeleteInput := vpclattice.DeleteListenerInput{ - ServiceIdentifier: aws.String(serviceID), - ListenerIdentifier: aws.String(listenerID), - } - - latticeDataStore := latticestore.NewLatticeDataStore() - - listenerDeleteOutput := vpclattice.DeleteListenerOutput{} - mockLattice.EXPECT().DeleteListener(&listenerDeleteInput).Return(&listenerDeleteOutput, nil) - - listenerManager := NewListenerManager(gwlog.FallbackLogger, cloud, latticeDataStore) - - err := listenerManager.Delete(ctx, listenerID, serviceID) - assert.Nil(t, err) + t.Run("test precondition errors", func(t *testing.T) { + ml := &model.Listener{} + + lm := NewListenerManager(gwlog.FallbackLogger, cloud) + assert.Error(t, lm.Delete(ctx, nil)) + assert.Error(t, lm.Delete(ctx, ml)) + }) + + t.Run("listener only success", func(t *testing.T) { + ml := &model.Listener{Status: &model.ListenerStatus{Id: "lid", ServiceId: "sid"}} + mockLattice.EXPECT().DeleteListenerWithContext(ctx, gomock.Any()).DoAndReturn( + func(ctx aws.Context, input *vpclattice.DeleteListenerInput, opts ...request.Option) (*vpclattice.DeleteListenerOutput, error) { + assert.Equal(t, "lid", *input.ListenerIdentifier) + assert.Equal(t, "sid", *input.ServiceIdentifier) + return &vpclattice.DeleteListenerOutput{}, nil + }, + ) + lm := NewListenerManager(gwlog.FallbackLogger, cloud) + assert.NoError(t, lm.Delete(ctx, ml)) + }) } diff --git a/pkg/deploy/lattice/listener_synthesizer.go b/pkg/deploy/lattice/listener_synthesizer.go index d660daba..43d65baf 100644 --- a/pkg/deploy/lattice/listener_synthesizer.go +++ b/pkg/deploy/lattice/listener_synthesizer.go @@ -4,128 +4,137 @@ import ( "context" "errors" "fmt" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" ) type listenerSynthesizer struct { - log gwlog.Logger - listenerMgr ListenerManager - stack core.Stack - latticestore *latticestore.LatticeDataStore + log gwlog.Logger + listenerMgr ListenerManager + stack core.Stack } func NewListenerSynthesizer( log gwlog.Logger, ListenerManager ListenerManager, stack core.Stack, - store *latticestore.LatticeDataStore, ) *listenerSynthesizer { return &listenerSynthesizer{ - log: log, - listenerMgr: ListenerManager, - stack: stack, - latticestore: store, + log: log, + listenerMgr: ListenerManager, + stack: stack, } } func (l *listenerSynthesizer) Synthesize(ctx context.Context) error { - var resListener []*model.Listener + var stackListeners []*model.Listener - err := l.stack.ListResources(&resListener) + err := l.stack.ListResources(&stackListeners) if err != nil { - l.log.Errorf("Failed to list stack Listeners during Listener synthesis due to err %s", err) + return err } - for _, listener := range resListener { - l.log.Debugf("Attempting to synthesize listener %s-%s", listener.Spec.Name, listener.Spec.Namespace) - status, err := l.listenerMgr.Create(ctx, listener) + for _, listener := range stackListeners { + resSvc, err := l.stack.GetResource(listener.Spec.StackServiceId, &model.Service{}) if err != nil { - errmsg := fmt.Sprintf("failed to create listener %s-%s during synthesis due to err %s", - listener.Spec.Name, listener.Spec.Namespace, err) - return errors.New(errmsg) + return err + } + + svc, ok := resSvc.(*model.Service) + if !ok { + return errors.New("unexpected type conversion failure for service stack object") } - l.log.Debugf("Successfully synthesized listener %s-%s", listener.Spec.Name, listener.Spec.Namespace) - l.latticestore.AddListener(listener.Spec.Name, listener.Spec.Namespace, listener.Spec.Port, - listener.Spec.Protocol, status.ListenerARN, status.ListenerID) + // deletes are deferred to the later logic comparing existing listeners + // to current listeners. + if !listener.IsDeleted { + status, err := l.listenerMgr.Upsert(ctx, listener, svc) + if err != nil { + return fmt.Errorf("Failed ListenerManager.Upsert %s-%s due to err %s", + listener.Spec.K8SRouteName, listener.Spec.K8SRouteNamespace, err) + } + + listener.Status = &status + } } - // handle delete - sdkListeners, err := l.getSDKListeners(ctx) + // All deletions happen here, we fetch all listeners for NON-deleted + // services, since service deletion will delete its listeners + latticeListenersAsModel, err := l.getLatticeListenersAsModels(ctx) if err != nil { - l.log.Debugf("Failed to get SDK Listeners during Listener synthesis due to err %s", err) + return err } - for _, sdkListener := range sdkListeners { - _, err := l.findMatchListener(ctx, sdkListener, resListener) - if err == nil { - continue - } - - l.log.Debugf("Deleting stale SDK Listener %s-%s", sdkListener.Name, sdkListener.Namespace) - err = l.listenerMgr.Delete(ctx, sdkListener.ListenerID, sdkListener.ServiceID) - if err != nil { - l.log.Errorf("Failed to delete SDK Listener %s", sdkListener.ListenerID) + for _, latticeListenerAsModel := range latticeListenersAsModel { + if l.shouldDelete(latticeListenerAsModel, stackListeners) { + err = l.listenerMgr.Delete(ctx, latticeListenerAsModel) + if err != nil { + l.log.Infof("Failed ListenerManager.Delete %s due to %s", latticeListenerAsModel.Status.Id, err) + } } - - k8sName, k8sNamespace := latticeName2k8s(sdkListener.Name) - l.latticestore.DelListener(k8sName, k8sNamespace, sdkListener.Port, sdkListener.Protocol) } return nil } -func (l *listenerSynthesizer) findMatchListener( - ctx context.Context, - sdkListener *model.ListenerStatus, - resListener []*model.Listener, -) (model.Listener, error) { - for _, moduleListener := range resListener { - if moduleListener.Spec.Port == sdkListener.Port && moduleListener.Spec.Protocol == sdkListener.Protocol { - return *moduleListener, nil +func (l *listenerSynthesizer) shouldDelete(listenerToFind *model.Listener, stackListeners []*model.Listener) bool { + for _, candidate := range stackListeners { + if candidate.Spec.Port == listenerToFind.Spec.Port && candidate.Spec.Protocol == listenerToFind.Spec.Protocol { + // found a match, delete if match is deleted + return candidate.IsDeleted } } - return model.Listener{}, errors.New("failed to find matching listener in model") + // there is no matching listener + return true } -func (l *listenerSynthesizer) getSDKListeners(ctx context.Context) ([]*model.ListenerStatus, error) { - var sdkListeners []*model.ListenerStatus +// retrieves all the listeners for all the non-deleted services currently in the stack +func (l *listenerSynthesizer) getLatticeListenersAsModels(ctx context.Context) ([]*model.Listener, error) { + var latticeListenersAsModel []*model.Listener + var modelSvcs []*model.Service - var resService []*model.Service - - err := l.stack.ListResources(&resService) + err := l.stack.ListResources(&modelSvcs) if err != nil { - l.log.Debugf("Ignoring error when listing services %s", err) + return latticeListenersAsModel, err } - for _, service := range resService { - latticeService, err := l.listenerMgr.Cloud().Lattice().FindService(ctx, service) - if err != nil { - errMessage := fmt.Sprintf("failed to find service in store for service %s-%s due to err %s", - service.Spec.Name, service.Spec.Namespace, err) - return sdkListeners, errors.New(errMessage) + // get the listeners for each service + for _, modelSvc := range modelSvcs { + if modelSvc.IsDeleted { + l.log.Debugf("Ignoring deleted service %s", modelSvc.LatticeServiceName()) + continue } - listenerSummaries, err := l.listenerMgr.List(ctx, aws.StringValue(latticeService.Id)) - for _, listenerSummary := range listenerSummaries { - sdkListeners = append(sdkListeners, &model.ListenerStatus{ - Name: aws.StringValue(listenerSummary.Name), - ListenerARN: aws.StringValue(listenerSummary.Arn), - ListenerID: aws.StringValue(listenerSummary.Id), - ServiceID: aws.StringValue(latticeService.Id), - Port: aws.Int64Value(listenerSummary.Port), - Protocol: aws.StringValue(listenerSummary.Protocol), + listenerSummaries, err := l.listenerMgr.List(ctx, modelSvc.Status.Id) + if err != nil { + l.log.Infof("Ignoring error when listing listeners %s", err) + continue + } + for _, latticeListener := range listenerSummaries { + + spec := model.ListenerSpec{ + StackServiceId: modelSvc.ID(), + Port: aws.Int64Value(latticeListener.Port), + Protocol: aws.StringValue(latticeListener.Protocol), + } + status := model.ListenerStatus{ + Name: aws.StringValue(latticeListener.Name), + ListenerArn: aws.StringValue(latticeListener.Arn), + Id: aws.StringValue(latticeListener.Id), + ServiceId: modelSvc.Status.Id, + } + + latticeListenersAsModel = append(latticeListenersAsModel, &model.Listener{ + Spec: spec, + Status: &status, }) } } - return sdkListeners, nil + return latticeListenersAsModel, nil } func (l *listenerSynthesizer) PostSynthesize(ctx context.Context) error { diff --git a/pkg/deploy/lattice/listener_synthesizer_test.go b/pkg/deploy/lattice/listener_synthesizer_test.go index 032f4631..b12aff71 100644 --- a/pkg/deploy/lattice/listener_synthesizer_test.go +++ b/pkg/deploy/lattice/listener_synthesizer_test.go @@ -2,204 +2,120 @@ package lattice import ( "context" - - mocks_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - mocks "github.com/aws/aws-application-networking-k8s/pkg/aws/services" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - "github.com/aws/aws-sdk-go/aws" - - //"errors" - "fmt" - "testing" - + "github.com/aws/aws-sdk-go/service/vpclattice" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "testing" +) - "github.com/aws/aws-sdk-go/service/vpclattice" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +func Test_SynthesizeListenerCreate(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockListenerMgr := NewMockListenerManager(c) - "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" -) + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) -// PortNumberPtr translates an int to a *PortNumber -func PortNumberPtr(p int) *gwv1beta1.PortNumber { - result := gwv1beta1.PortNumber(p) - return &result + svc := &model.Service{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Service", "stack-svc-id"), + Status: &model.ServiceStatus{Id: "svc-id"}, + } + assert.NoError(t, stack.AddResource(svc)) + + l := &model.Listener{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Listener", "l-id"), + Spec: model.ListenerSpec{ + StackServiceId: "stack-svc-id", + }, + } + assert.NoError(t, stack.AddResource(l)) + + mockListenerMgr.EXPECT().Upsert(ctx, gomock.Any(), gomock.Any()).Return( + model.ListenerStatus{Id: "new-listener-id"}, nil) + + mockListenerMgr.EXPECT().List(ctx, gomock.Any()).Return([]*vpclattice.ListenerSummary{}, nil) + + ls := NewListenerSynthesizer(gwlog.FallbackLogger, mockListenerMgr, stack) + err := ls.Synthesize(ctx) + assert.Nil(t, err) } -func Test_SynthesizeListener(t *testing.T) { - - tests := []struct { - name string - gwListenerPort gwv1beta1.PortNumber - gwProtocol string - httpRoute *gwv1beta1.HTTPRoute - listenerARN string - listenerID string - serviceARN string - serviceID string - mgrErr error - wantErrIsNil bool - wantIsDeleted bool - }{ - { - name: "Add Listener", - gwListenerPort: *PortNumberPtr(80), - gwProtocol: "HTTP", - httpRoute: &gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gateway1", - }, - }, - }, - }, - }, - listenerARN: "arn1234", - listenerID: "1234", - serviceARN: "arn56789", - serviceID: "56789", - mgrErr: nil, - wantIsDeleted: false, - wantErrIsNil: true, + +func Test_SynthesizeListenerDeleteNoOp(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockListenerMgr := NewMockListenerManager(c) + + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) + + svc := &model.Service{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Service", "stack-svc-id"), + Status: &model.ServiceStatus{Id: "svc-id"}, + } + assert.NoError(t, stack.AddResource(svc)) + + l := &model.Listener{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Listener", "l-id"), + Spec: model.ListenerSpec{ + StackServiceId: "stack-svc-id", }, - { - name: "Delete Listener", - gwListenerPort: *PortNumberPtr(80), - gwProtocol: "HTTP", - httpRoute: &gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service2", - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gateway2", - }, - }, - }, - }, - }, - listenerARN: "arn1234", - listenerID: "1234", - serviceARN: "arn56789", - serviceID: "56789", - mgrErr: nil, - wantIsDeleted: true, - wantErrIsNil: true, + IsDeleted: true, // <-- the bit that matters + } + assert.NoError(t, stack.AddResource(l)) + + // since we aren't returning any resources, we interpret that to mean the listener is already deleted + mockListenerMgr.EXPECT().List(ctx, gomock.Any()).Return([]*vpclattice.ListenerSummary{}, nil) + + ls := NewListenerSynthesizer(gwlog.FallbackLogger, mockListenerMgr, stack) + err := ls.Synthesize(ctx) + assert.Nil(t, err) +} + +func Test_SynthesizeListenerCreateWithReconcile(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockListenerMgr := NewMockListenerManager(c) + + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) + + svc := &model.Service{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Service", "stack-svc-id"), + Status: &model.ServiceStatus{Id: "svc-id"}, + } + assert.NoError(t, stack.AddResource(svc)) + + l := &model.Listener{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Listener", "l-id"), + Spec: model.ListenerSpec{ + StackServiceId: "stack-svc-id", + Port: 80, }, } - var protocol = "HTTP" - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - ds := latticestore.NewLatticeDataStore() - - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.httpRoute))) - - mockListenerManager := NewMockListenerManager(c) - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := mocks.NewMockLattice(c) - - mockListenerManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() - - pro := "HTTP" - protocols := []*string{&pro} - spec := model.ServiceSpec{ - Name: tt.httpRoute.Name, - Namespace: tt.httpRoute.Namespace, - Protocols: protocols, - } - - if tt.httpRoute.DeletionTimestamp.IsZero() { - spec.IsDeleted = false - } else { - spec.IsDeleted = true - } - - action := model.DefaultAction{ - BackendServiceName: "test", - BackendServiceNamespace: "default", - } - - stackService := model.NewLatticeService(stack, "", spec) - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String(stackService.LatticeServiceName()), - Arn: aws.String("svc-arn"), - Id: aws.String(tt.serviceID), - }, nil) - - port := int64(tt.gwListenerPort) - - mockListenerManager.EXPECT().List(ctx, tt.serviceID).Return( - []*vpclattice.ListenerSummary{ - { - Arn: &tt.listenerARN, - Id: &tt.listenerID, - Name: &tt.httpRoute.Name, - Port: &port, - Protocol: &protocol, - }, - }, tt.mgrErr) - - if !tt.wantIsDeleted { - listenerResourceName := fmt.Sprintf("%s-%s-%d-%s", tt.httpRoute.Name, tt.httpRoute.Namespace, - tt.gwListenerPort, protocol) - listener := model.NewListener(stack, listenerResourceName, int64(tt.gwListenerPort), tt.gwProtocol, - tt.httpRoute.Name, tt.httpRoute.Namespace, action) - - mockListenerManager.EXPECT().Create(ctx, listener).Return(model.ListenerStatus{ - Name: tt.httpRoute.Name, - Namespace: tt.httpRoute.Namespace, - ListenerARN: tt.listenerARN, - ListenerID: tt.listenerID, - ServiceID: tt.serviceID, - Port: int64(tt.gwListenerPort), - Protocol: tt.gwProtocol}, tt.mgrErr) - } else { - mockListenerManager.EXPECT().Delete(ctx, tt.listenerID, tt.serviceID).Return(tt.mgrErr) - } - - synthesizer := NewListenerSynthesizer(gwlog.FallbackLogger, mockListenerManager, stack, ds) - - err := synthesizer.Synthesize(ctx) - - if tt.wantErrIsNil { - assert.Nil(t, err) - } - - if !tt.wantIsDeleted { - listener, err := ds.GetlListener(spec.Name, spec.Namespace, int64(tt.gwListenerPort), tt.gwProtocol) - assert.Nil(t, err) - fmt.Printf("listener: %v \n", listener) - assert.Equal(t, listener.ARN, tt.listenerARN) - assert.Equal(t, listener.ID, tt.listenerID) - assert.Equal(t, listener.Key.Name, tt.httpRoute.Name) - assert.Equal(t, listener.Key.Namespace, tt.httpRoute.Namespace) - assert.Equal(t, listener.Key.Port, int64(tt.gwListenerPort)) - } else { - assert.Nil(t, err) - - // make sure listener is also deleted from datastore - _, err := ds.GetlListener(spec.Name, spec.Namespace, int64(tt.gwListenerPort), tt.gwProtocol) - assert.NotNil(t, err) - } + assert.NoError(t, stack.AddResource(l)) + + mockListenerMgr.EXPECT().Upsert(ctx, gomock.Any(), gomock.Any()).Return( + model.ListenerStatus{Id: "new-listener-id"}, nil) + + mockListenerMgr.EXPECT().List(ctx, gomock.Any()).Return([]*vpclattice.ListenerSummary{ + { + Id: aws.String("to-delete-id"), + Port: aws.Int64(443), // <-- makes this listener unique + }, + }, nil) + + mockListenerMgr.EXPECT().Delete(ctx, gomock.Any()).DoAndReturn( + func(ctx context.Context, ml *model.Listener) error { + assert.Equal(t, "to-delete-id", ml.Status.Id) + assert.Equal(t, "svc-id", ml.Status.ServiceId) + return nil }) - } + + ls := NewListenerSynthesizer(gwlog.FallbackLogger, mockListenerMgr, stack) + err := ls.Synthesize(ctx) + assert.Nil(t, err) } diff --git a/pkg/deploy/lattice/rule_manager.go b/pkg/deploy/lattice/rule_manager.go index eb657055..8ce88829 100644 --- a/pkg/deploy/lattice/rule_manager.go +++ b/pkg/deploy/lattice/rule_manager.go @@ -4,11 +4,8 @@ import ( "context" "errors" "fmt" - - "github.com/aws/aws-application-networking-k8s/pkg/utils" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - - "strings" + "reflect" "github.com/aws/aws-sdk-go/aws" @@ -16,49 +13,32 @@ import ( pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) type RuleManager interface { - Cloud() pkg_aws.Cloud - Create(ctx context.Context, rule *model.Rule) (model.RuleStatus, error) - Delete(ctx context.Context, ruleId string, listenerId string, serviceId string) error - Update(ctx context.Context, rules []*model.Rule) error - List(ctx context.Context, serviceId string, listenerId string) ([]*model.RuleStatus, error) + Upsert(ctx context.Context, modelRule *model.Rule, modelListener *model.Listener, modelSvc *model.Service) (model.RuleStatus, error) + Delete(ctx context.Context, ruleId string, serviceId string, listenerId string) error + UpdatePriorities(ctx context.Context, svcId string, listenerId string, rules []*model.Rule) error + List(ctx context.Context, serviceId string, listenerId string) ([]*vpclattice.RuleSummary, error) Get(ctx context.Context, serviceId string, listenerId string, ruleId string) (*vpclattice.GetRuleOutput, error) } type defaultRuleManager struct { - log gwlog.Logger - cloud pkg_aws.Cloud - latticeDataStore *latticestore.LatticeDataStore + log gwlog.Logger + cloud pkg_aws.Cloud } func NewRuleManager( log gwlog.Logger, cloud pkg_aws.Cloud, - store *latticestore.LatticeDataStore, ) *defaultRuleManager { return &defaultRuleManager{ - log: log, - cloud: cloud, - latticeDataStore: store, + log: log, + cloud: cloud, } } -func (r *defaultRuleManager) Cloud() pkg_aws.Cloud { - return r.cloud -} - -type RuleLSNProvider struct { - rule *model.Rule -} - -func (r *RuleLSNProvider) LatticeServiceName() string { - return utils.LatticeServiceName(r.rule.Spec.ServiceName, r.rule.Spec.ServiceNamespace) -} - func (r *defaultRuleManager) Get(ctx context.Context, serviceId string, listenerId string, ruleId string) (*vpclattice.GetRuleOutput, error) { getRuleInput := vpclattice.GetRuleInput{ ListenerIdentifier: aws.String(listenerId), @@ -66,62 +46,26 @@ func (r *defaultRuleManager) Get(ctx context.Context, serviceId string, listener RuleIdentifier: aws.String(ruleId), } - resp, err := r.cloud.Lattice().GetRule(&getRuleInput) + resp, err := r.cloud.Lattice().GetRuleWithContext(ctx, &getRuleInput) return resp, err } -// find out all rules in SDK lattice under a single service -func (r *defaultRuleManager) List(ctx context.Context, service string, listener string) ([]*model.RuleStatus, error) { - var sdkRules []*model.RuleStatus = nil - +func (r *defaultRuleManager) List(ctx context.Context, svcId string, listenerId string) ([]*vpclattice.RuleSummary, error) { ruleListInput := vpclattice.ListRulesInput{ - ListenerIdentifier: aws.String(listener), - ServiceIdentifier: aws.String(service), - } - - var resp *vpclattice.ListRulesOutput - resp, err := r.cloud.Lattice().ListRules(&ruleListInput) - if err != nil { - return sdkRules, err - } - - for _, ruleSum := range resp.Items { - if !aws.BoolValue(ruleSum.IsDefault) { - sdkRules = append(sdkRules, &model.RuleStatus{ - RuleID: aws.StringValue(ruleSum.Id), - ServiceID: service, - ListenerID: listener, - }) - } + ServiceIdentifier: aws.String(svcId), + ListenerIdentifier: aws.String(listenerId), } - return sdkRules, nil + return r.cloud.Lattice().ListRulesAsList(ctx, &ruleListInput) } -// today, it only batch update the priority -func (r *defaultRuleManager) Update(ctx context.Context, rules []*model.Rule) error { - firstRuleSpec := rules[0].Spec +func (r *defaultRuleManager) UpdatePriorities(ctx context.Context, svcId string, listenerId string, rules []*model.Rule) error { var ruleUpdateList []*vpclattice.RuleUpdate - latticeService, err := r.cloud.Lattice().FindService(ctx, &RuleLSNProvider{rules[0]}) - if err != nil { - return fmt.Errorf("service %s-%s not found during rule creation", - firstRuleSpec.ServiceName, firstRuleSpec.ServiceNamespace) - } - - listener, err := r.latticeDataStore.GetlListener(firstRuleSpec.ServiceName, firstRuleSpec.ServiceNamespace, - firstRuleSpec.ListenerPort, firstRuleSpec.ListenerProtocol) - - if err != nil { - return fmt.Errorf("listener not found during rule creation for service %s-%s, port %d, protocol %s", - firstRuleSpec.ServiceName, firstRuleSpec.ServiceNamespace, firstRuleSpec.ListenerPort, firstRuleSpec.ListenerProtocol) - } - for _, rule := range rules { - priority, _ := ruleID2Priority(rule.Spec.RuleID) ruleUpdate := vpclattice.RuleUpdate{ - RuleIdentifier: aws.String(rule.Status.RuleID), - Priority: aws.Int64(priority), + RuleIdentifier: aws.String(rule.Status.Id), + Priority: aws.Int64(rule.Spec.Priority), } ruleUpdateList = append(ruleUpdateList, &ruleUpdate) @@ -129,408 +73,292 @@ func (r *defaultRuleManager) Update(ctx context.Context, rules []*model.Rule) er // BatchUpdate rules using right priority batchRuleInput := vpclattice.BatchUpdateRuleInput{ - ListenerIdentifier: aws.String(listener.ID), - ServiceIdentifier: aws.String(*latticeService.Id), + ServiceIdentifier: aws.String(svcId), + ListenerIdentifier: aws.String(listenerId), Rules: ruleUpdateList, } - _, err = r.cloud.Lattice().BatchUpdateRule(&batchRuleInput) - return err -} - -func (r *defaultRuleManager) Create(ctx context.Context, rule *model.Rule) (model.RuleStatus, error) { - r.log.Debugf("Creating rule %s for service %s-%s and listener port %d and protocol %s", - rule.Spec.RuleID, rule.Spec.ServiceName, rule.Spec.ServiceNamespace, - rule.Spec.ListenerPort, rule.Spec.ListenerProtocol) - - latticeService, err := r.cloud.Lattice().FindService(ctx, &RuleLSNProvider{rule}) + _, err := r.cloud.Lattice().BatchUpdateRuleWithContext(ctx, &batchRuleInput) if err != nil { - return model.RuleStatus{}, err + r.log.Infof("Failed BatchUpdateRule %s, %s, due to %s", svcId, listenerId, err) + return err } - listener, err := r.latticeDataStore.GetlListener(rule.Spec.ServiceName, rule.Spec.ServiceNamespace, - rule.Spec.ListenerPort, rule.Spec.ListenerProtocol) - if err != nil { - return model.RuleStatus{}, err - } - - priority, err := ruleID2Priority(rule.Spec.RuleID) - if err != nil { - return model.RuleStatus{}, fmt.Errorf("failed to create rule due to invalid ruleId, err: %s", err) - } - r.log.Debugf("Converted rule id %s to priority %d", rule.Spec.RuleID, priority) + r.log.Infof("Success BatchUpdateRule %s, %s", svcId, listenerId) + return nil +} - ruleStatus, err := r.findMatchingRule(ctx, rule, *latticeService.Id, listener.ID) - if err == nil && !ruleStatus.UpdateTGsNeeded { - if ruleStatus.Priority != priority { - r.log.Debugf("Need to BatchUpdate priority for rule %s", rule.Spec.RuleID) - ruleStatus.UpdatePriorityNeeded = true - } - return ruleStatus, nil +func (r *defaultRuleManager) buildLatticeRule(modelRule *model.Rule) (*vpclattice.GetRuleOutput, error) { + gro := vpclattice.GetRuleOutput{ + IsDefault: aws.Bool(false), + Priority: aws.Int64(modelRule.Spec.Priority), } - // if not found, ruleStatus contains the next available priority - - var latticeTGs []*vpclattice.WeightedTargetGroup - - for _, tgRule := range rule.Spec.Action.TargetGroups { - tgName := latticestore.TargetGroupName(tgRule.Name, tgRule.Namespace) - - tg, err := r.latticeDataStore.GetTargetGroup(tgName, tgRule.RouteName, tgRule.IsServiceImport) - if err != nil { - return model.RuleStatus{}, err - } + httpMatch := vpclattice.HttpMatch{} + updateMatchFromRule(&httpMatch, modelRule) + gro.Match = &vpclattice.RuleMatch{HttpMatch: &httpMatch} + + if len(modelRule.Spec.Action.TargetGroups) > 0 { + var latticeTGs []*vpclattice.WeightedTargetGroup + for _, ruleTg := range modelRule.Spec.Action.TargetGroups { + latticeTG := vpclattice.WeightedTargetGroup{ + TargetGroupIdentifier: aws.String(ruleTg.LatticeTgId), + Weight: aws.Int64(ruleTg.Weight), + } - latticeTG := vpclattice.WeightedTargetGroup{ - TargetGroupIdentifier: aws.String(tg.ID), - Weight: aws.Int64(tgRule.Weight), + latticeTGs = append(latticeTGs, &latticeTG) } - latticeTGs = append(latticeTGs, &latticeTG) - } - - ruleName := fmt.Sprintf("k8s-%d-%s", rule.Spec.CreateTime.Unix(), rule.Spec.RuleID) - - if ruleStatus.UpdateTGsNeeded { - httpMatch := vpclattice.HttpMatch{} - - updateSDKhttpMatch(&httpMatch, rule) - - updateRuleInput := vpclattice.UpdateRuleInput{ - Action: &vpclattice.RuleAction{ - Forward: &vpclattice.ForwardAction{ - TargetGroups: latticeTGs, - }, - }, - ListenerIdentifier: aws.String(listener.ID), - Match: &vpclattice.RuleMatch{ - HttpMatch: &httpMatch, + gro.Action = &vpclattice.RuleAction{ + Forward: &vpclattice.ForwardAction{ + TargetGroups: latticeTGs, }, - Priority: aws.Int64(ruleStatus.Priority), - ServiceIdentifier: aws.String(*latticeService.Id), - RuleIdentifier: aws.String(ruleStatus.RuleID), } - - resp, err := r.cloud.Lattice().UpdateRule(&updateRuleInput) - if err != nil { - r.log.Errorf("Error updating rule, %s", err) + } else { + r.log.Debugf("There are no target groups, defaulting to 404 Fixed response") + gro.Action = &vpclattice.RuleAction{ + FixedResponse: &vpclattice.FixedResponseAction{ + StatusCode: aws.Int64(404), + }, } - - return model.RuleStatus{ - RuleID: aws.StringValue(resp.Id), - UpdatePriorityNeeded: ruleStatus.UpdatePriorityNeeded, - ServiceID: aws.StringValue(latticeService.Id), - ListenerID: listener.ID, - }, nil } - httpMatch := vpclattice.HttpMatch{} + gro.Name = aws.String(fmt.Sprintf("k8s-%d-rule-%d", modelRule.Spec.CreateTime.Unix(), modelRule.Spec.Priority)) + return &gro, nil +} - updateSDKhttpMatch(&httpMatch, rule) +func (r *defaultRuleManager) update(ctx context.Context, + latticeRule *vpclattice.GetRuleOutput, latticeSvcId, latticeListenerId string) error { - ruleInput := vpclattice.CreateRuleInput{ - Action: &vpclattice.RuleAction{ - Forward: &vpclattice.ForwardAction{ - TargetGroups: latticeTGs, - }, - }, - ClientToken: nil, - ListenerIdentifier: aws.String(listener.ID), - Match: &vpclattice.RuleMatch{ - HttpMatch: &httpMatch, - }, - Name: aws.String(ruleName), - Priority: aws.Int64(ruleStatus.Priority), - ServiceIdentifier: aws.String(*latticeService.Id), - Tags: r.cloud.DefaultTags(), + uri := vpclattice.UpdateRuleInput{ + Action: latticeRule.Action, + ServiceIdentifier: aws.String(latticeSvcId), + ListenerIdentifier: aws.String(latticeListenerId), + RuleIdentifier: latticeRule.Id, + Match: latticeRule.Match, + Priority: latticeRule.Priority, } - resp, err := r.cloud.Lattice().CreateRule(&ruleInput) + res, err := r.cloud.Lattice().UpdateRuleWithContext(ctx, &uri) if err != nil { - return model.RuleStatus{}, err + r.log.Infof("Failed UpdateRule %s due to %s", aws.StringValue(latticeRule.Id), err) + return err } - return model.RuleStatus{ - RuleID: *resp.Id, - ListenerID: listener.ID, - ServiceID: aws.StringValue(latticeService.Id), - UpdatePriorityNeeded: ruleStatus.UpdatePriorityNeeded, - UpdateTGsNeeded: ruleStatus.UpdatePriorityNeeded, - }, nil + r.log.Infof("Succcess UpdateRule %s, %s", aws.StringValue(res.Name), aws.StringValue(res.Id)) + return nil } -func updateSDKhttpMatch(httpMatch *vpclattice.HttpMatch, rule *model.Rule) { - // setup path based - if rule.Spec.PathMatchExact || rule.Spec.PathMatchPrefix { - matchType := vpclattice.PathMatchType{} - if rule.Spec.PathMatchExact { - matchType.Exact = aws.String(rule.Spec.PathMatchValue) - } - if rule.Spec.PathMatchPrefix { - matchType.Prefix = aws.String(rule.Spec.PathMatchValue) - } - - httpMatch.PathMatch = &vpclattice.PathMatch{ - Match: &matchType, - } +func (r *defaultRuleManager) Upsert( + ctx context.Context, + modelRule *model.Rule, + modelListener *model.Listener, + modelSvc *model.Service, +) (model.RuleStatus, error) { + if modelListener.Status == nil || modelListener.Status.Id == "" { + return model.RuleStatus{}, errors.New("listener is missing id") } - - httpMatch.Method = &rule.Spec.Method - - if rule.Spec.NumOfHeaderMatches > 0 { - for i := 0; i < rule.Spec.NumOfHeaderMatches; i++ { - headerMatch := vpclattice.HeaderMatch{ - Match: rule.Spec.MatchedHeaders[i].Match, - Name: rule.Spec.MatchedHeaders[i].Name, - } - httpMatch.HeaderMatches = append(httpMatch.HeaderMatches, &headerMatch) + if modelSvc.Status == nil || modelSvc.Status.Id == "" { + return model.RuleStatus{}, errors.New("model service is missing id") + } + for i, mtg := range modelRule.Spec.Action.TargetGroups { + if mtg.LatticeTgId == "" { + return model.RuleStatus{}, fmt.Errorf("rule %d action %d is missing lattice target group id", modelRule.Spec.Priority, i) } } -} + latticeServiceId := modelSvc.Status.Id + latticeListenerId := modelListener.Status.Id -func isRulesSame(log gwlog.Logger, modelRule *model.Rule, sdkRuleDetail *vpclattice.GetRuleOutput) bool { - // Exact Path Match - if modelRule.Spec.PathMatchExact { - if sdkRuleDetail.Match.HttpMatch.PathMatch == nil || - sdkRuleDetail.Match.HttpMatch.PathMatch.Match == nil || - sdkRuleDetail.Match.HttpMatch.PathMatch.Match.Exact == nil { - log.Debugf("no sdk PathMatchExact match") - return false - } + // this allows us to make apples to apples comparisons with what's in Lattice already + latticeRuleFromModel, err := r.buildLatticeRule(modelRule) + if err != nil { + return model.RuleStatus{}, err + } - if aws.StringValue(sdkRuleDetail.Match.HttpMatch.PathMatch.Match.Exact) != modelRule.Spec.PathMatchValue { - log.Debugf("Match.Exact mismatch") - return false - } + r.log.Debugf("Upsert rule %s for service %s-%s and listener port %d and protocol %s", + aws.StringValue(latticeRuleFromModel.Name), latticeServiceId, latticeListenerId, + modelListener.Spec.Port, modelListener.Spec.Protocol) - } else { - if sdkRuleDetail.Match.HttpMatch.PathMatch != nil && - sdkRuleDetail.Match.HttpMatch.PathMatch.Match != nil && - sdkRuleDetail.Match.HttpMatch.PathMatch.Match.Exact != nil { - log.Debugf("no sdk PathMatchExact match") - return false - } + lri := vpclattice.ListRulesInput{ + ServiceIdentifier: aws.String(modelSvc.Status.Id), + ListenerIdentifier: aws.String(modelListener.Status.Id), } - - // Path Prefix - if modelRule.Spec.PathMatchPrefix { - if sdkRuleDetail.Match.HttpMatch.PathMatch == nil || - sdkRuleDetail.Match.HttpMatch.PathMatch.Match == nil || - sdkRuleDetail.Match.HttpMatch.PathMatch.Match.Prefix == nil { - log.Debugf("no sdk HTTP PathPrefix") - return false + // TODO: fetching all rules every time is not efficient - maybe have a separate public method to prepopulate? + currentLatticeRules, err := r.cloud.Lattice().GetRulesAsList(ctx, &lri) + + var matchingRule *vpclattice.GetRuleOutput + for _, clr := range currentLatticeRules { + if isMatchEqual(latticeRuleFromModel, clr) { + matchingRule = clr + break } + } - if aws.StringValue(sdkRuleDetail.Match.HttpMatch.PathMatch.Match.Prefix) != modelRule.Spec.PathMatchValue { - log.Debugf("PathMatchPrefix mismatch ") - return false - } + if matchingRule == nil { + return r.create(ctx, currentLatticeRules, latticeRuleFromModel, latticeServiceId, latticeListenerId) } else { - if sdkRuleDetail.Match.HttpMatch.PathMatch != nil && - sdkRuleDetail.Match.HttpMatch.PathMatch.Match != nil && - sdkRuleDetail.Match.HttpMatch.PathMatch.Match.Prefix != nil { - log.Debugf("no sdk HTTP PathPrefix") - return false - } + return r.updateIfNeeded(ctx, latticeRuleFromModel, matchingRule, latticeServiceId, latticeListenerId) } +} - // Method match - if aws.StringValue(sdkRuleDetail.Match.HttpMatch.Method) != modelRule.Spec.Method { - log.Debugf("Method mismatch '%s' != '%s'", modelRule.Spec.Method, *sdkRuleDetail.Match.HttpMatch.Method) - return false +func (r *defaultRuleManager) updateIfNeeded( + ctx context.Context, + ruleToUpdate *vpclattice.GetRuleOutput, + matchingRule *vpclattice.GetRuleOutput, + latticeSvcId string, + latticeListenerId string, +) (model.RuleStatus, error) { + updatedRuleStatus := model.RuleStatus{ + Name: aws.StringValue(matchingRule.Name), + Arn: aws.StringValue(matchingRule.Arn), + Id: aws.StringValue(matchingRule.Id), + ListenerId: latticeListenerId, + ServiceId: latticeSvcId, + Priority: aws.Int64Value(matchingRule.Priority), } - // Header Match - if modelRule.Spec.NumOfHeaderMatches > 0 { - if len(sdkRuleDetail.Match.HttpMatch.HeaderMatches) != modelRule.Spec.NumOfHeaderMatches { - log.Debugf("header match number mismatch") - return false - } - - misMatch := false - - // compare 2 array - for _, sdkHeader := range sdkRuleDetail.Match.HttpMatch.HeaderMatches { - matchFound := false - // check if this is in module - for i := 0; i < modelRule.Spec.NumOfHeaderMatches; i++ { - // compare header - if aws.StringValue(modelRule.Spec.MatchedHeaders[i].Name) == - aws.StringValue(sdkHeader.Name) && - aws.StringValue(modelRule.Spec.MatchedHeaders[i].Match.Exact) == - aws.StringValue(sdkHeader.Match.Exact) { - matchFound = true - break - } - - } + // we already validated Match, if Action is also the same then no updates required + updateNeeded := !reflect.DeepEqual(ruleToUpdate.Action, matchingRule.Action) + if !updateNeeded { + r.log.Debugf("rule unchanged, no updates required") + return updatedRuleStatus, nil + } - if !matchFound { - misMatch = true - log.Debugf("header %s not found", *sdkHeader) - break - } - } + // when we update a rule, we use the priority of the existing rule to avoid conflicts + ruleToUpdate.Priority = matchingRule.Priority + ruleToUpdate.Id = matchingRule.Id + + uri := vpclattice.UpdateRuleInput{ + Action: ruleToUpdate.Action, + ServiceIdentifier: aws.String(latticeSvcId), + ListenerIdentifier: aws.String(latticeListenerId), + RuleIdentifier: ruleToUpdate.Id, + Match: ruleToUpdate.Match, + Priority: ruleToUpdate.Priority, + } - if misMatch { - log.Debugf("mismatch header") - return false - } + _, err := r.cloud.Lattice().UpdateRuleWithContext(ctx, &uri) + if err != nil { + return model.RuleStatus{}, fmt.Errorf("Failed UpdateRule %d for %s, %s due to %s", + ruleToUpdate.Priority, latticeListenerId, latticeSvcId, err) } - return true + r.log.Infof("Success UpdateRule %d for %s, %s", ruleToUpdate.Priority, latticeListenerId, latticeSvcId) + return updatedRuleStatus, nil } -// Determine if rule spec is same -// If rule spec is same, then determine if there is any changes on target groups -func (r *defaultRuleManager) findMatchingRule( +func (r *defaultRuleManager) create( ctx context.Context, - rule *model.Rule, - serviceId string, - listenerId string, + currentLatticeRules []*vpclattice.GetRuleOutput, + ruleToCreate *vpclattice.GetRuleOutput, + latticeSvcId string, + latticeListenerId string, ) (model.RuleStatus, error) { - var priorityMap [100]bool - - for i := 1; i < 100; i++ { - priorityMap[i] = false - + // when we create a rule, we just pick an available priority so we can + // successfully create the rule. After all rules are created, we update + // priorities based on the order they appear in the Route. Note, this + // approach is not fully compliant with the gw spec + priority, err := r.nextAvailablePriority(currentLatticeRules) + if err != nil { + return model.RuleStatus{}, err } - - ruleListInput := vpclattice.ListRulesInput{ - ListenerIdentifier: aws.String(listenerId), - ServiceIdentifier: aws.String(serviceId), + ruleToCreate.Priority = aws.Int64(priority) + + cri := vpclattice.CreateRuleInput{ + Action: ruleToCreate.Action, + ServiceIdentifier: aws.String(latticeSvcId), + ListenerIdentifier: aws.String(latticeListenerId), + Match: ruleToCreate.Match, + Name: ruleToCreate.Name, + Priority: ruleToCreate.Priority, + Tags: r.cloud.DefaultTags(), } - var resp *vpclattice.ListRulesOutput - resp, err := r.cloud.Lattice().ListRules(&ruleListInput) + res, err := r.cloud.Lattice().CreateRuleWithContext(ctx, &cri) if err != nil { - return model.RuleStatus{}, err + return model.RuleStatus{}, fmt.Errorf("Failed CreateRule %s, %s due to %s", latticeListenerId, latticeSvcId, err) } - var matchRule *vpclattice.GetRuleOutput = nil - var updateTGsNeeded = false - for _, ruleSum := range resp.Items { - if aws.BoolValue(ruleSum.IsDefault) { - // Ignore the default - continue - } + r.log.Infof("Succcess CreateRule %s, %s", aws.StringValue(res.Name), aws.StringValue(res.Id)) - // retrieve action - ruleInput := vpclattice.GetRuleInput{ - ListenerIdentifier: &listenerId, - ServiceIdentifier: &serviceId, - RuleIdentifier: ruleSum.Id, - } - - var ruleResp *vpclattice.GetRuleOutput - - ruleResp, err := r.cloud.Lattice().GetRule(&ruleInput) - if err != nil { - r.log.Debugf("Matching rule not found, err %s", err) - continue - } - - priorityMap[aws.Int64Value(ruleResp.Priority)] = true + return model.RuleStatus{ + Name: aws.StringValue(res.Name), + Arn: aws.StringValue(res.Arn), + Id: aws.StringValue(res.Id), + ServiceId: latticeSvcId, + ListenerId: latticeListenerId, + Priority: aws.Int64Value(res.Priority), + }, nil +} - ruleIsSame := isRulesSame(r.log, rule, ruleResp) - if !ruleIsSame { - continue +func updateMatchFromRule(httpMatch *vpclattice.HttpMatch, modelRule *model.Rule) { + // setup path based + if modelRule.Spec.PathMatchExact || modelRule.Spec.PathMatchPrefix { + matchType := vpclattice.PathMatchType{} + if modelRule.Spec.PathMatchExact { + matchType.Exact = aws.String(modelRule.Spec.PathMatchValue) } - - matchRule = ruleResp - - if len(ruleResp.Action.Forward.TargetGroups) != len(rule.Spec.Action.TargetGroups) { - r.log.Debugf("Skipping rule due to mismatched number of target groups to forward to") - updateTGsNeeded = true - continue + if modelRule.Spec.PathMatchPrefix { + matchType.Prefix = aws.String(modelRule.Spec.PathMatchValue) } - if len(ruleResp.Action.Forward.TargetGroups) == 0 { - r.log.Debugf("Skipping rule due to 0 targetGroups to forward to") - continue + httpMatch.PathMatch = &vpclattice.PathMatch{ + Match: &matchType, } + } - for _, tg := range ruleResp.Action.Forward.TargetGroups { - for _, k8sTG := range rule.Spec.Action.TargetGroups { - // get k8sTG id - tgName := latticestore.TargetGroupName(k8sTG.Name, k8sTG.Namespace) - k8sTGinStore, err := r.latticeDataStore.GetTargetGroup(tgName, rule.Spec.ServiceName, k8sTG.IsServiceImport) - - if err != nil { - r.log.Debugf("Failed to find k8s tg %s-%s in datastore", k8sTG.Name, k8sTG.Namespace) - updateTGsNeeded = true - continue - } - - if aws.StringValue(tg.TargetGroupIdentifier) != k8sTGinStore.ID { - r.log.Debugf("target group id mismatch in datastore, %s vs. %s", - aws.StringValue(tg.TargetGroupIdentifier), k8sTGinStore.ID) - updateTGsNeeded = true - continue - - } - - if k8sTG.Weight != aws.Int64Value(tg.Weight) { - r.log.Debugf("Weight has changed for tg %s, old %d vs. new %d", - aws.StringValue(tg.TargetGroupIdentifier), aws.Int64Value(tg.Weight), k8sTG.Weight) - updateTGsNeeded = true - continue - } - - break - } + httpMatch.Method = &modelRule.Spec.Method - if updateTGsNeeded { - break - } + for i := 0; i < len(modelRule.Spec.MatchedHeaders); i++ { + headerMatch := vpclattice.HeaderMatch{ + Match: modelRule.Spec.MatchedHeaders[i].Match, + Name: modelRule.Spec.MatchedHeaders[i].Name, } + httpMatch.HeaderMatches = append(httpMatch.HeaderMatches, &headerMatch) } +} - if matchRule != nil { - inputRulePriority, _ := ruleID2Priority(rule.Spec.RuleID) +func isMatchEqual(lr1, lr2 *vpclattice.GetRuleOutput) bool { + return reflect.DeepEqual(lr1.Match, lr2.Match) +} - UpdatePriority := false - if inputRulePriority != aws.Int64Value(matchRule.Priority) { - UpdatePriority = true - } +func (r *defaultRuleManager) nextAvailablePriority(latticeRules []*vpclattice.GetRuleOutput) (int64, error) { + var priorities [model.MaxRulePriority]bool + for i := 0; i < model.MaxRulePriority; i++ { + priorities[i] = false + } - return model.RuleStatus{ - RuleARN: aws.StringValue(matchRule.Arn), - RuleID: aws.StringValue(matchRule.Id), - Priority: aws.Int64Value(matchRule.Priority), - UpdateTGsNeeded: updateTGsNeeded, - UpdatePriorityNeeded: UpdatePriority, - }, nil - } else { - var nextPriority int64 = 0 - // find available priority - for i := 1; i < 100; i++ { - if !priorityMap[i] { - nextPriority = int64(i) - break - } + for _, lr := range latticeRules { + if lr.IsDefault != nil && aws.BoolValue(lr.IsDefault) { + continue + } + // priority range is 1 -> 100 + priorities[aws.Int64Value(lr.Priority)-1] = true + } + for i := 0; i < model.MaxRulePriority; i++ { + if !priorities[i] { + return int64(i + 1), nil } - return model.RuleStatus{Priority: nextPriority}, errors.New("rule not found") } -} -func ruleID2Priority(ruleId string) (int64, error) { - var priority int - ruleIDName := strings.NewReader(ruleId) - _, err := fmt.Fscanf(ruleIDName, "rule-%d", &priority) - return int64(priority), err + return 0, errors.New("no available priorities") } -func (r *defaultRuleManager) Delete(ctx context.Context, ruleId string, listenerId string, serviceId string) error { +func (r *defaultRuleManager) Delete(ctx context.Context, ruleId string, serviceId string, listenerId string) error { r.log.Debugf("Deleting rule %s for listener %s and service %s", ruleId, listenerId, serviceId) deleteInput := vpclattice.DeleteRuleInput{ - RuleIdentifier: aws.String(ruleId), - ListenerIdentifier: aws.String(listenerId), ServiceIdentifier: aws.String(serviceId), + ListenerIdentifier: aws.String(listenerId), + RuleIdentifier: aws.String(ruleId), + } + + _, err := r.cloud.Lattice().DeleteRuleWithContext(ctx, &deleteInput) + if err != nil { + return fmt.Errorf("Failed DeleteRule %s/%s/%s due to %s", serviceId, listenerId, ruleId, err) } - _, err := r.cloud.Lattice().DeleteRule(&deleteInput) - return err + r.log.Infof("Succcess DeleteRule %s/%s/%s", serviceId, listenerId, ruleId) + return nil } diff --git a/pkg/deploy/lattice/rule_manager_mock.go b/pkg/deploy/lattice/rule_manager_mock.go index d1d12ffc..58843608 100644 --- a/pkg/deploy/lattice/rule_manager_mock.go +++ b/pkg/deploy/lattice/rule_manager_mock.go @@ -8,7 +8,6 @@ import ( context "context" reflect "reflect" - aws "github.com/aws/aws-application-networking-k8s/pkg/aws" lattice "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" vpclattice "github.com/aws/aws-sdk-go/service/vpclattice" gomock "github.com/golang/mock/gomock" @@ -37,47 +36,18 @@ func (m *MockRuleManager) EXPECT() *MockRuleManagerMockRecorder { return m.recorder } -// Cloud mocks base method. -func (m *MockRuleManager) Cloud() aws.Cloud { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Cloud") - ret0, _ := ret[0].(aws.Cloud) - return ret0 -} - -// Cloud indicates an expected call of Cloud. -func (mr *MockRuleManagerMockRecorder) Cloud() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Cloud", reflect.TypeOf((*MockRuleManager)(nil).Cloud)) -} - -// Create mocks base method. -func (m *MockRuleManager) Create(ctx context.Context, rule *lattice.Rule) (lattice.RuleStatus, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", ctx, rule) - ret0, _ := ret[0].(lattice.RuleStatus) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Create indicates an expected call of Create. -func (mr *MockRuleManagerMockRecorder) Create(ctx, rule interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockRuleManager)(nil).Create), ctx, rule) -} - // Delete mocks base method. -func (m *MockRuleManager) Delete(ctx context.Context, ruleId, listenerId, serviceId string) error { +func (m *MockRuleManager) Delete(ctx context.Context, ruleId, serviceId, listenerId string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", ctx, ruleId, listenerId, serviceId) + ret := m.ctrl.Call(m, "Delete", ctx, ruleId, serviceId, listenerId) ret0, _ := ret[0].(error) return ret0 } // Delete indicates an expected call of Delete. -func (mr *MockRuleManagerMockRecorder) Delete(ctx, ruleId, listenerId, serviceId interface{}) *gomock.Call { +func (mr *MockRuleManagerMockRecorder) Delete(ctx, ruleId, serviceId, listenerId interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockRuleManager)(nil).Delete), ctx, ruleId, listenerId, serviceId) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockRuleManager)(nil).Delete), ctx, ruleId, serviceId, listenerId) } // Get mocks base method. @@ -96,10 +66,10 @@ func (mr *MockRuleManagerMockRecorder) Get(ctx, serviceId, listenerId, ruleId in } // List mocks base method. -func (m *MockRuleManager) List(ctx context.Context, serviceId, listenerId string) ([]*lattice.RuleStatus, error) { +func (m *MockRuleManager) List(ctx context.Context, serviceId, listenerId string) ([]*vpclattice.RuleSummary, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List", ctx, serviceId, listenerId) - ret0, _ := ret[0].([]*lattice.RuleStatus) + ret0, _ := ret[0].([]*vpclattice.RuleSummary) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -110,16 +80,31 @@ func (mr *MockRuleManagerMockRecorder) List(ctx, serviceId, listenerId interface return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockRuleManager)(nil).List), ctx, serviceId, listenerId) } -// Update mocks base method. -func (m *MockRuleManager) Update(ctx context.Context, rules []*lattice.Rule) error { +// UpdatePriorities mocks base method. +func (m *MockRuleManager) UpdatePriorities(ctx context.Context, svcId, listenerId string, rules []*lattice.Rule) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Update", ctx, rules) + ret := m.ctrl.Call(m, "UpdatePriorities", ctx, svcId, listenerId, rules) ret0, _ := ret[0].(error) return ret0 } -// Update indicates an expected call of Update. -func (mr *MockRuleManagerMockRecorder) Update(ctx, rules interface{}) *gomock.Call { +// UpdatePriorities indicates an expected call of UpdatePriorities. +func (mr *MockRuleManagerMockRecorder) UpdatePriorities(ctx, svcId, listenerId, rules interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdatePriorities", reflect.TypeOf((*MockRuleManager)(nil).UpdatePriorities), ctx, svcId, listenerId, rules) +} + +// Upsert mocks base method. +func (m *MockRuleManager) Upsert(ctx context.Context, modelRule *lattice.Rule, modelListener *lattice.Listener, modelSvc *lattice.Service) (lattice.RuleStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Upsert", ctx, modelRule, modelListener, modelSvc) + ret0, _ := ret[0].(lattice.RuleStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Upsert indicates an expected call of Upsert. +func (mr *MockRuleManagerMockRecorder) Upsert(ctx, modelRule, modelListener, modelSvc interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockRuleManager)(nil).Update), ctx, rules) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockRuleManager)(nil).Upsert), ctx, modelRule, modelListener, modelSvc) } diff --git a/pkg/deploy/lattice/rule_manager_test.go b/pkg/deploy/lattice/rule_manager_test.go index 4aef8650..0baa2a71 100644 --- a/pkg/deploy/lattice/rule_manager_test.go +++ b/pkg/deploy/lattice/rule_manager_test.go @@ -2,1510 +2,236 @@ package lattice import ( "context" - "fmt" - - "github.com/aws/aws-sdk-go/aws" - - "testing" - - "github.com/aws/aws-sdk-go/service/vpclattice" - - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" mocks "github.com/aws/aws-application-networking-k8s/pkg/aws/services" - + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/vpclattice" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + "testing" ) -var ruleList = []struct { - Arn string - Id string - IsDefault bool - Name string -}{ - { - - Arn: "Rule-Arn-1", - Id: "Rule-Id-1", - IsDefault: false, - Name: "Rule-1", - }, - { - - Arn: "Rule-Arn-2", - Id: "Rule-Id-2", - IsDefault: false, - Name: "Rule-2", - }, -} - -var rules = []*model.Rule{ - { - Spec: model.RuleSpec{ - ServiceName: "svc-1", - ServiceNamespace: "default", - ListenerPort: int64(80), - ListenerProtocol: "HTTP", - RuleID: "rule-1", //TODO, maybe rename this field to RuleName - }, - Status: &model.RuleStatus{ - ServiceID: "serviceID1", - ListenerID: "listenerID1", - RuleID: "rule-ID-1", - }, - }, - - { - Spec: model.RuleSpec{ - ServiceName: "svc-1", - ServiceNamespace: "default", - ListenerPort: int64(80), - ListenerProtocol: "HTTP", - RuleID: "rule-2", //TODO, maybe rename this field to RuleName - }, - Status: &model.RuleStatus{ - ServiceID: "serviceID1", - ListenerID: "listenerID1", - RuleID: "rule-ID-2", - }, - }, -} - -func Test_CreateRule(t *testing.T) { - ServiceName := "seviceName" - ServiceNameSpace := "defaultService" - ServiceID := "serviceID" - ListenerPort := int64(80) - ListenerProtocol := "HTTP" - ListenerID := "listenerID" - ruleID := "ruleID" - - var hdr1 = "env1" - var hdr1Value = "test1" - var hdr2 = "env2" - var hdr2Value = "test2" - - var weight1 = int64(90) - var weight2 = int64(10) - weightRulePriority := 1 - weightRuleID := fmt.Sprintf("rule-%d", weightRulePriority) - WeightedAction_1 := model.RuleTargetGroup{ - Name: "TestCreateWeighted1", - Namespace: "default", - IsServiceImport: false, - Weight: weight1, - } - - WeightedAction_11 := model.RuleTargetGroup{ - Name: "TestCreateWeighted1", - Namespace: "default", - IsServiceImport: false, - Weight: weight2, - } - - WeightedAction_2 := model.RuleTargetGroup{ - Name: "TestCreateWeighte2", - Namespace: "default", - IsServiceImport: false, - Weight: weight2, - } - - WeightedAction_22 := model.RuleTargetGroup{ - Name: "TestCreateWeighte2", - Namespace: "default", - IsServiceImport: false, - Weight: weight1, - } - - WeightedRule_1 := model.Rule{ - Spec: model.RuleSpec{ - ServiceName: ServiceName, - ServiceNamespace: ServiceNameSpace, - ListenerPort: ListenerPort, - ListenerProtocol: ListenerProtocol, - PathMatchPrefix: true, - PathMatchValue: "", - RuleID: weightRuleID, - Action: model.RuleAction{ - TargetGroups: []*model.RuleTargetGroup{ - &WeightedAction_1, - }, - }, - }, - Status: &model.RuleStatus{ - RuleARN: "ruleARn", - RuleID: "rule-id-1", - ListenerID: ListenerID, - ServiceID: ServiceID, - }, - } +func Test_Create(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - WeightedRule_1_2 := model.Rule{ - Spec: model.RuleSpec{ - ServiceName: ServiceName, - ServiceNamespace: ServiceNameSpace, - ListenerPort: ListenerPort, - ListenerProtocol: ListenerProtocol, - PathMatchValue: "", - PathMatchPrefix: true, - RuleID: weightRuleID, - Action: model.RuleAction{ - TargetGroups: []*model.RuleTargetGroup{ - &WeightedAction_1, - &WeightedAction_2, - }, - }, - }, - Status: &model.RuleStatus{ - RuleARN: "ruleARn", - RuleID: "rule-id-1-2", - ListenerID: ListenerID, - ServiceID: ServiceID, - }, + // each rule references a stack and a service which need to be present in the stack + // in order to proceed, these just need their status+id + svc := &model.Service{ + Status: &model.ServiceStatus{Id: "svc-id"}, } - WeightedRule_2_1 := model.Rule{ - Spec: model.RuleSpec{ - ServiceName: ServiceName, - ServiceNamespace: ServiceNameSpace, - ListenerPort: ListenerPort, - ListenerProtocol: ListenerProtocol, - PathMatchValue: "", - PathMatchPrefix: true, - RuleID: weightRuleID, - Action: model.RuleAction{ - TargetGroups: []*model.RuleTargetGroup{ - &WeightedAction_11, - &WeightedAction_22, - }, - }, - }, - Status: &model.RuleStatus{ - RuleARN: "ruleARn", - RuleID: "rule-id-2-1", - ListenerID: ListenerID, - ServiceID: ServiceID, + l := &model.Listener{ + Spec: model.ListenerSpec{ + Port: 80, + Protocol: "HTTP", }, + Status: &model.ListenerStatus{Id: "listener-id"}, } - pathRule_1 := model.Rule{ + r := &model.Rule{ Spec: model.RuleSpec{ - ServiceName: ServiceName, - ServiceNamespace: ServiceNameSpace, - ListenerPort: ListenerPort, - ListenerProtocol: ListenerProtocol, - RuleID: weightRuleID, - PathMatchPrefix: true, - PathMatchValue: "/ver-1", + Priority: 1, + Method: "POST", Action: model.RuleAction{ TargetGroups: []*model.RuleTargetGroup{ - &WeightedAction_1, + { + LatticeTgId: "tg-id", + Weight: 1, + }, }, }, }, - Status: &model.RuleStatus{ - RuleARN: "ruleARn", - RuleID: "rule-id-2-1", - ListenerID: ListenerID, - ServiceID: ServiceID, - }, } - pathRule_11 := model.Rule{ - Spec: model.RuleSpec{ - ServiceName: ServiceName, - ServiceNamespace: ServiceNameSpace, - ListenerPort: ListenerPort, - ListenerProtocol: ListenerProtocol, - RuleID: weightRuleID, - PathMatchPrefix: true, - PathMatchValue: "/ver-1", - Action: model.RuleAction{ - TargetGroups: []*model.RuleTargetGroup{ - &WeightedAction_2, - }, - }, - }, - Status: &model.RuleStatus{ - RuleARN: "ruleARn", - RuleID: "rule-id-2-1", - ListenerID: ListenerID, - ServiceID: ServiceID, - }, - } + t.Run("test create", func(t *testing.T) { + mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( + []*vpclattice.GetRuleOutput{}, nil) - pathRule_2 := model.Rule{ - Spec: model.RuleSpec{ - ServiceName: ServiceName, - ServiceNamespace: ServiceNameSpace, - ListenerPort: ListenerPort, - ListenerProtocol: ListenerProtocol, - RuleID: weightRuleID, - PathMatchPrefix: true, - PathMatchValue: "/ver-2", - Action: model.RuleAction{ - TargetGroups: []*model.RuleTargetGroup{ - &WeightedAction_1, - }, - }, - }, - Status: &model.RuleStatus{ - RuleARN: "ruleARn", - RuleID: "rule-id-2-1", - ListenerID: ListenerID, - ServiceID: ServiceID, - }, - } + mockLattice.EXPECT().CreateRuleWithContext(ctx, gomock.Any()).Return( + &vpclattice.CreateRuleOutput{ + Arn: aws.String("arn"), + Id: aws.String("id"), + Name: aws.String("name"), + }, nil) - headerRule_1 := model.Rule{ - Spec: model.RuleSpec{ - ServiceName: ServiceName, - ServiceNamespace: ServiceNameSpace, - ListenerPort: ListenerPort, - ListenerProtocol: ListenerProtocol, - RuleID: weightRuleID, - PathMatchPrefix: true, - PathMatchValue: "/ver-2", - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ + rm := NewRuleManager(gwlog.FallbackLogger, cloud) + ruleStatus, err := rm.Upsert(ctx, r, l, svc) + assert.Nil(t, err) + assert.Equal(t, "arn", ruleStatus.Arn) + }) + t.Run("test update", func(t *testing.T) { + mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( + []*vpclattice.GetRuleOutput{ { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, + Id: aws.String("existing-id"), + Arn: aws.String("existing-arn"), + Match: &vpclattice.RuleMatch{ + HttpMatch: &vpclattice.HttpMatch{ + Method: aws.String("POST"), + }, + }, + Action: &vpclattice.RuleAction{ + FixedResponse: &vpclattice.FixedResponseAction{}, // <-- this will trigger update + }, + Name: aws.String("existing-name"), + Priority: aws.Int64(1), }, + }, nil) - {}, - {}, - {}, - }, - Action: model.RuleAction{ - TargetGroups: []*model.RuleTargetGroup{ - &WeightedAction_1, - }, - }, - }, - Status: &model.RuleStatus{ - RuleARN: "ruleARn", - RuleID: "rule-id-2-1", - ListenerID: ListenerID, - ServiceID: ServiceID, - }, - } - headerRule_1_path_exact := headerRule_1 - headerRule_1_path_exact.Spec.PathMatchPrefix = false - headerRule_1_path_exact.Spec.PathMatchExact = true + mockLattice.EXPECT().UpdateRuleWithContext(ctx, gomock.Any()).Return( + &vpclattice.UpdateRuleOutput{ + Arn: aws.String("existing-arn"), + Id: aws.String("existing-id"), + Name: aws.String("existing-name"), + }, nil) - headerRule_1_2 := model.Rule{ - Spec: model.RuleSpec{ - ServiceName: ServiceName, - ServiceNamespace: ServiceNameSpace, - ListenerPort: ListenerPort, - ListenerProtocol: ListenerProtocol, - RuleID: weightRuleID, - PathMatchPrefix: true, - PathMatchValue: "/ver-2", - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ + rm := NewRuleManager(gwlog.FallbackLogger, cloud) + ruleStatus, err := rm.Upsert(ctx, r, l, svc) + assert.Nil(t, err) + assert.Equal(t, "existing-arn", ruleStatus.Arn) + }) + t.Run("test update - nothing to do", func(t *testing.T) { + mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( + []*vpclattice.GetRuleOutput{ { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - Action: model.RuleAction{ - TargetGroups: []*model.RuleTargetGroup{ - &WeightedAction_1, - &WeightedAction_2, + Id: aws.String("existing-id"), + Arn: aws.String("existing-arn"), + Match: &vpclattice.RuleMatch{ + HttpMatch: &vpclattice.HttpMatch{ + Method: aws.String("POST"), + }, + }, + Action: &vpclattice.RuleAction{ + Forward: &vpclattice.ForwardAction{ + TargetGroups: []*vpclattice.WeightedTargetGroup{ + { + TargetGroupIdentifier: aws.String("tg-id"), + Weight: aws.Int64(1), + }, + }, + }, + }, + Name: aws.String("existing-name"), + Priority: aws.Int64(1), }, - }, - }, - Status: &model.RuleStatus{ - RuleARN: "ruleARn", - RuleID: "rule-id-2-1", - ListenerID: ListenerID, - ServiceID: ServiceID, - }, - } - headerRule_1_2_path_exact := headerRule_1_2 - headerRule_1_2_path_exact.Spec.PathMatchPrefix = false - headerRule_1_2_path_exact.Spec.PathMatchExact = true - - tests := []struct { - name string - oldRule *model.Rule - newRule *model.Rule - listRuleOuput []*model.Rule - createRule bool - updateRule bool - noServiceID bool - noListenerID bool - noTargetGroupID bool - updatePriorityNeeded bool - }{ - - { - name: "create header-based + path prefix rule with 1 TG", - oldRule: nil, - newRule: &headerRule_1, - createRule: true, - updateRule: false, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, - - { - name: "create header-based + path prefix rule with 2 TG", - oldRule: &headerRule_1, - newRule: &headerRule_1_2, - createRule: false, - updateRule: true, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, - - { - name: "create header-based + path exact rule with 1 TG", - oldRule: nil, - newRule: &headerRule_1_path_exact, - createRule: true, - updateRule: false, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, - - { - name: "create header-based + path prefix rule with 2 TG", - oldRule: &headerRule_1_path_exact, - newRule: &headerRule_1_2_path_exact, - createRule: false, - updateRule: true, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, - - { - name: "test1, create weighted rule with 1 TG", - oldRule: nil, - newRule: &WeightedRule_1, - createRule: true, - updateRule: false, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, - - { - name: "create weighted rule with 2 TGs", - oldRule: &WeightedRule_1, - newRule: &WeightedRule_1_2, - createRule: false, - updateRule: true, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, - - { - name: "update weighted rule with 2 TGs", - oldRule: &WeightedRule_1_2, - newRule: &WeightedRule_2_1, - createRule: false, - updateRule: true, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, + }, nil) // <-- should be an exact match, no update required - { - name: "create path-based rule, no need to update priority", - oldRule: nil, - newRule: &pathRule_1, - createRule: true, - updateRule: false, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, - - { + rm := NewRuleManager(gwlog.FallbackLogger, cloud) + ruleStatus, err := rm.Upsert(ctx, r, l, svc) + assert.Nil(t, err) + assert.Equal(t, "existing-arn", ruleStatus.Arn) + }) +} - name: "create path-based rule, need to update priority", - oldRule: &pathRule_1, - newRule: &pathRule_2, - createRule: true, - updateRule: false, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: true, - }, +func Test_CreateWithTempPriority(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - { - name: "update path-based rule with a different TG", - oldRule: &pathRule_1, - newRule: &pathRule_11, - createRule: false, - updateRule: true, - noServiceID: false, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, + svc := &model.Service{ + Status: &model.ServiceStatus{Id: "svc-id"}, + } - { - name: "no serviceID", - oldRule: nil, - newRule: &pathRule_1, - createRule: false, - updateRule: false, - noServiceID: true, - noListenerID: false, - noTargetGroupID: false, - updatePriorityNeeded: false, - }, - { - name: "no listenerID", - oldRule: nil, - newRule: &pathRule_1, - createRule: false, - updateRule: false, - noServiceID: false, - noListenerID: true, - noTargetGroupID: false, - updatePriorityNeeded: false, + l := &model.Listener{ + Spec: model.ListenerSpec{ + Port: 80, + Protocol: "HTTP", }, + Status: &model.ListenerStatus{Id: "listener-id"}, + } - { - name: "no TG IDs", - oldRule: nil, - newRule: &pathRule_1, - createRule: false, - updateRule: false, - noServiceID: false, - noListenerID: false, - noTargetGroupID: true, - updatePriorityNeeded: false, + r := &model.Rule{ + Spec: model.RuleSpec{ + Priority: 1, + Method: "POST", }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - latticeDataStore := latticestore.NewLatticeDataStore() - - ruleManager := NewRuleManager(gwlog.FallbackLogger, cloud, latticeDataStore) - - if !tt.noServiceID { - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String((&RuleLSNProvider{tt.newRule}).LatticeServiceName()), - Arn: aws.String("serviceARN"), - Id: aws.String(tt.newRule.Status.ServiceID), - DnsEntry: &vpclattice.DnsEntry{ - DomainName: aws.String("test-dns"), - HostedZoneId: aws.String("my-favourite-zone"), - }, - }, nil).Times(1) - } else { - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return(nil, &mocks.NotFoundError{}).Times(1) - } - - if !tt.noListenerID { - latticeDataStore.AddListener(tt.newRule.Spec.ServiceName, tt.newRule.Spec.ServiceNamespace, - tt.newRule.Spec.ListenerPort, "HTTP", - "listernerARN", tt.newRule.Status.ListenerID) - } - - if !tt.noTargetGroupID { - for _, tg := range tt.newRule.Spec.Action.TargetGroups { - tgName := latticestore.TargetGroupName(tg.Name, tg.Namespace) - latticeDataStore.AddTargetGroup(tgName, "vpc", "arn", "tg-id", tg.IsServiceImport, "") - } - - } - - if !tt.noListenerID && !tt.noServiceID { - ruleInput := vpclattice.ListRulesInput{ - ListenerIdentifier: aws.String(tt.newRule.Status.ListenerID), - ServiceIdentifier: aws.String(tt.newRule.Status.ServiceID), - } - - ruleOutput := vpclattice.ListRulesOutput{} - - if tt.oldRule != nil { - items := []*vpclattice.RuleSummary{} - - items = append(items, &vpclattice.RuleSummary{ - Id: aws.String(tt.oldRule.Spec.RuleID), - }) - ruleOutput = vpclattice.ListRulesOutput{ - Items: items, - } - } - mockLattice.EXPECT().ListRules(&ruleInput).Return(&ruleOutput, nil) - - if tt.oldRule != nil { - ruleGetInput := vpclattice.GetRuleInput{ - ListenerIdentifier: aws.String(ListenerID), - ServiceIdentifier: aws.String(ServiceID), - RuleIdentifier: aws.String(tt.oldRule.Spec.RuleID), - } - - // listenerID := tt.oldRule.Status.ListenerID - latticeTGs := []*vpclattice.WeightedTargetGroup{} - // ruleName := fmt.Sprintf("rule-%d-%s", tt.oldRule.Spec.CreateTime.Unix(), tt.oldRule.Spec.RuleID) - priority, _ := ruleID2Priority(tt.oldRule.Spec.RuleID) - - for _, tg := range tt.oldRule.Spec.Action.TargetGroups { - latticeTG := vpclattice.WeightedTargetGroup{ - TargetGroupIdentifier: aws.String("tg-id"), - Weight: aws.Int64(tg.Weight), - } - latticeTGs = append(latticeTGs, &latticeTG) - } - - httpMatch := vpclattice.HttpMatch{} - updateSDKhttpMatch(&httpMatch, tt.oldRule) - ruleGetOutput := vpclattice.GetRuleOutput{ - Id: aws.String(tt.oldRule.Spec.RuleID), - Priority: aws.Int64(priority), - Action: &vpclattice.RuleAction{ - Forward: &vpclattice.ForwardAction{ - TargetGroups: latticeTGs, - }, - }, - Match: &vpclattice.RuleMatch{ - HttpMatch: &httpMatch, - }, - } - - mockLattice.EXPECT().GetRule(&ruleGetInput).Return(&ruleGetOutput, nil) - - } - } - - if tt.createRule || tt.updateRule { - listenerID := tt.newRule.Status.ListenerID - latticeTGs := []*vpclattice.WeightedTargetGroup{} - ruleName := fmt.Sprintf("k8s-%d-%s", tt.newRule.Spec.CreateTime.Unix(), tt.newRule.Spec.RuleID) - priority, _ := ruleID2Priority(tt.newRule.Spec.RuleID) - - if tt.updatePriorityNeeded { - priority, _ = ruleID2Priority(tt.oldRule.Spec.RuleID) - priority++ - } - - for _, tg := range tt.newRule.Spec.Action.TargetGroups { - latticeTG := vpclattice.WeightedTargetGroup{ - TargetGroupIdentifier: aws.String("tg-id"), - Weight: aws.Int64(tg.Weight), - } - latticeTGs = append(latticeTGs, &latticeTG) - } - - if tt.createRule { - httpMatch := vpclattice.HttpMatch{} - updateSDKhttpMatch(&httpMatch, tt.newRule) - ruleInput := vpclattice.CreateRuleInput{ - Action: &vpclattice.RuleAction{ - Forward: &vpclattice.ForwardAction{ - TargetGroups: latticeTGs, - }, - }, - - ListenerIdentifier: aws.String(listenerID), - Name: aws.String(ruleName), - Priority: aws.Int64(priority), - ServiceIdentifier: aws.String(ServiceID), - Match: &vpclattice.RuleMatch{ - HttpMatch: &httpMatch, - }, - Tags: cloud.DefaultTags(), - } - ruleOutput := vpclattice.CreateRuleOutput{ - Id: aws.String(ruleID), - } - mockLattice.EXPECT().CreateRule(&ruleInput).Return(&ruleOutput, nil) - } - - if tt.updateRule { - httpMatch := vpclattice.HttpMatch{} - updateSDKhttpMatch(&httpMatch, tt.newRule) - ruleInput := vpclattice.UpdateRuleInput{ - Action: &vpclattice.RuleAction{ - Forward: &vpclattice.ForwardAction{ - TargetGroups: latticeTGs, - }, - }, - - ListenerIdentifier: aws.String(listenerID), - //Name: aws.String(ruleName), - RuleIdentifier: aws.String(tt.newRule.Spec.RuleID), - Priority: aws.Int64(priority), - ServiceIdentifier: aws.String(ServiceID), - Match: &vpclattice.RuleMatch{ - HttpMatch: &httpMatch, - }, - /* - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - // TODO, what if not specfied this - //Method: aws.String(vpclattice.HttpMethodGet), - PathMatch: &vpclattice.PathMatch{ - CaseSensitive: nil, - Match: &vpclattice.PathMatchType{ - Exact: nil, - Prefix: aws.String(tt.newRule.Spec.PathMatchValue), - }, - }, - }, - }, - */ - } - ruleOutput := vpclattice.UpdateRuleOutput{ - Id: aws.String(ruleID), - } - mockLattice.EXPECT().UpdateRule(&ruleInput).Return(&ruleOutput, nil) - } - } - - resp, err := ruleManager.Create(ctx, tt.newRule) + mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( + []*vpclattice.GetRuleOutput{ + { + Id: aws.String("existing-id"), + Arn: aws.String("existing-arn"), + Match: &vpclattice.RuleMatch{ + HttpMatch: &vpclattice.HttpMatch{ + Method: aws.String("GET"), // <-- will be considered a different rule + }, + }, + Name: aws.String("existing-name"), + Priority: aws.Int64(1), // <-- we have the same priority + }, + }, nil) - if !tt.noListenerID && !tt.noServiceID && !tt.noTargetGroupID { - assert.NoError(t, err) + expectedPriority := int64(2) - assert.Equal(t, resp.ListenerID, ListenerID) - assert.Equal(t, resp.ServiceID, ServiceID) - assert.Equal(t, resp.RuleID, ruleID) - } + mockLattice.EXPECT().CreateRuleWithContext(ctx, gomock.Any()).DoAndReturn( + func(ctx context.Context, input *vpclattice.CreateRuleInput, i ...interface{}) (*vpclattice.CreateRuleOutput, error) { + // 2 is the "next" available priority + assert.Equal(t, expectedPriority, aws.Int64Value(input.Priority)) - fmt.Printf(" rulemanager.Create :%v, err %d\n", resp, err) + return &vpclattice.CreateRuleOutput{ + Arn: aws.String("new-arn"), + Id: aws.String("new-id"), + Name: aws.String("new-name"), + Priority: aws.Int64(expectedPriority), + }, nil }) - } + + rm := NewRuleManager(gwlog.FallbackLogger, cloud) + ruleStatus, err := rm.Upsert(ctx, r, l, svc) + assert.Nil(t, err) + assert.Equal(t, "new-arn", ruleStatus.Arn) + assert.Equal(t, expectedPriority, ruleStatus.Priority) } -func Test_UpdateRule(t *testing.T) { - tests := []struct { - name string - noServiceID bool - noListenerID bool - }{ - { - name: "update", - noServiceID: false, - noListenerID: false, - }, +func Test_UpdatePriorities(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) + // note that priorities are actually just assigned in order + // so this example of descending priority is contrived + rules := []*model.Rule{ { - name: "update -- no service", - noServiceID: true, - noListenerID: false, + Spec: model.RuleSpec{Priority: 2}, + Status: &model.RuleStatus{Id: "rule-0"}, }, { - name: "update -- no listenerID", - noServiceID: false, - noListenerID: true, + Spec: model.RuleSpec{Priority: 1}, + Status: &model.RuleStatus{Id: "rule-1"}, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - latticeDataStore := latticestore.NewLatticeDataStore() - - ruleManager := NewRuleManager(gwlog.FallbackLogger, cloud, latticeDataStore) - - var i = 0 - if !tt.noServiceID { - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String((&RuleLSNProvider{rules[i]}).LatticeServiceName()), - Arn: aws.String("serviceARN"), - Id: aws.String(rules[i].Status.ServiceID), - DnsEntry: &vpclattice.DnsEntry{ - DomainName: aws.String("test-dns"), - HostedZoneId: aws.String("my-favourite-zone"), - }, - }, nil).Times(1) - } else { - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return(nil, &mocks.NotFoundError{}).Times(1) - } - - if !tt.noListenerID { - latticeDataStore.AddListener(rules[i].Spec.ServiceName, rules[i].Spec.ServiceNamespace, - rules[i].Spec.ListenerPort, "HTTP", - "listenerARN", rules[i].Status.ListenerID) - } - - var ruleUpdateList []*vpclattice.RuleUpdate - - for _, rule := range rules { - priority, _ := ruleID2Priority(rule.Spec.RuleID) - ruleupdate := vpclattice.RuleUpdate{ - RuleIdentifier: aws.String(rule.Status.RuleID), - Priority: aws.Int64(priority), + mockLattice.EXPECT().BatchUpdateRuleWithContext(ctx, gomock.Any()).DoAndReturn( + func(ctx context.Context, input *vpclattice.BatchUpdateRuleInput, i ...interface{}) (*vpclattice.BatchUpdateRuleOutput, error) { + for _, rule := range input.Rules { + if *rule.RuleIdentifier == "rule-0" { + assert.Equal(t, int64(2), *rule.Priority) + continue } - - ruleUpdateList = append(ruleUpdateList, &ruleupdate) - - } - - batchRuleInput := vpclattice.BatchUpdateRuleInput{ - ListenerIdentifier: aws.String(rules[0].Status.ListenerID), - ServiceIdentifier: aws.String(rules[0].Status.ServiceID), - Rules: ruleUpdateList, - } - - if !tt.noListenerID && !tt.noServiceID { - var batchRuleOutput vpclattice.BatchUpdateRuleOutput - mockLattice.EXPECT().BatchUpdateRule(&batchRuleInput).Return(&batchRuleOutput, nil) + if *rule.RuleIdentifier == "rule-1" { + assert.Equal(t, int64(1), *rule.Priority) + continue + } + assert.Fail(t, "should not reach this point") } - err := ruleManager.Update(ctx, rules) - - if !tt.noListenerID && !tt.noServiceID { - assert.NoError(t, err) - } else { - assert.NotNil(t, err) - } + return &vpclattice.BatchUpdateRuleOutput{}, nil }) - } -} - -func Test_List(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - serviceID := "service1-ID" - listenerID := "listener-ID" - - ruleInput := vpclattice.ListRulesInput{ - ListenerIdentifier: aws.String(listenerID), - ServiceIdentifier: aws.String(serviceID), - } - ruleOutput := vpclattice.ListRulesOutput{ - Items: []*vpclattice.RuleSummary{ - { - Arn: &ruleList[0].Arn, - Id: &ruleList[0].Id, - IsDefault: &ruleList[0].IsDefault, - }, - { - Arn: &ruleList[1].Arn, - Id: &ruleList[1].Id, - IsDefault: &ruleList[1].IsDefault, - }, - }, - } - - latticeDataStore := latticestore.NewLatticeDataStore() - - mockLattice.EXPECT().ListRules(&ruleInput).Return(&ruleOutput, nil) - - ruleManager := NewRuleManager(gwlog.FallbackLogger, cloud, latticeDataStore) - - resp, err := ruleManager.List(ctx, serviceID, listenerID) - - assert.NoError(t, err) - - for i := 0; i < 2; i++ { - assert.Equal(t, resp[i].ListenerID, listenerID) - assert.Equal(t, resp[i].RuleID, ruleList[i].Id) - assert.Equal(t, resp[i].ServiceID, serviceID) - } - fmt.Printf("rule Manager List resp %v\n", resp) - -} - -func Test_GetRule(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - serviceID := "service1-ID" - listenerID := "listener-ID" - ruleID := "rule-ID" - ruleARN := "rule-ARN" - rulePriority := int64(10) - - ruleGetInput := vpclattice.GetRuleInput{ - ListenerIdentifier: aws.String(listenerID), - ServiceIdentifier: aws.String(serviceID), - RuleIdentifier: aws.String(ruleID), - } - - latticeDataStore := latticestore.NewLatticeDataStore() - - ruleGetOutput := vpclattice.GetRuleOutput{ - Arn: aws.String(ruleARN), - Id: aws.String(ruleID), - Priority: aws.Int64(int64(rulePriority)), - } - - mockLattice.EXPECT().GetRule(&ruleGetInput).Return(&ruleGetOutput, nil) - - ruleManager := NewRuleManager(gwlog.FallbackLogger, cloud, latticeDataStore) - - resp, err := ruleManager.Get(ctx, serviceID, listenerID, ruleID) - - fmt.Printf("resp :%v \n", resp) - assert.NoError(t, err) - assert.Equal(t, aws.StringValue(resp.Id), ruleID) - assert.Equal(t, aws.Int64Value(resp.Priority), rulePriority) - -} - -func Test_DeleteRule(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - serviceID := "service1-ID" - listenerID := "listener-ID" - ruleID := "rule-ID" - - ruleDeleteInput := vpclattice.DeleteRuleInput{ - ServiceIdentifier: aws.String(serviceID), - ListenerIdentifier: aws.String(listenerID), - RuleIdentifier: aws.String(ruleID), - } - - latticeDataStore := latticestore.NewLatticeDataStore() - - ruleDeleteOuput := vpclattice.DeleteRuleOutput{} - mockLattice.EXPECT().DeleteRule(&ruleDeleteInput).Return(&ruleDeleteOuput, nil) - - ruleManager := NewRuleManager(gwlog.FallbackLogger, cloud, latticeDataStore) - - ruleManager.Delete(ctx, ruleID, listenerID, serviceID) - -} - -func Test_isRulesSame(t *testing.T) { - var path1 = string("/ver1") - var path2 = string("/ver2") - var hdr1 = "env1" - var hdr1Value = "test1" - var hdr2 = "env2" - var hdr2Value = "test2" - - tests := []struct { - name string - k8sRule *model.Rule - sdkRule *vpclattice.GetRuleOutput - ruleMatched bool - }{ - { - name: "PathMatchEaxt Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - PathMatchExact: true, - PathMatchValue: path1, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Exact: &path1, - }, - }, - }, - }, - }, - ruleMatched: true, - }, - { - name: "PathMatchPrefix Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - PathMatchPrefix: true, - PathMatchValue: path1, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Prefix: &path1, - }, - }, - }, - }, - }, - ruleMatched: true, - }, - { - name: "2 headers + PathPrefix Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - PathMatchPrefix: true, - PathMatchValue: path1, - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - }, - - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Prefix: &path1, - }, - }, - }, - }, - }, - ruleMatched: true, - }, - { - name: "2 headers + path exact Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - PathMatchExact: true, - PathMatchValue: path1, - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - }, - - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Exact: &path1, - }, - }, - }, - }, - }, - ruleMatched: true, - }, - { - name: "2 headers + header mis Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - {}, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr2, - }, - {}, - }, - }, - }, - }, - ruleMatched: false, - }, - { - name: "2 headers + value mis Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - {}, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr1, - }, - {}, - }, - }, - }, - }, - ruleMatched: false, - }, - { - name: "PathMatchEaxt MisMatch", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - PathMatchExact: true, - PathMatchValue: path1, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Exact: &path2, - }, - }, - }, - }, - }, - ruleMatched: false, - }, - { - name: "PathMatchEaxt PathPrefix MisMatch", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - PathMatchPrefix: true, - PathMatchValue: path1, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Exact: &path1, - }, - }, - }, - }, - }, - ruleMatched: false, - }, - { - name: "2 headers + path exact Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - PathMatchExact: true, - PathMatchValue: path1, - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - }, - - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Exact: &path1, - }, - }, - }, - }, - }, - ruleMatched: true, - }, - { - name: "number of header mis Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - NumOfHeaderMatches: 1, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - {}, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - }, - }, - }, - }, - ruleMatched: false, - }, - { - name: "2nd header value mis Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - }, - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Exact: &path1, - }, - }, - }, - }, - }, - ruleMatched: false, - }, - { - name: "header match, but one has pathexat -- mis Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr2, - }, - }, - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Exact: &path1, - }, - }, - }, - }, - }, - ruleMatched: false, - }, - { - name: "header match, but one has pathprefix -- mis Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr2, - }, - }, - PathMatch: &vpclattice.PathMatch{ - Match: &vpclattice.PathMatchType{ - Prefix: &path1, - }, - }, - }, - }, - }, - ruleMatched: false, - }, - { - name: "header match, but one has pathprefix -- mis Match", - k8sRule: &model.Rule{ - Spec: model.RuleSpec{ - PathMatchPrefix: true, - PathMatchValue: path1, - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, - }, - sdkRule: &vpclattice.GetRuleOutput{ - Match: &vpclattice.RuleMatch{ - HttpMatch: &vpclattice.HttpMatch{ - HeaderMatches: []*vpclattice.HeaderMatch{ - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr2, - }, - }, - }, - }, - }, - ruleMatched: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - sameRule := isRulesSame(gwlog.FallbackLogger, tt.k8sRule, tt.sdkRule) - - if tt.ruleMatched { - assert.True(t, sameRule) - } else { - assert.False(t, sameRule) - } - }) - } + rm := NewRuleManager(gwlog.FallbackLogger, cloud) + err := rm.UpdatePriorities(ctx, "svc-id", "l-id", rules) + assert.Nil(t, err) } diff --git a/pkg/deploy/lattice/rule_synthesizer.go b/pkg/deploy/lattice/rule_synthesizer.go index 8a522d81..b2226c37 100644 --- a/pkg/deploy/lattice/rule_synthesizer.go +++ b/pkg/deploy/lattice/rule_synthesizer.go @@ -4,161 +4,273 @@ import ( "context" "errors" "fmt" - - "github.com/aws/aws-sdk-go/aws" - - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/aws/aws-sdk-go/aws" ) type ruleSynthesizer struct { - log gwlog.Logger - rule RuleManager - stack core.Stack - latticestore *latticestore.LatticeDataStore + log gwlog.Logger + ruleManager RuleManager + tgManager TargetGroupManager + stack core.Stack } func NewRuleSynthesizer( log gwlog.Logger, ruleManager RuleManager, + tgManager TargetGroupManager, stack core.Stack, - store *latticestore.LatticeDataStore, ) *ruleSynthesizer { return &ruleSynthesizer{ - log: log, - rule: ruleManager, - stack: stack, - latticestore: store, + log: log, + ruleManager: ruleManager, + tgManager: tgManager, + stack: stack, } } -func (r *ruleSynthesizer) Synthesize(ctx context.Context) error { - var resRule []*model.Rule - - err := r.stack.ListResources(&resRule) - if err != nil { - r.log.Debugf("Error while listing rules %s", err) +// populates all target group ids in the rule's actions +func (r *ruleSynthesizer) resolveRuleTgIds(ctx context.Context, modelRule *model.Rule) error { + if len(modelRule.Spec.Action.TargetGroups) == 0 { + r.log.Debugf("no target groups to resolve for rule %d", modelRule.Spec.Priority) + return nil } - updatePriority := false - - for _, rule := range resRule { - ruleResp, err := r.rule.Create(ctx, rule) - if err != nil { - return err + for i, rtg := range modelRule.Spec.Action.TargetGroups { + if rtg.StackTargetGroupId == "" && rtg.SvcImportTG == nil && rtg.LatticeTgId == "" { + return errors.New("rule TG is missing a required target group identifier") } - if ruleResp.UpdatePriorityNeeded { - updatePriority = true + if rtg.LatticeTgId != "" { + r.log.Debugf("Rule TG %d already resolved %s", i, rtg.LatticeTgId) + continue } - r.log.Debugf("Synthesise rule %s, ruleResp: %+v", rule.Spec.RuleID, ruleResp) - rule.Status = &ruleResp - } - - // handle delete - sdkRules, err := r.getSDKRules(ctx) - if err != nil { - r.log.Debugf("Error while getting rules due to %s", err) - } + if rtg.StackTargetGroupId != "" { + r.log.Debugf("Fetching TG %d from the stack (ID %s)", i, rtg.StackTargetGroupId) - for _, sdkRule := range sdkRules { - _, err := r.findMatchedRule(ctx, sdkRule.RuleID, sdkRule.ListenerID, sdkRule.ServiceID, resRule) - if err != nil { - r.log.Debugf("Error while finding matching rule for service %s, listener %s, rule %s. %s", - sdkRule.ServiceID, sdkRule.ListenerID, sdkRule.RuleID, err) - err := r.rule.Delete(ctx, sdkRule.RuleID, sdkRule.ListenerID, sdkRule.ServiceID) + resTg, err := r.stack.GetResource(rtg.StackTargetGroupId, &model.TargetGroup{}) if err != nil { - r.log.Debugf("Error while deleting rule for service %s, listener %s, rule %s. %s", - sdkRule.ServiceID, sdkRule.ListenerID, sdkRule.RuleID, err) + return err + } + + stackTg, ok := resTg.(*model.TargetGroup) + if !ok { + return errors.New("unexpected type conversion failure for target group stack object") + } + + if stackTg.Status == nil { + return errors.New("stack target group is missing Status field") } + rtg.LatticeTgId = stackTg.Status.Id } - } - if updatePriority { - err := r.rule.Update(ctx, resRule) - if err != nil { - r.log.Debugf("Error while updating rule priority for rules %+v. %s", resRule, err) + if rtg.SvcImportTG != nil { + r.log.Debugf("Getting target group for service import %s %s (%s, %s)", + rtg.SvcImportTG.K8SServiceName, rtg.SvcImportTG.K8SServiceNamespace, + rtg.SvcImportTG.EKSClusterName, rtg.SvcImportTG.VpcId) + tgId, err := r.findSvcExportTG(ctx, *rtg.SvcImportTG) + + if err != nil { + return err + } + rtg.LatticeTgId = tgId } } return nil } -func (r *ruleSynthesizer) findMatchedRule( - ctx context.Context, - sdkRuleId string, - listener string, - service string, - resRule []*model.Rule, -) (*model.Rule, error) { - var modelRule *model.Rule = nil - sdkRuleDetail, err := r.rule.Get(ctx, service, listener, sdkRuleId) +func (r *ruleSynthesizer) findSvcExportTG(ctx context.Context, svcImportTg model.SvcImportTargetGroup) (string, error) { + tgs, err := r.tgManager.List(ctx) if err != nil { - return modelRule, err + return "", err } - if sdkRuleDetail.Match == nil || - sdkRuleDetail.Match.HttpMatch == nil { - return modelRule, errors.New("rule not found, no HTTPMatch") - } + for _, tg := range tgs { + if tg.targetGroupTags == nil { + continue + } + + tgTags := model.TGTagFieldsFromTags(tg.targetGroupTags.Tags) + + svcMatch := tgTags.IsServiceExport() && (tgTags.K8SServiceName == svcImportTg.K8SServiceName) && + (tgTags.K8SServiceNamespace == svcImportTg.K8SServiceNamespace) + + clusterMatch := (svcImportTg.EKSClusterName == "") || (tgTags.EKSClusterName == svcImportTg.EKSClusterName) + + vpcMatch := (svcImportTg.VpcId == "") || (svcImportTg.VpcId == aws.StringValue(tg.getTargetGroupOutput.Config.VpcIdentifier)) - for _, modelRule := range resRule { - sameRule := isRulesSame(r.log, modelRule, sdkRuleDetail) - if sameRule { - return modelRule, nil + if svcMatch && clusterMatch && vpcMatch { + return *tg.getTargetGroupOutput.Id, nil } } - return modelRule, fmt.Errorf("failed to find matching rule in model for rule %s", sdkRuleId) + return "", errors.New("target group for service import could not be found") } -func (r *ruleSynthesizer) getSDKRules(ctx context.Context) ([]*model.RuleStatus, error) { - var sdkRules []*model.RuleStatus - var resService []*model.Service - var resListener []*model.Listener +// helper types for checking which leftover rules are no longer referenced +// and need to be deleted +type ruleIdMap map[string]*model.Rule +type snlKey struct { + SvcId string + ListenerId string +} + +func (r *ruleSynthesizer) Synthesize(ctx context.Context) error { var resRule []*model.Rule - err := r.stack.ListResources(&resService) + err := r.stack.ListResources(&resRule) if err != nil { - r.log.Errorf("Error listing services: %s", err) + return err + } + + // svc id -> listener id -> rule id + snlStackRules := make(map[snlKey]ruleIdMap) + + for _, rule := range resRule { + // this will also populate our map with rules for each service+listener + err = r.createOrUpdateRules(ctx, rule, snlStackRules) + if err != nil { + return err + } } - err = r.stack.ListResources(&resListener) + // for each service/listener, remove any lingering lattice rules + err = r.deleteStaleLatticeRules(ctx, snlStackRules) if err != nil { - r.log.Errorf("Error listing listeners: %s", err) + return err } - err = r.stack.ListResources(&resRule) + // now we have a clean set of rules, update priorities accordingly + err = r.adjustPriorities(ctx, snlStackRules, resRule) if err != nil { - r.log.Errorf("Error listing rules: %s", err) + return err } - for _, service := range resService { - latticeService, err := r.rule.Cloud().Lattice().FindService(ctx, service) - if err != nil { - return sdkRules, fmt.Errorf("failed to find service %s-%s, %s", - service.Spec.Name, service.Spec.Namespace, err) - } + return nil +} + +func (r *ruleSynthesizer) createOrUpdateRules(ctx context.Context, rule *model.Rule, snlRules map[snlKey]ruleIdMap) error { + stackListener, stackSvc, err := r.getStackObjects(rule) + if err != nil { + return err + } + + err = r.resolveRuleTgIds(ctx, rule) + if err != nil { + return err + } + + status, err := r.ruleManager.Upsert(ctx, rule, stackListener, stackSvc) + if err != nil { + return fmt.Errorf("Failed RuleManager.Upsert due to %s", err) + } + rule.Status = &status + + // build a map svc + listener -> all current rules + key := snlKey{ + SvcId: stackSvc.Status.Id, + ListenerId: stackListener.Status.Id, + } + var ok bool + var ruleMap ruleIdMap + if ruleMap, ok = snlRules[key]; !ok { + // create and add a map if there isn't one already + ruleMap = make(ruleIdMap) + snlRules[key] = ruleMap + } - listeners, err := r.latticestore.GetAllListeners(service.Spec.Name, service.Spec.Namespace) + ruleMap[rule.Status.Id] = rule + return nil +} + +func (r *ruleSynthesizer) deleteStaleLatticeRules(ctx context.Context, snlRules map[snlKey]ruleIdMap) error { + var lastDelErr error + for snl := range snlRules { + allLatticeRules, err := r.ruleManager.List(ctx, snl.SvcId, snl.ListenerId) if err != nil { - return sdkRules, err + return fmt.Errorf("Failed RuleManager.List %s/%s, due to %s", snl.SvcId, snl.ListenerId, err) } - if len(listeners) == 0 { - return sdkRules, errors.New("failed to find listener in store") + activeRules, _ := snlRules[snl] + for _, lr := range allLatticeRules { + if aws.BoolValue(lr.IsDefault) { + continue + } + + // if the rule is not in our list of ids, we need to remove it + // make sure to skip the default + ruleId := aws.StringValue(lr.Id) + if _, ok := activeRules[ruleId]; !ok { + err := r.ruleManager.Delete(ctx, ruleId, snl.SvcId, snl.ListenerId) + if err != nil { + r.log.Infof("Failed RuleManager.Delete %s/%s/%s, due to %s", snl.SvcId, snl.ListenerId, ruleId, err) + err = lastDelErr + } + } } + } + if lastDelErr != nil { + return lastDelErr + } + return nil +} + +func (r *ruleSynthesizer) adjustPriorities(ctx context.Context, snlStackRules map[snlKey]ruleIdMap, resRule []*model.Rule) error { + var lastUpdateErr error + for snl := range snlStackRules { + activeRules, _ := snlStackRules[snl] + for _, rule := range activeRules { + if rule.Spec.Priority != rule.Status.Priority { + // *any* mismatch in priority prompts a batch update of ALL priorities + r.log.Debugf("Found rule priority mismatch, update required") + + var rulesToUpdate []*model.Rule + for _, snlRule := range activeRules { + rulesToUpdate = append(rulesToUpdate, snlRule) + } - for _, listener := range listeners { - rules, _ := r.rule.List(ctx, aws.StringValue(latticeService.Id), listener.ID) - sdkRules = append(sdkRules, rules...) + err := r.ruleManager.UpdatePriorities(ctx, snl.SvcId, snl.ListenerId, rulesToUpdate) + if err != nil { + r.log.Infof("Failed RuleManager.UpdatePriorities for rules %+v due to %s", resRule, err) + lastUpdateErr = err + } + break + } } } - return sdkRules, nil + if lastUpdateErr != nil { + return lastUpdateErr + } + return nil +} + +func (r *ruleSynthesizer) getStackObjects(rule *model.Rule) (*model.Listener, *model.Service, error) { + resListener, err := r.stack.GetResource(rule.Spec.StackListenerId, &model.Listener{}) + if err != nil { + return nil, nil, err + } + + listener, ok := resListener.(*model.Listener) + if !ok { + return nil, nil, errors.New("unexpected type conversion failure for listener stack object") + } + + resSvc, err := r.stack.GetResource(listener.Spec.StackServiceId, &model.Service{}) + if err != nil { + return nil, nil, err + } + + svc, ok := resSvc.(*model.Service) + if !ok { + return nil, nil, errors.New("unexpected type conversion failure for service stack object") + } + + return listener, svc, nil } func (r *ruleSynthesizer) PostSynthesize(ctx context.Context) error { diff --git a/pkg/deploy/lattice/rule_synthesizer_test.go b/pkg/deploy/lattice/rule_synthesizer_test.go index fbfc01bf..b42d4178 100644 --- a/pkg/deploy/lattice/rule_synthesizer_test.go +++ b/pkg/deploy/lattice/rule_synthesizer_test.go @@ -2,411 +2,199 @@ package lattice import ( "context" - "github.com/aws/aws-sdk-go/aws" - - mocks_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - mocks "github.com/aws/aws-application-networking-k8s/pkg/aws/services" + "github.com/aws/aws-application-networking-k8s/pkg/config" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - - //"errors" - "fmt" - "testing" - + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/vpclattice" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - - "github.com/aws/aws-sdk-go/service/vpclattice" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + "testing" + "time" ) func Test_SynthesizeRule(t *testing.T) { - //var httpSectionName gwv1beta1.SectionName = "http" - var serviceKind gwv1beta1.Kind = "Service" - var serviceimportKind gwv1beta1.Kind = "ServiceImport" - var weight1 = int32(10) - var weight2 = int32(90) - var namespace = gwv1beta1.Namespace("default") - var path1 = string("/ver1") - var path2 = string("/ver2") - var backendRef1 = gwv1beta1.BackendRef{ - BackendObjectReference: gwv1beta1.BackendObjectReference{ - Name: "targetgroup1", - Namespace: &namespace, - Kind: &serviceKind, - }, - Weight: &weight1, + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockRuleMgr := NewMockRuleManager(c) + mockTgMgr := NewMockTargetGroupManager(c) + + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) + + // each rule references a stack and a service which need to be present in the stack + // in order to proceed, these just need their status+id + svc := &model.Service{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Service", "svc-id"), + Status: &model.ServiceStatus{Id: "svc-id"}, } - var backendRef2 = gwv1beta1.BackendRef{ - BackendObjectReference: gwv1beta1.BackendObjectReference{ - Name: "targetgroup2", - Namespace: &namespace, - Kind: &serviceimportKind, + assert.NoError(t, stack.AddResource(svc)) + + l := &model.Listener{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Listener", "listener-id"), + Spec: model.ListenerSpec{StackServiceId: svc.ID()}, + Status: &model.ListenerStatus{Id: "listener-id"}, + } + assert.NoError(t, stack.AddResource(l)) + + // then we resolve target groups, which sets the LatticeTgId field on each rule + // these can already be populated, or can come from the stack as a svcExport or svc + // we unit test tg resolution separately, so we'll take the easy way here and not + // have any actions/tg references + r := &model.Rule{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Rule", "rule-id"), + Spec: model.RuleSpec{ + StackListenerId: l.ID(), + Priority: 1, + CreateTime: time.Time{}, }, - Weight: &weight2, + Status: nil, } - /* - var backendServiceImportRef = gwv1beta1.BackendRef{ - BackendObjectReference: gwv1beta1.BackendObjectReference{ - Name: "targetgroup1", - Kind: &serviceimportKind, - }, - } - */ + assert.NoError(t, stack.AddResource(r)) + + // then we call create on the rule manager, using the rule status + // if there were no pre-existing rules, then we're done + t.Run("no pre-existing rules", func(t *testing.T) { + mockRuleMgr.EXPECT().Upsert(ctx, r, l, svc).Return(model.RuleStatus{ + Id: "rule-id", + Priority: 1, // <-- this matching means we don't update rules + }, nil) - tests := []struct { - name string - gwListenerPort gwv1beta1.PortNumber - httpRoute *gwv1beta1.HTTPRoute - listenerARN string - listenerID string - serviceARN string - serviceID string - rulespec []model.RuleSpec - updatedTGs bool - mgrErr error - wantErrIsNil bool - wantIsDeleted bool - }{ - { - name: "test1: Add Rule", - gwListenerPort: *PortNumberPtr(80), - httpRoute: &gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", + mockRuleMgr.EXPECT().List(ctx, "svc-id", "listener-id").Return( + []*vpclattice.RuleSummary{ + { + Id: aws.String("default-id"), + IsDefault: aws.Bool(true), }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gateway1", - }, - }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ - { - Matches: []gwv1beta1.HTTPRouteMatch{ - { + { + Id: aws.String("rule-id"), + }, + }, nil) - Path: &gwv1beta1.HTTPPathMatch{ + rs := NewRuleSynthesizer(gwlog.FallbackLogger, mockRuleMgr, mockTgMgr, stack) + rs.Synthesize(ctx) + }) - Value: &path1, - }, - }, - }, - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef1, - }, - }, - }, - { - Matches: []gwv1beta1.HTTPRouteMatch{ - { - - Path: &gwv1beta1.HTTPPathMatch{ + // if there were pre-existing rules, we need to remove the previous ones that are no longer valid + t.Run("pre-existing rule to remove", func(t *testing.T) { + mockRuleMgr.EXPECT().Upsert(ctx, r, l, svc).Return(model.RuleStatus{ + Id: "rule-id", + Priority: 1, + }, nil) - Value: &path2, - }, - }, - }, - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef2, - }, - }, - }, - }, - }, - }, - rulespec: []model.RuleSpec{ + mockRuleMgr.EXPECT().List(ctx, "svc-id", "listener-id").Return( + []*vpclattice.RuleSummary{ { - PathMatchPrefix: true, - PathMatchValue: path1, + Id: aws.String("default-id"), + IsDefault: aws.Bool(true), }, { - PathMatchPrefix: true, - PathMatchValue: path2, + Id: aws.String("rule-id"), }, - }, - - listenerARN: "arn1234", - listenerID: "1234", - serviceARN: "arn56789", - serviceID: "56789", - updatedTGs: false, - mgrErr: nil, - wantIsDeleted: false, - wantErrIsNil: true, - }, - { - name: "Test2: Add Rule", - gwListenerPort: *PortNumberPtr(80), - httpRoute: &gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", + { + Id: aws.String("delete-rule-id"), // <-- should delete this rule }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gateway1", - }, - }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ - { - Matches: []gwv1beta1.HTTPRouteMatch{ - { + }, nil) - Path: &gwv1beta1.HTTPPathMatch{ + mockRuleMgr.EXPECT().Delete(ctx, "delete-rule-id", "svc-id", "listener-id").Return(nil) - Value: &path1, - }, - }, - }, - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef1, - }, - }, - }, - { - Matches: []gwv1beta1.HTTPRouteMatch{ - { + rs := NewRuleSynthesizer(gwlog.FallbackLogger, mockRuleMgr, mockTgMgr, stack) + rs.Synthesize(ctx) + }) - Path: &gwv1beta1.HTTPPathMatch{ + // if there are pre-existing rules, we need to update priorities afterward + t.Run("pre-existing rule to update", func(t *testing.T) { + mockRuleMgr.EXPECT().Upsert(ctx, r, l, svc).Return(model.RuleStatus{ + Id: "rule-id", + Priority: r.Spec.Priority + 1, // <-- this should trigger an update + }, nil) - Value: &path2, - }, - }, - }, - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef2, - }, - }, - }, - }, - }, - }, - rulespec: []model.RuleSpec{ + mockRuleMgr.EXPECT().List(ctx, "svc-id", "listener-id").Return( + []*vpclattice.RuleSummary{ { - PathMatchPrefix: true, - PathMatchValue: path1, + Id: aws.String("default-id"), + IsDefault: aws.Bool(true), }, { - PathMatchPrefix: true, - PathMatchValue: path2, + Id: aws.String("rule-id"), }, - }, - - listenerARN: "arn1234", - listenerID: "1234", - serviceARN: "arn56789", - serviceID: "56789", - updatedTGs: true, - mgrErr: nil, - wantIsDeleted: false, - wantErrIsNil: true, - }, - } + }, nil) - var protocol = "HTTP" + mockRuleMgr.EXPECT().UpdatePriorities(ctx, "svc-id", "listener-id", gomock.Any()).DoAndReturn( + func(ctx context.Context, svcId string, listenerId string, rules []*model.Rule) error { + assert.Equal(t, 1, len(rules)) + return nil + }) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - ds := latticestore.NewLatticeDataStore() - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.httpRoute))) - - mockRuleManager := NewMockRuleManager(c) - - var ruleID = 1 - for i, httpRule := range tt.httpRoute.Spec.Rules { - //var ruleValue string - tgList := []*model.RuleTargetGroup{} - - for _, httpBackendRef := range httpRule.BackendRefs { - ruleTG := model.RuleTargetGroup{} - - ruleTG.Name = string(httpBackendRef.Name) - ruleTG.Namespace = string(*httpBackendRef.Namespace) - - if httpBackendRef.Weight != nil { - ruleTG.Weight = int64(*httpBackendRef.Weight) - } - - tgList = append(tgList, &ruleTG) - } - - ruleIDName := fmt.Sprintf("rule-%d", ruleID) - ruleAction := model.RuleAction{ - TargetGroups: tgList, - } - rule := model.NewRule(stack, ruleIDName, tt.httpRoute.Name, tt.httpRoute.Namespace, int64(tt.gwListenerPort), - protocol, ruleAction, tt.rulespec[i]) - - var ruleResp model.RuleStatus - - if tt.updatedTGs { - ruleResp.UpdatePriorityNeeded = true - } else { - ruleResp.UpdatePriorityNeeded = false - } - mockRuleManager.EXPECT().Create(ctx, rule).Return(ruleResp, nil) - - ruleID++ - - } - - var resRule []*model.Rule - stack.ListResources(&resRule) - - if tt.updatedTGs { - // TODO, resRule return from stack.ListResources is not consistent with the ordering - // so we use gomock.Any() instead of resRule below - mockRuleManager.EXPECT().Update(ctx, gomock.Any()) - } - - synthesizer := NewRuleSynthesizer(gwlog.FallbackLogger, mockRuleManager, stack, ds) - - err := synthesizer.Synthesize(ctx) - - if tt.wantErrIsNil { - assert.Nil(t, err) - } else { - assert.NotNil(t, err) - } - }) - } + rs := NewRuleSynthesizer(gwlog.FallbackLogger, mockRuleMgr, mockTgMgr, stack) + rs.Synthesize(ctx) + }) } -func Test_SynthesizeDeleteRule(t *testing.T) { +func Test_resolveRuleTgs(t *testing.T) { + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" + c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() + mockRuleMgr := NewMockRuleManager(c) + mockTgMgr := NewMockTargetGroupManager(c) - ds := latticestore.NewLatticeDataStore() - - mockRuleManager := NewMockRuleManager(c) - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := mocks.NewMockLattice(c) - - mockRuleManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) - var serviceName = "service1" - var serviceNamespace = "test" - var serviceID = "service1-id" - - var httpRoute = gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: serviceName, - }, - } - - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(&httpRoute.ObjectMeta))) - pro := "HTTP" - protocols := []*string{&pro} - spec := model.ServiceSpec{ - Name: serviceName, - Namespace: serviceNamespace, - Protocols: protocols, + tg := &model.TargetGroup{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", "stack-tg-id"), + Status: &model.TargetGroupStatus{Id: "tg-id"}, } - stackService := model.NewLatticeService(stack, "", spec) - - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String(stackService.LatticeServiceName()), - Arn: aws.String("svc-arn"), - Id: aws.String(serviceID), - }, nil) - - rule1 := model.RuleStatus{ - RuleARN: "rule1-arn", - RuleID: "rule1-id", - Priority: 1, - ServiceID: serviceID, - ListenerID: "listener1-ID", - } - - rule2 := model.RuleStatus{ - RuleARN: "rule2-arn", - RuleID: "rule2-id", - Priority: 2, - ServiceID: serviceID, - ListenerID: "listener1-ID", - } - - rule3 := model.RuleStatus{ - RuleARN: "rule3-arn", - RuleID: "rule3-id", - Priority: 1, - ServiceID: serviceID, - ListenerID: "listener2-ID", - } - - rule4 := model.RuleStatus{ - RuleARN: "rule4-arn", - RuleID: "rule4-id", - Priority: 2, - ServiceID: serviceID, - ListenerID: "listener2-ID", - } - - listeners := []struct { - port int64 - listenerARN string - listenerID string - - rulelist []*model.RuleStatus - }{ - { - port: 80, - listenerARN: "listener1-ARN", - listenerID: "listener1-ID", - rulelist: []*model.RuleStatus{ - &rule1, - &rule2, - }, - }, - { - port: 443, - listenerARN: "listener2-ARN", - listenerID: "listener2-ID", - rulelist: []*model.RuleStatus{ - &rule3, - &rule4, + assert.NoError(t, stack.AddResource(tg)) + + r := &model.Rule{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Rule", "rule-id"), + Spec: model.RuleSpec{ + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + SvcImportTG: &model.SvcImportTargetGroup{ + EKSClusterName: "cluster-name", + K8SServiceName: "svc-name", + K8SServiceNamespace: "ns", + VpcId: "vpc-id", + }, + }, + { + StackTargetGroupId: "stack-tg-id", + }, + }, }, }, } + assert.NoError(t, stack.AddResource(r)) + + mockTgMgr.EXPECT().List(ctx).Return( + []tgListOutput{ + { + getTargetGroupOutput: vpclattice.GetTargetGroupOutput{ + Arn: aws.String("svc-export-tg-arn"), + Config: &vpclattice.TargetGroupConfig{ + VpcIdentifier: aws.String("vpc-id"), + }, + Id: aws.String("svc-export-tg-id"), + Name: aws.String("svc-export-tg-name"), + }, + targetGroupTags: &vpclattice.ListTagsForResourceOutput{Tags: map[string]*string{ + model.K8SServiceNameKey: aws.String("svc-name"), + model.K8SServiceNamespaceKey: aws.String("ns"), + model.EKSClusterNameKey: aws.String("cluster-name"), + model.K8SParentRefTypeKey: aws.String(string(model.ParentRefTypeSvcExport)), + }}, + }, + }, nil) - for _, listener := range listeners { - ds.AddListener(serviceName, serviceNamespace, listener.port, "HTTP", - listener.listenerARN, listener.listenerID) - - mockRuleManager.EXPECT().List(ctx, serviceID, listener.listenerID).Return(listener.rulelist, nil) - - for _, rule := range listener.rulelist { - sdkRuleDetail := vpclattice.GetRuleOutput{} - - mockRuleManager.EXPECT().Get(ctx, serviceID, listener.listenerID, rule.RuleID).Return(&sdkRuleDetail, nil) - mockRuleManager.EXPECT().Delete(ctx, rule.RuleID, listener.listenerID, serviceID) - } - - } - - synthesizer := NewRuleSynthesizer(gwlog.FallbackLogger, mockRuleManager, stack, ds) + rs := NewRuleSynthesizer(gwlog.FallbackLogger, mockRuleMgr, mockTgMgr, stack) + assert.NoError(t, rs.resolveRuleTgIds(ctx, r)) - err := synthesizer.Synthesize(ctx) - assert.Nil(t, err) + assert.Equal(t, "svc-export-tg-id", r.Spec.Action.TargetGroups[0].LatticeTgId) + assert.Equal(t, "tg-id", r.Spec.Action.TargetGroups[1].LatticeTgId) } diff --git a/pkg/deploy/lattice/service_manager.go b/pkg/deploy/lattice/service_manager.go index 2e27bac4..838ce550 100644 --- a/pkg/deploy/lattice/service_manager.go +++ b/pkg/deploy/lattice/service_manager.go @@ -5,12 +5,12 @@ import ( "fmt" "github.com/aws/aws-application-networking-k8s/pkg/aws/services" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/vpclattice" pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) @@ -32,19 +32,19 @@ type ListSnSvcAssocsReq = vpclattice.ListServiceNetworkServiceAssociationsInput type SnSvcAssocSummary = vpclattice.ServiceNetworkServiceAssociationSummary type ServiceManager interface { - Create(ctx context.Context, service *model.Service) (model.ServiceStatus, error) + Upsert(ctx context.Context, service *model.Service) (model.ServiceStatus, error) Delete(ctx context.Context, service *model.Service) error } type defaultServiceManager struct { - cloud pkg_aws.Cloud - datastore *latticestore.LatticeDataStore + log gwlog.Logger + cloud pkg_aws.Cloud } -func NewServiceManager(cloud pkg_aws.Cloud, latticeDataStore *latticestore.LatticeDataStore) *defaultServiceManager { +func NewServiceManager(log gwlog.Logger, cloud pkg_aws.Cloud) *defaultServiceManager { return &defaultServiceManager{ - cloud: cloud, - datastore: latticeDataStore, + log: log, + cloud: cloud, } } @@ -52,9 +52,12 @@ func (m *defaultServiceManager) createServiceAndAssociate(ctx context.Context, s createSvcReq := m.newCreateSvcReq(svc) createSvcResp, err := m.cloud.Lattice().CreateServiceWithContext(ctx, createSvcReq) if err != nil { - return ServiceInfo{}, err + return ServiceInfo{}, fmt.Errorf("Failed CreateService %s due to %s", aws.StringValue(createSvcReq.Name), err) } + m.log.Infof("Success CreateService %s %s", + aws.StringValue(createSvcResp.Name), aws.StringValue(createSvcResp.Id)) + for _, snName := range svc.Spec.ServiceNetworkNames { err = m.createAssociation(ctx, createSvcResp.Id, snName) if err != nil { @@ -78,8 +81,12 @@ func (m *defaultServiceManager) createAssociation(ctx context.Context, svcId *st } assocResp, err := m.cloud.Lattice().CreateServiceNetworkServiceAssociationWithContext(ctx, assocReq) if err != nil { - return err + return fmt.Errorf("Failed CreateServiceNetworkServiceAssociation %s %s due to %s", + aws.StringValue(assocReq.ServiceNetworkIdentifier), aws.StringValue(assocReq.ServiceIdentifier), err) } + m.log.Infof("Success CreateServiceNetworkServiceAssociation %s %s", + aws.StringValue(assocReq.ServiceNetworkIdentifier), aws.StringValue(assocReq.ServiceIdentifier)) + err = handleCreateAssociationResp(assocResp) if err != nil { return err @@ -262,8 +269,11 @@ func (m *defaultServiceManager) deleteAssociation(ctx context.Context, assocArn delReq := &DelSnSvcAssocReq{ServiceNetworkServiceAssociationIdentifier: assocArn} _, err := m.cloud.Lattice().DeleteServiceNetworkServiceAssociationWithContext(ctx, delReq) if err != nil { - return err + return fmt.Errorf("Failed DeleteServiceNetworkServiceAssociation %s due to %s", + aws.StringValue(assocArn), err) } + + m.log.Infof("Success DeleteServiceNetworkServiceAssociation %s", aws.StringValue(assocArn)) return nil } @@ -272,12 +282,17 @@ func (m *defaultServiceManager) deleteService(ctx context.Context, svc *SvcSumma ServiceIdentifier: svc.Id, } _, err := m.cloud.Lattice().DeleteServiceWithContext(ctx, &delInput) - return err + if err != nil { + return fmt.Errorf("Failed DeleteService %s due to %s", aws.StringValue(svc.Id), err) + } + + m.log.Infof("Success DeleteService %s", svc.Id) + return nil } // Create or update Service and ServiceNetwork-Service associations -func (m *defaultServiceManager) Create(ctx context.Context, svc *Service) (ServiceInfo, error) { - svcSum, err := m.cloud.Lattice().FindService(ctx, svc) +func (m *defaultServiceManager) Upsert(ctx context.Context, svc *Service) (ServiceInfo, error) { + svcSum, err := m.cloud.Lattice().FindService(ctx, svc.LatticeServiceName()) if err != nil && !services.IsNotFoundError(err) { return ServiceInfo{}, err } @@ -295,7 +310,7 @@ func (m *defaultServiceManager) Create(ctx context.Context, svc *Service) (Servi } func (m *defaultServiceManager) Delete(ctx context.Context, svc *Service) error { - svcSum, err := m.cloud.Lattice().FindService(ctx, svc) + svcSum, err := m.cloud.Lattice().FindService(ctx, svc.LatticeServiceName()) if err != nil { if services.IsNotFoundError(err) { return nil // already deleted diff --git a/pkg/deploy/lattice/service_manager_mock.go b/pkg/deploy/lattice/service_manager_mock.go index ca4788ba..905969bf 100644 --- a/pkg/deploy/lattice/service_manager_mock.go +++ b/pkg/deploy/lattice/service_manager_mock.go @@ -35,21 +35,6 @@ func (m *MockServiceManager) EXPECT() *MockServiceManagerMockRecorder { return m.recorder } -// Create mocks base method. -func (m *MockServiceManager) Create(arg0 context.Context, arg1 *lattice.Service) (lattice.ServiceStatus, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", arg0, arg1) - ret0, _ := ret[0].(lattice.ServiceStatus) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Create indicates an expected call of Create. -func (mr *MockServiceManagerMockRecorder) Create(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockServiceManager)(nil).Create), arg0, arg1) -} - // Delete mocks base method. func (m *MockServiceManager) Delete(arg0 context.Context, arg1 *lattice.Service) error { m.ctrl.T.Helper() @@ -63,3 +48,18 @@ func (mr *MockServiceManagerMockRecorder) Delete(arg0, arg1 interface{}) *gomock mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockServiceManager)(nil).Delete), arg0, arg1) } + +// Upsert mocks base method. +func (m *MockServiceManager) Upsert(arg0 context.Context, arg1 *lattice.Service) (lattice.ServiceStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Upsert", arg0, arg1) + ret0, _ := ret[0].(lattice.ServiceStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Upsert indicates an expected call of Upsert. +func (mr *MockServiceManagerMockRecorder) Upsert(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockServiceManager)(nil).Upsert), arg0, arg1) +} diff --git a/pkg/deploy/lattice/service_manager_test.go b/pkg/deploy/lattice/service_manager_test.go index 72c0ff3b..167c0ccd 100644 --- a/pkg/deploy/lattice/service_manager_test.go +++ b/pkg/deploy/lattice/service_manager_test.go @@ -3,16 +3,15 @@ package lattice import ( "context" "errors" - "testing" - pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" mocks "github.com/aws/aws-application-networking-k8s/pkg/aws/services" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/vpclattice" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "testing" ) func TestServiceManagerInteg(t *testing.T) { @@ -22,9 +21,8 @@ func TestServiceManagerInteg(t *testing.T) { mockLattice := mocks.NewMockLattice(c) cfg := pkg_aws.CloudConfig{VpcId: "vpc-id", AccountId: "account-id"} cl := pkg_aws.NewDefaultCloud(mockLattice, cfg) - ds := latticestore.NewLatticeDataStore() ctx := context.Background() - m := NewServiceManager(cl, ds) + m := NewServiceManager(gwlog.FallbackLogger, cl) // Case for single service and single sn-svc association // Make sure that we send requests to Lattice for create Service and create Sn-Svc @@ -89,7 +87,7 @@ func TestServiceManagerInteg(t *testing.T) { }). Times(1) - status, err := m.Create(ctx, svc) + status, err := m.Upsert(ctx, svc) assert.Nil(t, err) assert.Equal(t, "arn", status.Arn) }) @@ -190,7 +188,7 @@ func TestServiceManagerInteg(t *testing.T) { }). AnyTimes() - status, err := m.Create(ctx, svc) + status, err := m.Upsert(ctx, svc) assert.Nil(t, err) assert.Equal(t, "svc-arn", status.Arn) }) @@ -242,8 +240,7 @@ func TestServiceManagerInteg(t *testing.T) { func TestCreateSvcReq(t *testing.T) { cfg := pkg_aws.CloudConfig{VpcId: "vpc-id", AccountId: "account-id"} cl := pkg_aws.NewDefaultCloud(nil, cfg) - ds := latticestore.NewLatticeDataStore() - m := NewServiceManager(cl, ds) + m := NewServiceManager(gwlog.FallbackLogger, cl) spec := model.ServiceSpec{ Name: "name", diff --git a/pkg/deploy/lattice/service_network_manager.go b/pkg/deploy/lattice/service_network_manager.go index b2da5b57..ad878bc7 100644 --- a/pkg/deploy/lattice/service_network_manager.go +++ b/pkg/deploy/lattice/service_network_manager.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "golang.org/x/exp/slices" "github.com/aws/aws-application-networking-k8s/pkg/aws/services" @@ -122,16 +121,12 @@ func (m *defaultServiceNetworkManager) CreateOrUpdate(ctx context.Context, servi } serviceNetworkVpcAssociationStatus := aws.StringValue(resp.Status) - switch serviceNetworkVpcAssociationStatus { - case vpclattice.ServiceNetworkVpcAssociationStatusCreateInProgress: - return model.ServiceNetworkStatus{ServiceNetworkARN: "", ServiceNetworkID: ""}, errors.New(LATTICE_RETRY) - case vpclattice.ServiceNetworkVpcAssociationStatusActive: + if serviceNetworkVpcAssociationStatus == vpclattice.ServiceNetworkVpcAssociationStatusActive { return model.ServiceNetworkStatus{ServiceNetworkARN: serviceNetworkArn, ServiceNetworkID: serviceNetworkId}, nil - case vpclattice.ServiceNetworkVpcAssociationStatusCreateFailed: - return model.ServiceNetworkStatus{ServiceNetworkARN: "", ServiceNetworkID: ""}, errors.New(LATTICE_RETRY) - case vpclattice.ServiceNetworkVpcAssociationStatusDeleteFailed: - return model.ServiceNetworkStatus{ServiceNetworkARN: "", ServiceNetworkID: ""}, errors.New(LATTICE_RETRY) - case vpclattice.ServiceNetworkVpcAssociationStatusDeleteInProgress: + } else { + m.log.Infof("Service network/vpc association is not in the active state. State is %s, will retry", + serviceNetworkVpcAssociationStatus) + return model.ServiceNetworkStatus{ServiceNetworkARN: "", ServiceNetworkID: ""}, errors.New(LATTICE_RETRY) } } diff --git a/pkg/deploy/lattice/service_network_manager_test.go b/pkg/deploy/lattice/service_network_manager_test.go index de6b9aeb..b1141f47 100644 --- a/pkg/deploy/lattice/service_network_manager_test.go +++ b/pkg/deploy/lattice/service_network_manager_test.go @@ -95,7 +95,7 @@ func Test_CreateOrUpdateServiceNetwork_SnNotExist_NeedToAssociate(t *testing.T) ServiceNetworkIdentifier: &snId, VpcIdentifier: &config.VpcID, } - associationStatus := vpclattice.ServiceNetworkVpcAssociationStatusUpdateInProgress + associationStatus := vpclattice.ServiceNetworkVpcAssociationStatusActive createServiceNetworkVPCAssociationOutput := &vpclattice.CreateServiceNetworkVpcAssociationOutput{ Status: &associationStatus, } @@ -110,8 +110,8 @@ func Test_CreateOrUpdateServiceNetwork_SnNotExist_NeedToAssociate(t *testing.T) resp, err := snMgr.CreateOrUpdate(ctx, &snCreateInput) assert.Nil(t, err) - assert.Equal(t, resp.ServiceNetworkARN, arn) - assert.Equal(t, resp.ServiceNetworkID, id) + assert.Equal(t, arn, resp.ServiceNetworkARN) + assert.Equal(t, id, resp.ServiceNetworkID) } // List and find sn does not work. diff --git a/pkg/deploy/lattice/service_synthesizer.go b/pkg/deploy/lattice/service_synthesizer.go index 1cd83a10..035eb538 100644 --- a/pkg/deploy/lattice/service_synthesizer.go +++ b/pkg/deploy/lattice/service_synthesizer.go @@ -4,7 +4,6 @@ import ( "context" "github.com/aws/aws-application-networking-k8s/pkg/deploy/externaldns" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -15,14 +14,12 @@ func NewServiceSynthesizer( serviceManager ServiceManager, dnsEndpointManager externaldns.DnsEndpointManager, stack core.Stack, - latticeDataStore *latticestore.LatticeDataStore, ) *serviceSynthesizer { return &serviceSynthesizer{ log: log, serviceManager: serviceManager, dnsEndpointManager: dnsEndpointManager, stack: stack, - latticeDataStore: latticeDataStore, } } @@ -31,57 +28,47 @@ type serviceSynthesizer struct { serviceManager ServiceManager dnsEndpointManager externaldns.DnsEndpointManager stack core.Stack - latticeDataStore *latticestore.LatticeDataStore } func (s *serviceSynthesizer) Synthesize(ctx context.Context) error { var resServices []*model.Service s.stack.ListResources(&resServices) + var lastErr error for _, resService := range resServices { s.log.Debugf("Synthesizing service: %s-%s", resService.Spec.Name, resService.Spec.Namespace) - if resService.Spec.IsDeleted { - // handle service delete + if resService.IsDeleted { err := s.serviceManager.Delete(ctx, resService) + if err != nil { + s.log.Infof("Failed ServiceManager.Delete %s-%s due to %s", + resService.Spec.Name, resService.Spec.Namespace, err) - if err == nil { - s.log.Debugf("Successfully synthesized service deletion %s-%s", resService.Spec.Name, resService.Spec.Namespace) - - // Also delete all listeners of this service - listeners, err := s.latticeDataStore.GetAllListeners(resService.Spec.Name, resService.Spec.Namespace) - if err != nil { - return err - } - - for _, l := range listeners { - err := s.latticeDataStore.DelListener(resService.Spec.Name, resService.Spec.Namespace, - l.Key.Port, l.Key.Protocol) - if err != nil { - s.log.Errorf("Error deleting listener for service %s-%s, port %d, protocol %s: %s", - resService.Spec.Name, resService.Spec.Namespace, l.Key.Port, l.Key.Protocol, err) - } - } - // Deleting DNSEndpoint is not required, as it has ownership relation. + lastErr = err + continue } - return err } else { - serviceStatus, err := s.serviceManager.Create(ctx, resService) + serviceStatus, err := s.serviceManager.Upsert(ctx, resService) if err != nil { - return err + s.log.Infof("Failed ServiceManager.Upsert %s-%s due to %s", + resService.Spec.Name, resService.Spec.Namespace, err) + + lastErr = err + continue } resService.Status = &serviceStatus err = s.dnsEndpointManager.Create(ctx, resService) if err != nil { - return err - } + s.log.Infof("Failed DnsEndpointManager.Create %s-%s due to %s", + resService.Spec.Name, resService.Spec.Namespace, err) - s.log.Debugf("Successfully created service %s-%s with status %s", - resService.Spec.Name, resService.Spec.Namespace, serviceStatus) + lastErr = err + continue + } } } - return nil + return lastErr } func (s *serviceSynthesizer) PostSynthesize(ctx context.Context) error { diff --git a/pkg/deploy/lattice/service_synthesizer_test.go b/pkg/deploy/lattice/service_synthesizer_test.go index 5e46ec7c..2caf4bb6 100644 --- a/pkg/deploy/lattice/service_synthesizer_test.go +++ b/pkg/deploy/lattice/service_synthesizer_test.go @@ -3,21 +3,16 @@ package lattice import ( "context" "errors" - "fmt" - "testing" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - "github.com/aws/aws-application-networking-k8s/pkg/deploy/externaldns" "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "testing" ) func Test_SynthesizeService(t *testing.T) { @@ -159,44 +154,33 @@ func Test_SynthesizeService(t *testing.T) { defer c.Finish() ctx := context.TODO() - ds := latticestore.NewLatticeDataStore() - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.httpRoute))) mockSvcManager := NewMockServiceManager(c) mockDnsManager := externaldns.NewMockDnsEndpointManager(c) - pro := "HTTP" - protocols := []*string{&pro} spec := model.ServiceSpec{ Name: tt.httpRoute.Name, Namespace: tt.httpRoute.Namespace, - Protocols: protocols, - } - - if tt.httpRoute.DeletionTimestamp.IsZero() { - spec.IsDeleted = false - } else { - spec.IsDeleted = true } - latticeService := model.NewLatticeService(stack, "", spec) - fmt.Printf("latticeService :%v\n", latticeService) + latticeService, err := model.NewLatticeService(stack, spec) + assert.Nil(t, err) + latticeService.IsDeleted = !tt.httpRoute.DeletionTimestamp.IsZero() - if tt.httpRoute.DeletionTimestamp.IsZero() { - mockSvcManager.EXPECT().Create(ctx, latticeService).Return(model.ServiceStatus{Arn: tt.serviceARN, Id: tt.serviceID}, tt.mgrErr) - } else { + if latticeService.IsDeleted { mockSvcManager.EXPECT().Delete(ctx, latticeService).Return(tt.mgrErr) + } else { + mockSvcManager.EXPECT().Upsert(ctx, latticeService).Return(model.ServiceStatus{Arn: tt.serviceARN, Id: tt.serviceID}, tt.mgrErr) } - if !spec.IsDeleted && tt.mgrErr == nil { + if !latticeService.IsDeleted && tt.mgrErr == nil { mockDnsManager.EXPECT().Create(ctx, gomock.Any()).Return(tt.dnsErr) } - synthesizer := NewServiceSynthesizer(gwlog.FallbackLogger, mockSvcManager, mockDnsManager, stack, ds) - - err := synthesizer.Synthesize(ctx) + synthesizer := NewServiceSynthesizer(gwlog.FallbackLogger, mockSvcManager, mockDnsManager, stack) + err = synthesizer.Synthesize(ctx) if tt.wantErrIsNil { assert.Nil(t, err) } else { diff --git a/pkg/deploy/lattice/target_group_manager.go b/pkg/deploy/lattice/target_group_manager.go index 608523a4..33b5a63c 100644 --- a/pkg/deploy/lattice/target_group_manager.go +++ b/pkg/deploy/lattice/target_group_manager.go @@ -3,26 +3,24 @@ package lattice import ( "context" "errors" - + "fmt" + "github.com/aws/aws-application-networking-k8s/pkg/aws/services" "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/vpclattice" - - "fmt" - "strings" + apierrors "k8s.io/apimachinery/pkg/api/errors" pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-application-networking-k8s/pkg/config" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" ) type TargetGroupManager interface { - Create(ctx context.Context, targetGroup *model.TargetGroup) (model.TargetGroupStatus, error) - Delete(ctx context.Context, targetGroup *model.TargetGroup) error - List(ctx context.Context) ([]targetGroupOutput, error) - Get(tx context.Context, targetGroup *model.TargetGroup) (model.TargetGroupStatus, error) + Upsert(ctx context.Context, modelTg *model.TargetGroup) (model.TargetGroupStatus, error) + Delete(ctx context.Context, modelTg *model.TargetGroup) error + List(ctx context.Context) ([]tgListOutput, error) + IsTargetGroupMatch(ctx context.Context, modelTg *model.TargetGroup, latticeTg *vpclattice.TargetGroupSummary, + latticeTags *model.TargetGroupTagFields) (bool, error) } type defaultTargetGroupManager struct { @@ -37,252 +35,213 @@ func NewTargetGroupManager(log gwlog.Logger, cloud pkg_aws.Cloud) *defaultTarget } } -// Determines the "actual" target group name used in VPC Lattice. -func getLatticeTGName(targetGroup *model.TargetGroup) string { - var ( - namePrefix = targetGroup.Spec.Name - protocol = strings.ToLower(targetGroup.Spec.Config.Protocol) - protocolVersion = strings.ToLower(targetGroup.Spec.Config.ProtocolVersion) - ) - if config.UseLongTGName { - namePrefix = latticestore.TargetGroupLongName(namePrefix, - targetGroup.Spec.Config.K8SHTTPRouteName, config.VpcID) - } - return fmt.Sprintf("%s-%s-%s", namePrefix, protocol, protocolVersion) -} - -// Create will try to create a target group -// return error when: -// -// ListTargetGroupsAsList() returns error -// CreateTargetGroupWithContext returns error -// -// return errors.New(LATTICE_RETRY) when: -// -// CreateTargetGroupWithContext returns -// TG is TargetGroupStatusUpdateInProgress -// TG is TargetGroupStatusCreateFailed -// TG is TargetGroupStatusCreateInProgress -// TG is TargetGroupStatusDeleteFailed -// TG is TargetGroupStatusDeleteInProgress -// -// return nil when: -// -// TG is TargetGroupStatusActive -func (s *defaultTargetGroupManager) Create( +func (s *defaultTargetGroupManager) Upsert( ctx context.Context, - targetGroup *model.TargetGroup, + modelTg *model.TargetGroup, ) (model.TargetGroupStatus, error) { - s.log.Debugf("Creating VPC Lattice Target Group %s", targetGroup.Spec.Name) - - latticeTGName := getLatticeTGName(targetGroup) // check if exists - tgSummary, err := s.findTargetGroup(ctx, targetGroup) - + latticeTgSummary, err := s.findTargetGroup(ctx, modelTg) if err != nil { - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, err + return model.TargetGroupStatus{}, err } - vpcLatticeSess := s.cloud.Lattice() - - // this means Target Group already existed, so this is an update request - if tgSummary != nil { - return s.update(ctx, targetGroup, tgSummary) + if latticeTgSummary == nil { + return s.create(ctx, modelTg, err) + } else { + return s.update(ctx, modelTg, latticeTgSummary) } +} - port := int64(targetGroup.Spec.Config.Port) - ipAddressType := &targetGroup.Spec.Config.IpAddressType - - // if IpAddressTypeIpv4 is not set, then default to nil - if targetGroup.Spec.Config.IpAddressType == "" { - ipAddressType = nil +func (s *defaultTargetGroupManager) create(ctx context.Context, modelTg *model.TargetGroup, err error) (model.TargetGroupStatus, error) { + var ipAddressType *string + if modelTg.Spec.IpAddressType != "" { + ipAddressType = &modelTg.Spec.IpAddressType } - tgConfig := &vpclattice.TargetGroupConfig{ - Port: &port, - Protocol: &targetGroup.Spec.Config.Protocol, - ProtocolVersion: &targetGroup.Spec.Config.ProtocolVersion, - VpcIdentifier: &targetGroup.Spec.Config.VpcID, + latticeTgCfg := &vpclattice.TargetGroupConfig{ + Port: aws.Int64(int64(modelTg.Spec.Port)), + Protocol: &modelTg.Spec.Protocol, + ProtocolVersion: &modelTg.Spec.ProtocolVersion, + VpcIdentifier: &modelTg.Spec.VpcId, IpAddressType: ipAddressType, - HealthCheck: targetGroup.Spec.Config.HealthCheckConfig, + HealthCheck: modelTg.Spec.HealthCheckConfig, } - targetGroupType := string(targetGroup.Spec.Type) + latticeTgType := string(modelTg.Spec.Type) - createTargetGroupInput := vpclattice.CreateTargetGroupInput{ - Config: tgConfig, - Name: &latticeTGName, - Type: &targetGroupType, + latticeTgName := model.GenerateTgName(modelTg.Spec) + createInput := vpclattice.CreateTargetGroupInput{ + Config: latticeTgCfg, + Name: &latticeTgName, + Type: &latticeTgType, Tags: s.cloud.DefaultTags(), } - createTargetGroupInput.Tags[model.K8SServiceNameKey] = &targetGroup.Spec.Config.K8SServiceName - createTargetGroupInput.Tags[model.K8SServiceNamespaceKey] = &targetGroup.Spec.Config.K8SServiceNamespace - if targetGroup.Spec.Config.IsServiceExport { - value := model.K8SServiceExportType - createTargetGroupInput.Tags[model.K8SParentRefTypeKey] = &value - } else { - value := model.K8SHTTPRouteType - createTargetGroupInput.Tags[model.K8SParentRefTypeKey] = &value - createTargetGroupInput.Tags[model.K8SHTTPRouteNameKey] = &targetGroup.Spec.Config.K8SHTTPRouteName - createTargetGroupInput.Tags[model.K8SHTTPRouteNamespaceKey] = &targetGroup.Spec.Config.K8SHTTPRouteNamespace + createInput.Tags[model.EKSClusterNameKey] = &modelTg.Spec.EKSClusterName + createInput.Tags[model.K8SServiceNameKey] = &modelTg.Spec.K8SServiceName + createInput.Tags[model.K8SServiceNamespaceKey] = &modelTg.Spec.K8SServiceNamespace + createInput.Tags[model.K8SParentRefTypeKey] = aws.String(string(modelTg.Spec.K8SParentRefType)) + + if modelTg.Spec.IsRoute() { + createInput.Tags[model.K8SRouteNameKey] = &modelTg.Spec.K8SRouteName + createInput.Tags[model.K8SRouteNamespaceKey] = &modelTg.Spec.K8SRouteNamespace } - resp, err := vpcLatticeSess.CreateTargetGroupWithContext(ctx, &createTargetGroupInput) + lattice := s.cloud.Lattice() + resp, err := lattice.CreateTargetGroupWithContext(ctx, &createInput) if err != nil { - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, err - } else { - tgArn := aws.StringValue(resp.Arn) - tgId := aws.StringValue(resp.Id) - tgStatus := aws.StringValue(resp.Status) - switch tgStatus { - case vpclattice.TargetGroupStatusCreateInProgress: - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, errors.New(LATTICE_RETRY) - case vpclattice.TargetGroupStatusActive: - return model.TargetGroupStatus{TargetGroupARN: tgArn, TargetGroupID: tgId}, nil - case vpclattice.TargetGroupStatusCreateFailed: - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, errors.New(LATTICE_RETRY) - case vpclattice.TargetGroupStatusDeleteFailed: - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, errors.New(LATTICE_RETRY) - case vpclattice.TargetGroupStatusDeleteInProgress: - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, errors.New(LATTICE_RETRY) - } + return model.TargetGroupStatus{}, + fmt.Errorf("Failed CreateTargetGroup %s due to %s", latticeTgName, err) } - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, nil -} + s.log.Infof("Success CreateTargetGroup %s", latticeTgName) -func (s *defaultTargetGroupManager) Get(ctx context.Context, targetGroup *model.TargetGroup) (model.TargetGroupStatus, error) { - s.log.Debugf("Getting VPC Lattice Target Group %s", targetGroup.Spec.Name) + latticeTgStatus := aws.StringValue(resp.Status) + if latticeTgStatus != vpclattice.TargetGroupStatusActive && + latticeTgStatus != vpclattice.TargetGroupStatusCreateInProgress { - // check if exists - tgSummary, err := s.findTargetGroup(ctx, targetGroup) - if err != nil { - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, err - } - if tgSummary != nil { - return model.TargetGroupStatus{TargetGroupARN: aws.StringValue(tgSummary.Arn), TargetGroupID: aws.StringValue(tgSummary.Id)}, err + s.log.Infof("Target group is not in the desired state. State is %s, will retry", latticeTgStatus) + return model.TargetGroupStatus{}, errors.New(LATTICE_RETRY) } - return model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, errors.New("Non existing Target Group") + // create-in-progress is considered success + // later, target reg may need to retry due to the state, and that's OK + return model.TargetGroupStatus{ + Name: aws.StringValue(resp.Name), + Arn: aws.StringValue(resp.Arn), + Id: aws.StringValue(resp.Id)}, nil } -func (s *defaultTargetGroupManager) update(ctx context.Context, targetGroup *model.TargetGroup, tgSummary *vpclattice.TargetGroupSummary) (model.TargetGroupStatus, error) { - s.log.Debugf("Updating VPC Lattice Target Group %s", targetGroup.Spec.Name) - - vpcLatticeSess := s.cloud.Lattice() - healthCheckConfig := targetGroup.Spec.Config.HealthCheckConfig - targetGroupStatus := model.TargetGroupStatus{ - TargetGroupARN: aws.StringValue(tgSummary.Arn), - TargetGroupID: aws.StringValue(tgSummary.Id), - } +func (s *defaultTargetGroupManager) update(ctx context.Context, targetGroup *model.TargetGroup, latticeTgSummary *vpclattice.TargetGroupSummary) (model.TargetGroupStatus, error) { + healthCheckConfig := targetGroup.Spec.HealthCheckConfig if healthCheckConfig == nil { s.log.Debugf("HealthCheck is empty. Resetting to default settings") - targetGroupProtocolVersion := targetGroup.Spec.Config.ProtocolVersion - healthCheckConfig = s.getDefaultHealthCheckConfig(targetGroupProtocolVersion) + protocolVersion := targetGroup.Spec.ProtocolVersion + healthCheckConfig = s.getDefaultHealthCheckConfig(protocolVersion) } - _, err := vpcLatticeSess.UpdateTargetGroupWithContext(ctx, &vpclattice.UpdateTargetGroupInput{ + _, err := s.cloud.Lattice().UpdateTargetGroupWithContext(ctx, &vpclattice.UpdateTargetGroupInput{ HealthCheck: healthCheckConfig, - TargetGroupIdentifier: tgSummary.Id, + TargetGroupIdentifier: latticeTgSummary.Id, }) if err != nil { - return model.TargetGroupStatus{}, err + return model.TargetGroupStatus{}, + fmt.Errorf("Failed UpdateTargetGroup %s due to %s", aws.StringValue(latticeTgSummary.Id), err) + } + s.log.Infof("Success UpdateTargetGroup %s", aws.StringValue(latticeTgSummary.Id)) + + modelTgStatus := model.TargetGroupStatus{ + Name: aws.StringValue(latticeTgSummary.Name), + Arn: aws.StringValue(latticeTgSummary.Arn), + Id: aws.StringValue(latticeTgSummary.Id), } - return targetGroupStatus, nil + return modelTgStatus, nil } -func (s *defaultTargetGroupManager) Delete(ctx context.Context, targetGroup *model.TargetGroup) error { - s.log.Debugf("Deleting VPC Lattice Target Group %s", targetGroup.Spec.Name) +func (s *defaultTargetGroupManager) Delete(ctx context.Context, modelTg *model.TargetGroup) error { + if modelTg.Status == nil || modelTg.Status.Id == "" { + latticeTgSummary, err := s.findTargetGroup(ctx, modelTg) + if err != nil { + return err + } - if targetGroup.Spec.LatticeID == "" { - s.log.Debugf("No ID found for target group, ignoring.") - return nil + if latticeTgSummary == nil { + // nothing to delete + s.log.Infof("Target group with name prefix %s does not exist, nothing to delete", model.TgNamePrefix(modelTg.Spec)) + return nil + } + + modelTg.Status = &model.TargetGroupStatus{ + Name: aws.StringValue(latticeTgSummary.Name), + Arn: aws.StringValue(latticeTgSummary.Arn), + Id: aws.StringValue(latticeTgSummary.Id), + } } + s.log.Debugf("Deleting target group %s", modelTg.Status.Id) + + lattice := s.cloud.Lattice() - vpcLatticeSess := s.cloud.Lattice() // de-register all targets first listTargetsInput := vpclattice.ListTargetsInput{ - TargetGroupIdentifier: &targetGroup.Spec.LatticeID, + TargetGroupIdentifier: &modelTg.Status.Id, } - listResp, err := vpcLatticeSess.ListTargetsAsList(ctx, &listTargetsInput) + listResp, err := lattice.ListTargetsAsList(ctx, &listTargetsInput) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == vpclattice.ErrCodeResourceNotFoundException { - // already deleted in lattice, this is OK - s.log.Debugf("Target group %s was already deleted", targetGroup.Spec.LatticeID) - err = nil - } + if services.IsLatticeAPINotFoundErr(err) { + s.log.Debugf("Target group %s was already deleted", modelTg.Status.Id) + return nil } + return fmt.Errorf("Failed ListTargets %s due to %s", modelTg.Status.Id, err) + } - if err != nil { - return err - } - } else { - // deregister targets - var targets []*vpclattice.Target - for _, t := range listResp { - if t.Status != nil && *t.Status != vpclattice.TargetStatusUnused { - s.log.Debugf("Target Group %s has non-unused status target(s), which means this targetGroup"+ - " is still in use by a VPC Lattice Service, so it cannot be deleted now", targetGroup.Spec.LatticeID) - // Before call the defaultTargetGroupManager.Delete(), we always call the latticeServiceManager.Delete() first, - // *t.Status != vpclattice.TargetStatusUnused means previous delete latticeService still in the progress, we could wait for 20 seconds and then retry - return errors.New(LATTICE_RETRY) - } - targets = append(targets, &vpclattice.Target{ - Id: t.Id, - Port: t.Port, - }) + var targetsToDeregister []*vpclattice.Target + drainCount := 0 + for _, t := range listResp { + targetsToDeregister = append(targetsToDeregister, &vpclattice.Target{ + Id: t.Id, + Port: t.Port, + }) + + if aws.StringValue(t.Status) == vpclattice.TargetStatusDraining { + drainCount++ } + } - targetsAreRegistered := len(targets) > 0 - if targetsAreRegistered { - deRegisterInput := vpclattice.DeregisterTargetsInput{ - TargetGroupIdentifier: &targetGroup.Spec.LatticeID, - Targets: targets, - } + if drainCount > 0 { + // no point in trying to deregister may as well wait + return fmt.Errorf("Cannot deregister targets for %s as %d targets are DRAINING", modelTg.Status.Id, drainCount) + } - deRegResp, err := vpcLatticeSess.DeregisterTargetsWithContext(ctx, &deRegisterInput) - if err != nil { - return err - } + if len(targetsToDeregister) > 0 { + deRegisterInput := vpclattice.DeregisterTargetsInput{ + TargetGroupIdentifier: &modelTg.Status.Id, + Targets: targetsToDeregister, + } - isDeRegRespUnsuccessful := len(deRegResp.Unsuccessful) > 0 - if isDeRegRespUnsuccessful { - s.log.Debugf("Target deregistration was unsuccessful, will retry later") - return errors.New(LATTICE_RETRY) - } + deRegResp, err := lattice.DeregisterTargetsWithContext(ctx, &deRegisterInput) + if err != nil { + return fmt.Errorf("Failed DeregisterTargets %s due to %s", modelTg.Status.Id, err) } + + isDeRegRespUnsuccessful := len(deRegResp.Unsuccessful) > 0 + if isDeRegRespUnsuccessful { + s.log.Infof("Unsuccessful (%d total) DeregisterTargets %s (0->%s), will retry", + len(deRegResp.Unsuccessful), modelTg.Status.Id, aws.StringValue(deRegResp.Unsuccessful[0].FailureMessage)) + return errors.New(LATTICE_RETRY) + } + s.log.Infof("Success DeregisterTargets %s", modelTg.Status.Id) } deleteTGInput := vpclattice.DeleteTargetGroupInput{ - TargetGroupIdentifier: &targetGroup.Spec.LatticeID, + TargetGroupIdentifier: &modelTg.Status.Id, } - _, err = vpcLatticeSess.DeleteTargetGroupWithContext(ctx, &deleteTGInput) + _, err = lattice.DeleteTargetGroupWithContext(ctx, &deleteTGInput) if err != nil { - if aerr, ok := err.(awserr.Error); ok { - if aerr.Code() == vpclattice.ErrCodeResourceNotFoundException { - s.log.Debugf("Target group %s was already deleted", targetGroup.Spec.LatticeID) - err = nil - } + if services.IsLatticeAPINotFoundErr(err) { + s.log.Infof("Target group %s was already deleted", modelTg.Status.Id) + return nil + } else { + return fmt.Errorf("Failed DeleteTargetGroup %s due to %s", modelTg.Status.Id, err) } } - return err + s.log.Infof("Success DeleteTargetGroup %s", modelTg.Status.Id) + return nil } -type targetGroupOutput struct { +type tgListOutput struct { getTargetGroupOutput vpclattice.GetTargetGroupOutput targetGroupTags *vpclattice.ListTagsForResourceOutput } -func (s *defaultTargetGroupManager) List(ctx context.Context) ([]targetGroupOutput, error) { - vpcLatticeSess := s.cloud.Lattice() - var tgList []targetGroupOutput +// Retrieve all TGs in the account, including tags. If individual tags fetch fails, tags will be nil for that tg +func (s *defaultTargetGroupManager) List(ctx context.Context) ([]tgListOutput, error) { + lattice := s.cloud.Lattice() + var tgList []tgListOutput targetGroupListInput := vpclattice.ListTargetGroupsInput{} - resp, err := vpcLatticeSess.ListTargetGroupsAsList(ctx, &targetGroupListInput) + resp, err := lattice.ListTargetGroupsAsList(ctx, &targetGroupListInput) if err != nil { return nil, err } @@ -292,25 +251,23 @@ func (s *defaultTargetGroupManager) List(ctx context.Context) ([]targetGroupOutp TargetGroupIdentifier: tg.Id, } - tgOutput, err := vpcLatticeSess.GetTargetGroupWithContext(ctx, &tgInput) + tgOutput, err := lattice.GetTargetGroupWithContext(ctx, &tgInput) if err != nil { continue } if tgOutput.Config != nil && aws.StringValue(tgOutput.Config.VpcIdentifier) == config.VpcID { - // retrieve target group tags - //ListTagsForResourceWithContext tagsInput := vpclattice.ListTagsForResourceInput{ ResourceArn: tg.Arn, } - tagsOutput, err := vpcLatticeSess.ListTagsForResourceWithContext(ctx, &tagsInput) + tagsOutput, err := lattice.ListTagsForResourceWithContext(ctx, &tagsInput) if err != nil { - s.log.Debugf("Error listing tags for target group %s: %s", *tg.Arn, err) - // setting it to nil, so the caller knows there is tag resource associated to this target group + s.log.Infof("Failed ListTags %s: %s", aws.StringValue(tg.Arn), err) + // setting it to nil, so the caller knows this failed tagsOutput = nil } - tgOutput := targetGroupOutput{ + tgOutput := tgListOutput{ getTargetGroupOutput: *tgOutput, targetGroupTags: tagsOutput, } @@ -320,69 +277,36 @@ func (s *defaultTargetGroupManager) List(ctx context.Context) ([]targetGroupOutp return tgList, err } -func isNameOfTargetGroup(targetGroup *model.TargetGroup, name string) bool { - if targetGroup.Spec.Config.IsServiceImport { - // We are missing protocol info for ServiceImport, but we do know the RouteType. - // Relying on the assumption that we have one TG per (RouteType, Service), - // do a simple guess to find the matching TG. - validProtocols := []string{ - vpclattice.TargetGroupProtocolHttp, - vpclattice.TargetGroupProtocolHttps, - } - validProtocolVersions := []string{ - vpclattice.TargetGroupProtocolVersionHttp1, - vpclattice.TargetGroupProtocolVersionHttp2, - } - if targetGroup.Spec.Config.ProtocolVersion == vpclattice.TargetGroupProtocolVersionGrpc { - validProtocolVersions = []string{vpclattice.TargetGroupProtocolVersionGrpc} - } - - for _, p := range validProtocols { - for _, pv := range validProtocolVersions { - candidate := &model.TargetGroup{ - Spec: model.TargetGroupSpec{ - Name: targetGroup.Spec.Name, - Config: model.TargetGroupConfig{ - Protocol: p, - ProtocolVersion: pv, - }, - }, - } - if name == getLatticeTGName(candidate) { - return true - } - } - } - return false - } else { - return name == getLatticeTGName(targetGroup) - } -} - func (s *defaultTargetGroupManager) findTargetGroup( ctx context.Context, - targetGroup *model.TargetGroup, + modelTargetGroup *model.TargetGroup, ) (*vpclattice.TargetGroupSummary, error) { - vpcLatticeSess := s.cloud.Lattice() - targetGroupListInput := vpclattice.ListTargetGroupsInput{} - resp, err := vpcLatticeSess.ListTargetGroupsAsList(ctx, &targetGroupListInput) + listInput := vpclattice.ListTargetGroupsInput{} + resp, err := s.cloud.Lattice().ListTargetGroupsAsList(ctx, &listInput) if err != nil { return nil, err } - for _, r := range resp { - if isNameOfTargetGroup(targetGroup, *r.Name) { - s.log.Debugf("Target group %s already exists with arn %s", *r.Name, *r.Arn) - status := aws.StringValue(r.Status) + for _, latticeTg := range resp { + // we ignore create failed status, so may as well check for it first + status := aws.StringValue(latticeTg.Status) + if status == vpclattice.TargetGroupStatusCreateFailed { + continue + } + + isMatch, err := s.IsTargetGroupMatch(ctx, modelTargetGroup, latticeTg, nil) + if err != nil { + return nil, err + } + if isMatch { + s.log.Debugf("Target group %s already exists with arn %s", *latticeTg.Name, *latticeTg.Arn) switch status { case vpclattice.TargetGroupStatusCreateInProgress: return nil, errors.New(LATTICE_RETRY) case vpclattice.TargetGroupStatusActive: - return r, nil - case vpclattice.TargetGroupStatusCreateFailed: - return nil, nil + return latticeTg, nil case vpclattice.TargetGroupStatusDeleteFailed: - return r, nil + return latticeTg, nil case vpclattice.TargetGroupStatusDeleteInProgress: return nil, errors.New(LATTICE_RETRY) } @@ -392,6 +316,55 @@ func (s *defaultTargetGroupManager) findTargetGroup( return nil, nil } +// latticeTags will be fetched if nil +func (s *defaultTargetGroupManager) IsTargetGroupMatch(ctx context.Context, + modelTg *model.TargetGroup, latticeTg *vpclattice.TargetGroupSummary, + latticeTagsAsModelTags *model.TargetGroupTagFields) (bool, error) { + + // first check fields we have before we try tags + if aws.Int64Value(latticeTg.Port) != int64(modelTg.Spec.Port) || + aws.StringValue(latticeTg.Protocol) != modelTg.Spec.Protocol || + aws.StringValue(latticeTg.IpAddressType) != modelTg.Spec.IpAddressType || + aws.StringValue(latticeTg.Type) != string(modelTg.Spec.Type) || + aws.StringValue(latticeTg.VpcIdentifier) != modelTg.Spec.VpcId { + + return false, nil + } + + // so far so good, now we check tags + if latticeTagsAsModelTags == nil { + // fetch the tags if we don't have them already + req := vpclattice.ListTagsForResourceInput{ResourceArn: latticeTg.Arn} + res, err := s.cloud.Lattice().ListTagsForResourceWithContext(ctx, &req) + if err != nil { + if apierrors.IsNotFound(err) { + // may have been deleted in the meantime, that's OK + return false, nil + } + + return false, err + } + + tags := model.TGTagFieldsFromTags(res.Tags) + latticeTagsAsModelTags = &tags + } + + tagsMatch := model.TagFieldsMatch(modelTg.Spec, *latticeTagsAsModelTags) + if !tagsMatch { + return false, nil + } + + // one last check - ProtocolVersion is not present on TargetGroupSummary, so we have to do a Get + gtgInput := vpclattice.GetTargetGroupInput{TargetGroupIdentifier: latticeTg.Id} + gtgOutput, err := s.cloud.Lattice().GetTargetGroupWithContext(ctx, >gInput) + if err != nil { + return false, err + } + + pvMatch := aws.StringValue(gtgOutput.Config.ProtocolVersion) == modelTg.Spec.ProtocolVersion + return pvMatch, nil +} + // Get default health check configuration according to // https://docs.aws.amazon.com/vpc-lattice/latest/ug/target-group-health-checks.html#health-check-settings func (s *defaultTargetGroupManager) getDefaultHealthCheckConfig(targetGroupProtocolVersion string) *vpclattice.HealthCheckConfig { diff --git a/pkg/deploy/lattice/target_group_manager_mock.go b/pkg/deploy/lattice/target_group_manager_mock.go index 7f5f6973..8814e294 100644 --- a/pkg/deploy/lattice/target_group_manager_mock.go +++ b/pkg/deploy/lattice/target_group_manager_mock.go @@ -9,6 +9,7 @@ import ( reflect "reflect" lattice "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + vpclattice "github.com/aws/aws-sdk-go/service/vpclattice" gomock "github.com/golang/mock/gomock" ) @@ -35,55 +36,40 @@ func (m *MockTargetGroupManager) EXPECT() *MockTargetGroupManagerMockRecorder { return m.recorder } -// Create mocks base method. -func (m *MockTargetGroupManager) Create(ctx context.Context, targetGroup *lattice.TargetGroup) (lattice.TargetGroupStatus, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", ctx, targetGroup) - ret0, _ := ret[0].(lattice.TargetGroupStatus) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Create indicates an expected call of Create. -func (mr *MockTargetGroupManagerMockRecorder) Create(ctx, targetGroup interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockTargetGroupManager)(nil).Create), ctx, targetGroup) -} - // Delete mocks base method. -func (m *MockTargetGroupManager) Delete(ctx context.Context, targetGroup *lattice.TargetGroup) error { +func (m *MockTargetGroupManager) Delete(ctx context.Context, modelTg *lattice.TargetGroup) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", ctx, targetGroup) + ret := m.ctrl.Call(m, "Delete", ctx, modelTg) ret0, _ := ret[0].(error) return ret0 } // Delete indicates an expected call of Delete. -func (mr *MockTargetGroupManagerMockRecorder) Delete(ctx, targetGroup interface{}) *gomock.Call { +func (mr *MockTargetGroupManagerMockRecorder) Delete(ctx, modelTg interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockTargetGroupManager)(nil).Delete), ctx, targetGroup) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockTargetGroupManager)(nil).Delete), ctx, modelTg) } -// Get mocks base method. -func (m *MockTargetGroupManager) Get(tx context.Context, targetGroup *lattice.TargetGroup) (lattice.TargetGroupStatus, error) { +// IsTargetGroupMatch mocks base method. +func (m *MockTargetGroupManager) IsTargetGroupMatch(ctx context.Context, modelTg *lattice.TargetGroup, latticeTg *vpclattice.TargetGroupSummary, latticeTags *lattice.TargetGroupTagFields) (bool, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Get", tx, targetGroup) - ret0, _ := ret[0].(lattice.TargetGroupStatus) + ret := m.ctrl.Call(m, "IsTargetGroupMatch", ctx, modelTg, latticeTg, latticeTags) + ret0, _ := ret[0].(bool) ret1, _ := ret[1].(error) return ret0, ret1 } -// Get indicates an expected call of Get. -func (mr *MockTargetGroupManagerMockRecorder) Get(tx, targetGroup interface{}) *gomock.Call { +// IsTargetGroupMatch indicates an expected call of IsTargetGroupMatch. +func (mr *MockTargetGroupManagerMockRecorder) IsTargetGroupMatch(ctx, modelTg, latticeTg, latticeTags interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockTargetGroupManager)(nil).Get), tx, targetGroup) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsTargetGroupMatch", reflect.TypeOf((*MockTargetGroupManager)(nil).IsTargetGroupMatch), ctx, modelTg, latticeTg, latticeTags) } // List mocks base method. -func (m *MockTargetGroupManager) List(ctx context.Context) ([]targetGroupOutput, error) { +func (m *MockTargetGroupManager) List(ctx context.Context) ([]tgListOutput, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "List", ctx) - ret0, _ := ret[0].([]targetGroupOutput) + ret0, _ := ret[0].([]tgListOutput) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -93,3 +79,18 @@ func (mr *MockTargetGroupManagerMockRecorder) List(ctx interface{}) *gomock.Call mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "List", reflect.TypeOf((*MockTargetGroupManager)(nil).List), ctx) } + +// Upsert mocks base method. +func (m *MockTargetGroupManager) Upsert(ctx context.Context, modelTg *lattice.TargetGroup) (lattice.TargetGroupStatus, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Upsert", ctx, modelTg) + ret0, _ := ret[0].(lattice.TargetGroupStatus) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Upsert indicates an expected call of Upsert. +func (mr *MockTargetGroupManagerMockRecorder) Upsert(ctx, modelTg interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Upsert", reflect.TypeOf((*MockTargetGroupManager)(nil).Upsert), ctx, modelTg) +} diff --git a/pkg/deploy/lattice/target_group_manager_test.go b/pkg/deploy/lattice/target_group_manager_test.go index 28d4768b..6344bd13 100644 --- a/pkg/deploy/lattice/target_group_manager_test.go +++ b/pkg/deploy/lattice/target_group_manager_test.go @@ -4,21 +4,18 @@ import ( "context" "errors" "fmt" - "reflect" - "testing" - pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" mocks "github.com/aws/aws-application-networking-k8s/pkg/aws/services" "github.com/aws/aws-application-networking-k8s/pkg/config" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/vpclattice" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "reflect" + "testing" ) // target group does not exist, and is active after creation @@ -26,102 +23,93 @@ func Test_CreateTargetGroup_TGNotExist_Active(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() + + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" mockLattice := mocks.NewMockLattice(c) cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - tg_types := [2]string{"by-backendref", "by-serviceexport"} + tgTypes := [2]string{"by-backendref", "by-serviceexport"} - for _, tg_type := range tg_types { + for _, tgType := range tgTypes { var tgSpec model.TargetGroupSpec - if tg_type == "by-serviceexport" { + if tgType == "by-serviceexport" { // testing targetgroup for serviceexport tgSpec = model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{ - Port: int32(8080), - Protocol: "HTTP", - ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, - VpcID: config.VpcID, - EKSClusterName: "", - IsServiceImport: false, - IsServiceExport: true, - K8SServiceName: "exportsvc1", - K8SServiceNamespace: "default", - }, + Port: int32(8080), + Protocol: "HTTP", + ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, } - } else if tg_type == "by-backendref" { + tgSpec.VpcId = config.VpcID + tgSpec.EKSClusterName = config.ClusterName + tgSpec.K8SParentRefType = model.ParentRefTypeSvcExport + tgSpec.K8SServiceName = "exportsvc1" + tgSpec.K8SServiceNamespace = "default" + tgSpec.Type = model.TargetGroupTypeIP + } else if tgType == "by-backendref" { // testing targetgroup for backendref tgSpec = model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{ - Port: int32(8080), - Protocol: "HTTP", - ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, - VpcID: config.VpcID, - EKSClusterName: "", - IsServiceImport: false, - IsServiceExport: false, - K8SServiceName: "backend-svc1", - K8SServiceNamespace: "default", - K8SHTTPRouteName: "httproute1", - K8SHTTPRouteNamespace: "default", - }, + Port: int32(8080), + Protocol: "HTTP", + ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, } + tgSpec.VpcId = config.VpcID + tgSpec.EKSClusterName = config.ClusterName + tgSpec.K8SParentRefType = model.ParentRefTypeHTTPRoute + tgSpec.K8SServiceName = "backend-svc1" + tgSpec.K8SServiceNamespace = "default" + tgSpec.K8SRouteName = "httproute1" + tgSpec.K8SRouteNamespace = "default" + tgSpec.Type = model.TargetGroupTypeIP } tgCreateInput := model.TargetGroup{ ResourceMeta: core.ResourceMeta{}, Spec: tgSpec, } - arn := "12345678912345678912" - id := "12345678912345678912" - name := "test-http-http1" - tgStatus := vpclattice.TargetGroupStatusActive - tgCreateOutput := &vpclattice.CreateTargetGroupOutput{ - Arn: &arn, - Id: &id, - Name: &name, - Status: &tgStatus, - } - p := int64(8080) - emptystring := "" - config := &vpclattice.TargetGroupConfig{ - Port: &p, - Protocol: &tgSpec.Config.Protocol, - VpcIdentifier: &config.VpcID, - ProtocolVersion: &tgSpec.Config.ProtocolVersion, - } - - createTargetGroupInput := vpclattice.CreateTargetGroupInput{ - Config: config, - Name: &name, - Type: &emptystring, - Tags: cloud.DefaultTags(), - } - createTargetGroupInput.Tags[model.K8SServiceNameKey] = &tgSpec.Config.K8SServiceName - createTargetGroupInput.Tags[model.K8SServiceNamespaceKey] = &tgSpec.Config.K8SServiceNamespace - - if tg_type == "by-serviceexport" { - value := model.K8SServiceExportType - createTargetGroupInput.Tags[model.K8SParentRefTypeKey] = &value - } else if tg_type == "by-backendref" { - value := model.K8SHTTPRouteType - createTargetGroupInput.Tags[model.K8SParentRefTypeKey] = &value - createTargetGroupInput.Tags[model.K8SHTTPRouteNameKey] = &tgSpec.Config.K8SHTTPRouteName - createTargetGroupInput.Tags[model.K8SHTTPRouteNamespaceKey] = &tgSpec.Config.K8SHTTPRouteNamespace + expectedTags := cloud.DefaultTags() + expectedTags[model.K8SServiceNameKey] = &tgSpec.K8SServiceName + expectedTags[model.K8SServiceNamespaceKey] = &tgSpec.K8SServiceNamespace + expectedTags[model.EKSClusterNameKey] = &tgSpec.EKSClusterName + + if tgType == "by-serviceexport" { + value := string(model.ParentRefTypeSvcExport) + expectedTags[model.K8SParentRefTypeKey] = &value + } else if tgType == "by-backendref" { + value := string(model.ParentRefTypeHTTPRoute) + expectedTags[model.K8SParentRefTypeKey] = &value + expectedTags[model.K8SRouteNameKey] = &tgSpec.K8SRouteName + expectedTags[model.K8SRouteNamespaceKey] = &tgSpec.K8SRouteNamespace } - listTgOutput := []*vpclattice.TargetGroupSummary{} + var listTgOutput []*vpclattice.TargetGroupSummary mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) - mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, &createTargetGroupInput).Return(tgCreateOutput, nil) + mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).DoAndReturn( + func(ctx context.Context, input *vpclattice.CreateTargetGroupInput, arg3 ...interface{}) (*vpclattice.CreateTargetGroupOutput, error) { + assert.Equal(t, aws.Int64(int64(tgSpec.Port)), input.Config.Port) + assert.Equal(t, tgSpec.Protocol, *input.Config.Protocol) + assert.Equal(t, tgSpec.ProtocolVersion, *input.Config.ProtocolVersion) + assert.Equal(t, expectedTags, input.Tags) + assert.Equal(t, tgSpec.VpcId, *input.Config.VpcIdentifier) + + return &vpclattice.CreateTargetGroupOutput{ + Arn: aws.String("tg-arn-1"), + Id: aws.String("tg-id-1"), + Name: aws.String("tg-name-1"), + Status: aws.String(vpclattice.TargetGroupStatusActive), + }, nil + }, + ) + tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) + resp, err := tgManager.Upsert(ctx, &tgCreateInput) assert.Nil(t, err) - assert.Equal(t, resp.TargetGroupARN, arn) - assert.Equal(t, resp.TargetGroupID, id) + assert.Equal(t, "tg-arn-1", resp.Arn) + assert.Equal(t, "tg-id-1", resp.Id) + assert.Equal(t, "tg-name-1", resp.Name) } } @@ -130,17 +118,16 @@ func Test_CreateTargetGroup_TGFailed_Active(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - } + tgSpec := model.TargetGroupSpec{} + tgSpec.K8SRouteName = "route1" + tgSpec.K8SRouteNamespace = "ns1" tgCreateInput := model.TargetGroup{ ResourceMeta: core.ResourceMeta{}, Spec: tgSpec, - Status: nil, } arn := "12345678912345678912" @@ -165,32 +152,35 @@ func Test_CreateTargetGroup_TGFailed_Active(t *testing.T) { mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).Return(tgCreateOutput, nil) - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) + resp, err := tgManager.Upsert(ctx, &tgCreateInput) assert.Nil(t, err) - assert.Equal(t, resp.TargetGroupARN, arn) - assert.Equal(t, resp.TargetGroupID, id) + assert.Equal(t, arn, resp.Arn) + assert.Equal(t, id, resp.Id) } // target group status is active before creation, no need to recreate func Test_CreateTargetGroup_TGActive_UpdateHealthCheck(t *testing.T) { tests := []struct { + name string healthCheckConfig *vpclattice.HealthCheckConfig wantErr bool }{ { + name: "includes health check", healthCheckConfig: &vpclattice.HealthCheckConfig{ Enabled: aws.Bool(false), }, wantErr: false, }, { + name: "health check nil", healthCheckConfig: nil, wantErr: false, }, { + name: "health check missing", wantErr: true, }, } @@ -200,22 +190,19 @@ func Test_CreateTargetGroup_TGActive_UpdateHealthCheck(t *testing.T) { arn := "12345678912345678912" id := "12345678912345678912" - for i, test := range tests { - t.Run(fmt.Sprintf("Test_%d", i), func(t *testing.T) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { c := gomock.NewController(t) defer c.Finish() mockLattice := mocks.NewMockLattice(c) cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{ - Protocol: vpclattice.TargetGroupProtocolHttps, - ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, - HealthCheckConfig: test.healthCheckConfig, - }, + Port: 80, + Protocol: vpclattice.TargetGroupProtocolHttps, + ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, + HealthCheckConfig: tt.healthCheckConfig, } tgCreateInput := model.TargetGroup{ @@ -224,38 +211,52 @@ func Test_CreateTargetGroup_TGActive_UpdateHealthCheck(t *testing.T) { } tgSummary := vpclattice.TargetGroupSummary{ - Arn: &arn, - Id: &id, - Name: aws.String("test-https-http1"), - Status: aws.String(vpclattice.TargetGroupStatusActive), - Port: aws.Int64(80), + Arn: &arn, + Id: &id, + Name: aws.String("test-https-http1"), + Status: aws.String(vpclattice.TargetGroupStatusActive), + Port: aws.Int64(80), + Protocol: aws.String(vpclattice.TargetGroupProtocolHttps), } listTgOutput := []*vpclattice.TargetGroupSummary{&tgSummary} mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) - if test.wantErr { + // empty tags should be OK and should match since all tag values on the spec are empty + mockLattice.EXPECT().ListTagsForResourceWithContext(ctx, gomock.Any()).Return( + &vpclattice.ListTagsForResourceOutput{}, nil) + + // we use Get to do a last check on the protocol version + mockLattice.EXPECT().GetTargetGroupWithContext(ctx, gomock.Any()).Return( + &vpclattice.GetTargetGroupOutput{ + Config: &vpclattice.TargetGroupConfig{ + ProtocolVersion: aws.String(tgSpec.ProtocolVersion), + }, + }, nil) + + if tt.wantErr { mockLattice.EXPECT().UpdateTargetGroupWithContext(ctx, gomock.Any()).Return(nil, errors.New("error")) } else { mockLattice.EXPECT().UpdateTargetGroupWithContext(ctx, gomock.Any()).Return(nil, nil) } - resp, err := tgManager.Create(ctx, &tgCreateInput) + tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) + resp, err := tgManager.Upsert(ctx, &tgCreateInput) - if test.wantErr { + if tt.wantErr { assert.NotNil(t, err) } else { assert.Nil(t, err) - assert.Equal(t, resp.TargetGroupARN, arn) - assert.Equal(t, resp.TargetGroupID, id) + assert.Equal(t, arn, resp.Arn) + assert.Equal(t, id, resp.Id) } }) } } // target group status is create-in-progress before creation, return Retry -func Test_CreateTargetGroup_TGCreateInProgress_Retry(t *testing.T) { +func Test_CreateTargetGroup_ExistingTG_Status_Retry(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() @@ -263,79 +264,54 @@ func Test_CreateTargetGroup_TGCreateInProgress_Retry(t *testing.T) { cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, + Port: 80, + Protocol: "HTTP", } tgCreateInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, + Spec: tgSpec, } - arn := "12345678912345678912" id := "12345678912345678912" name := "test" - beforeCreateStatus := vpclattice.TargetGroupStatusCreateInProgress - tgSummary := vpclattice.TargetGroupSummary{ - Arn: &arn, - Id: &id, - Name: &name, - Status: &beforeCreateStatus, + retryStatuses := []string{ + vpclattice.TargetGroupStatusCreateInProgress, + vpclattice.TargetGroupStatusDeleteInProgress, } - listTgOutput := []*vpclattice.TargetGroupSummary{&tgSummary} - - mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, errors.New(LATTICE_RETRY)) - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) - - assert.NotNil(t, err) - assert.Equal(t, err, errors.New(LATTICE_RETRY)) - assert.Equal(t, resp.TargetGroupARN, "") - assert.Equal(t, resp.TargetGroupID, "") -} -// target group status is delete-in-progress before creation, return Retry -func Test_CreateTargetGroup_TGDeleteInProgress_Retry(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) + for _, retryStatus := range retryStatuses { + t.Run(fmt.Sprintf("retry on status %s", retryStatus), func(t *testing.T) { + beforeCreateStatus := retryStatus + tgSummary := vpclattice.TargetGroupSummary{ + Arn: &arn, + Id: &id, + Name: &name, + Status: &beforeCreateStatus, + Port: aws.Int64(80), + Protocol: aws.String("HTTP"), + } + listTgOutput := []*vpclattice.TargetGroupSummary{&tgSummary} - tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - } - tgCreateInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, - } + mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) + mockLattice.EXPECT().ListTagsForResourceWithContext(ctx, gomock.Any()).Return( + &vpclattice.ListTagsForResourceOutput{}, nil) + mockLattice.EXPECT().GetTargetGroupWithContext(ctx, gomock.Any()).Return( + &vpclattice.GetTargetGroupOutput{ + Config: &vpclattice.TargetGroupConfig{ + ProtocolVersion: aws.String(tgSpec.ProtocolVersion), + }, + }, nil) - arn := "12345678912345678912" - id := "12345678912345678912" - name := "test" + tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) + _, err := tgManager.Upsert(ctx, &tgCreateInput) - beforeCreateStatus := vpclattice.TargetGroupStatusDeleteInProgress - tgSummary := vpclattice.TargetGroupSummary{ - Arn: &arn, - Id: &id, - Name: &name, - Status: &beforeCreateStatus, + assert.Equal(t, errors.New(LATTICE_RETRY), err) + }) } - listTgOutput := []*vpclattice.TargetGroupSummary{&tgSummary} - - mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, errors.New(LATTICE_RETRY)) - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) - - assert.NotNil(t, err) - assert.Equal(t, err, errors.New(LATTICE_RETRY)) - assert.Equal(t, resp.TargetGroupARN, "") - assert.Equal(t, resp.TargetGroupID, "") } // target group is not in-progress before, get create-in-progress, should return retry -func Test_CreateTargetGroup_TGNotExist_CreateInProgress(t *testing.T) { +func Test_CreateTargetGroup_NewTG_RetryStatus(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() @@ -343,125 +319,48 @@ func Test_CreateTargetGroup_TGNotExist_CreateInProgress(t *testing.T) { cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, + Port: 80, + Protocol: "HTTP", } tgCreateInput := model.TargetGroup{ ResourceMeta: core.ResourceMeta{}, Spec: tgSpec, } - arn := "12345678912345678912" id := "12345678912345678912" name := "test" - tgStatus := vpclattice.TargetGroupStatusCreateInProgress - tgCreateOutput := &vpclattice.CreateTargetGroupOutput{ - Arn: &arn, - Id: &id, - Name: &name, - Status: &tgStatus, - } - - listTgOutput := []*vpclattice.TargetGroupSummary{} - - mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) - mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).Return(tgCreateOutput, nil) - - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) - - assert.NotNil(t, err) - assert.NotNil(t, err, errors.New(LATTICE_RETRY)) - assert.Equal(t, resp.TargetGroupARN, "") - assert.Equal(t, resp.TargetGroupID, "") -} - -// target group is not in-progress before, get delete-in-progress, should return retry -func Test_CreateTargetGroup_TGNotExist_DeleteInProgress(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - } - tgCreateInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, - } + var listTgOutput []*vpclattice.TargetGroupSummary - arn := "12345678912345678912" - id := "12345678912345678912" - name := "test" - tgStatus := vpclattice.TargetGroupStatusDeleteInProgress - tgCreateOutput := &vpclattice.CreateTargetGroupOutput{ - Arn: &arn, - Id: &id, - Name: &name, - Status: &tgStatus, + retryStatuses := []string{ + vpclattice.TargetGroupStatusDeleteInProgress, + vpclattice.TargetGroupStatusCreateFailed, + vpclattice.TargetGroupStatusDeleteFailed, + vpclattice.TargetGroupStatusDeleteInProgress, } + for _, retryStatus := range retryStatuses { + t.Run(fmt.Sprintf("retry on status %s", retryStatus), func(t *testing.T) { - listTgOutput := []*vpclattice.TargetGroupSummary{} - - mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) - mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).Return(tgCreateOutput, nil) - - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) - - assert.NotNil(t, err) - assert.NotNil(t, err, errors.New(LATTICE_RETRY)) - assert.Equal(t, resp.TargetGroupARN, "") - assert.Equal(t, resp.TargetGroupID, "") -} + tgStatus := retryStatus + tgCreateOutput := &vpclattice.CreateTargetGroupOutput{ + Arn: &arn, + Id: &id, + Name: &name, + Status: &tgStatus, + } -// target group is not in-progress before, get failed, should return retry -func Test_CreateTargetGroup_TGNotExist_Failed(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) + mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) + mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).Return(tgCreateOutput, nil) - tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - } - tgCreateInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, - } + tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) + _, err := tgManager.Upsert(ctx, &tgCreateInput) - arn := "12345678912345678912" - id := "12345678912345678912" - name := "test" - tgStatus := vpclattice.TargetGroupStatusCreateFailed - tgCreateOutput := &vpclattice.CreateTargetGroupOutput{ - Arn: &arn, - Id: &id, - Name: &name, - Status: &tgStatus, + assert.Equal(t, errors.New(LATTICE_RETRY), err) + }) } - - listTgOutput := []*vpclattice.TargetGroupSummary{} - - mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) - mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).Return(tgCreateOutput, nil) - - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) - - assert.NotNil(t, err) - assert.NotNil(t, err, errors.New(LATTICE_RETRY)) - assert.Equal(t, resp.TargetGroupARN, "") - assert.Equal(t, resp.TargetGroupID, "") } -// Failed to list target group, should return error -func Test_CreateTargetGroup_ListTGError(t *testing.T) { +func Test_Lattice_API_Errors(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() @@ -469,78 +368,33 @@ func Test_CreateTargetGroup_ListTGError(t *testing.T) { cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, + Port: 80, + Protocol: "HTTP", } tgCreateInput := model.TargetGroup{ ResourceMeta: core.ResourceMeta{}, Spec: tgSpec, } + var listTgOutput []*vpclattice.TargetGroupSummary - listTgOutput := []*vpclattice.TargetGroupSummary{} - + // list error mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, errors.New("test")) tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) - - assert.NotNil(t, err) - assert.NotNil(t, err, errors.New("test")) - assert.Equal(t, resp.TargetGroupARN, "") - assert.Equal(t, resp.TargetGroupID, "") -} - -// Failed to create target group, should return error -func Test_CreateTargetGroup_CreateTGFailed(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) + _, err := tgManager.Upsert(ctx, &tgCreateInput) - tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - } - tgCreateInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, - } - - arn := "12345678912345678912" - id := "12345678912345678912" - name := "test" - tgStatus := vpclattice.TargetGroupStatusCreateFailed - tgCreateOutput := &vpclattice.CreateTargetGroupOutput{ - Arn: &arn, - Id: &id, - Name: &name, - Status: &tgStatus, - } - - listTgOutput := []*vpclattice.TargetGroupSummary{} + assert.Equal(t, errors.New("test"), err) + // create error mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) - mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).Return(tgCreateOutput, errors.New("test")) - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - resp, err := tgManager.Create(ctx, &tgCreateInput) + mockLattice.EXPECT().CreateTargetGroupWithContext(ctx, gomock.Any()).Return(nil, errors.New("test")) + tgManager = NewTargetGroupManager(gwlog.FallbackLogger, cloud) + _, err = tgManager.Upsert(ctx, &tgCreateInput) assert.NotNil(t, err) - assert.NotNil(t, err, errors.New("test")) - assert.Equal(t, resp.TargetGroupARN, "") - assert.Equal(t, resp.TargetGroupID, "") } -// Case1: Deregister unused status targets and delete target group work perfectly fine -// Case2: Delete target group that no targets register on it -// Case3: While deleting target group, deregister targets fails -// Case4: While deleting target group, list targets fails -// Case5: While deleting target group, deregister targets unsuccessfully -// Case6: Delete target group fails -// Case7: While deleting target group, that it has non-unused status targets, return LATTICE_RETRY -// Case8: While deleting target group, the vpcLatticeSess.DeleteTargetGroupWithContext() return ResourceNotFoundException, delete target group success and return err nil - -// Case1: Deregister unused status targets and delete target group work perfectly fine +// Deregister unused status targets and delete target group work perfectly fine func Test_DeleteTG_DeRegisterTargets_DeleteTargetGroup(t *testing.T) { sId := "123.456.7.890" sPort := int64(80) @@ -563,25 +417,24 @@ func Test_DeleteTG_DeRegisterTargets_DeleteTargetGroup(t *testing.T) { deleteTargetGroupOutput := &vpclattice.DeleteTargetGroupOutput{} tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - Type: "IP", - IsDeleted: false, - LatticeID: "123", + Type: "IP", } tgDeleteInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, + Spec: tgSpec, + Status: &model.TargetGroupStatus{ + Name: "name", + Arn: "arn", + Id: "id", + }, } c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, nil) mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(deRegisterTargetsOutput, nil) mockLattice.EXPECT().DeleteTargetGroupWithContext(ctx, gomock.Any()).Return(deleteTargetGroupOutput, nil) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) @@ -590,14 +443,8 @@ func Test_DeleteTG_DeRegisterTargets_DeleteTargetGroup(t *testing.T) { assert.Nil(t, err) } -// Case2: Delete target group that no targets register on it +// Delete target group that no targets register on it func Test_DeleteTG_NoRegisteredTargets_DeleteTargetGroup(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - sId := "123.456.7.890" sPort := int64(80) targets := &vpclattice.TargetSummary{ @@ -609,19 +456,24 @@ func Test_DeleteTG_NoRegisteredTargets_DeleteTargetGroup(t *testing.T) { deleteTargetGroupOutput := &vpclattice.DeleteTargetGroupOutput{} tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - Type: "IP", - IsDeleted: false, - LatticeID: "123", + Type: "IP", } tgDeleteInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, + Spec: tgSpec, + Status: &model.TargetGroupStatus{ + Name: "name", + Arn: "arn", + Id: "id", + }, } + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, nil) mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(deRegisterTargetsOutput, nil) mockLattice.EXPECT().DeleteTargetGroupWithContext(ctx, gomock.Any()).Return(deleteTargetGroupOutput, nil) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) @@ -630,53 +482,94 @@ func Test_DeleteTG_NoRegisteredTargets_DeleteTargetGroup(t *testing.T) { assert.Nil(t, err) } -// Case3: While deleting target group, deregister targets fails -func Test_DeleteTG_DeRegisteredTargetsFailed(t *testing.T) { +func Test_DeleteTG_WithExistingTG(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() mockLattice := mocks.NewMockLattice(c) cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - sId := "123.456.7.890" - sPort := int64(80) - targets := &vpclattice.TargetSummary{ - Id: &sId, - Port: &sPort, + tgSummary := vpclattice.TargetGroupSummary{ + Arn: aws.String("existing-tg-arn"), + Id: aws.String("existing-tg-id"), + Name: aws.String("name"), + Status: aws.String(vpclattice.TargetGroupStatusActive), + Port: aws.Int64(80), + Protocol: aws.String(vpclattice.TargetGroupProtocolHttps), } - listTargetsOutput := []*vpclattice.TargetSummary{targets} - deRegisterTargetsOutput := &vpclattice.DeregisterTargetsOutput{} tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - Type: "IP", - IsDeleted: false, - LatticeID: "123", + Port: 80, + Protocol: "HTTPS", + ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, } tgDeleteInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, + Spec: tgSpec, + Status: nil, } + var listTargetsOutput []*vpclattice.TargetSummary + listTgOutput := []*vpclattice.TargetGroupSummary{&tgSummary} + + mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) + mockLattice.EXPECT().ListTagsForResourceWithContext(ctx, gomock.Any()).Return( + &vpclattice.ListTagsForResourceOutput{}, nil) + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, nil) - mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(deRegisterTargetsOutput, errors.New("Deregister_failed")) - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) + mockLattice.EXPECT().GetTargetGroupWithContext(ctx, gomock.Any()).Return( + &vpclattice.GetTargetGroupOutput{ + Config: &vpclattice.TargetGroupConfig{ + ProtocolVersion: aws.String(tgSpec.ProtocolVersion), + }, + }, nil) + + dtgInput := &vpclattice.DeleteTargetGroupInput{TargetGroupIdentifier: tgSummary.Id} + dtgOutput := &vpclattice.DeleteTargetGroupOutput{} + mockLattice.EXPECT().DeleteTargetGroupWithContext(ctx, dtgInput).Return(dtgOutput, nil) + tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) err := tgManager.Delete(ctx, &tgDeleteInput) - assert.NotNil(t, err) - assert.Equal(t, err, errors.New("Deregister_failed")) + assert.Nil(t, err) } -// Case4: While deleting target group, list targets fails -func Test_DeleteTG_ListTargetsFailed(t *testing.T) { +func Test_DeleteTG_NothingToDelete(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() mockLattice := mocks.NewMockLattice(c) cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) + tgSummary := vpclattice.TargetGroupSummary{ + Arn: aws.String("existing-tg-arn"), + Id: aws.String("existing-tg-id"), + Name: aws.String("name"), + Status: aws.String(vpclattice.TargetGroupStatusActive), + Port: aws.Int64(443), // <-- important difference, so not a match + Protocol: aws.String(vpclattice.TargetGroupProtocolHttps), + } + + tgSpec := model.TargetGroupSpec{ + Port: 80, + Protocol: "HTTPS", + ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, + } + tgDeleteInput := model.TargetGroup{ + Spec: tgSpec, + Status: nil, + } + listTgOutput := []*vpclattice.TargetGroupSummary{&tgSummary} + + mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTgOutput, nil) + + tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) + err := tgManager.Delete(ctx, &tgDeleteInput) + + assert.Nil(t, err) +} + +// While deleting target group, deregister targets fails +func Test_DeleteTG_DeRegisteredTargetsFailed(t *testing.T) { sId := "123.456.7.890" sPort := int64(80) targets := &vpclattice.TargetSummary{ @@ -684,37 +577,69 @@ func Test_DeleteTG_ListTargetsFailed(t *testing.T) { Port: &sPort, } listTargetsOutput := []*vpclattice.TargetSummary{targets} + deRegisterTargetsOutput := &vpclattice.DeregisterTargetsOutput{} tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - Type: "IP", - IsDeleted: false, - LatticeID: "123", + Type: "IP", } tgDeleteInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, - Status: nil, + Spec: tgSpec, + Status: &model.TargetGroupStatus{ + Name: "name", + Arn: "arn", + Id: "id", + }, } - mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, errors.New("Listregister_failed")) + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, nil) + mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(deRegisterTargetsOutput, errors.New("Deregister_failed")) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) err := tgManager.Delete(ctx, &tgDeleteInput) - assert.NotNil(t, err) - assert.Equal(t, err, errors.New("Listregister_failed")) } -// Case5: While deleting target group, deregister targets unsuccessfully -func Test_DeleteTG_DeRegisterTargetsUnsuccessfully(t *testing.T) { +// While deleting target group, list targets fails +func Test_DeleteTG_ListTargetsFailed(t *testing.T) { + sId := "123.456.7.890" + sPort := int64(80) + targets := &vpclattice.TargetSummary{ + Id: &sId, + Port: &sPort, + } + listTargetsOutput := []*vpclattice.TargetSummary{targets} + + tgSpec := model.TargetGroupSpec{ + Type: "IP", + } + tgDeleteInput := model.TargetGroup{ + Spec: tgSpec, + Status: &model.TargetGroupStatus{ + Name: "name", + Arn: "arn", + Id: "id", + }, + } c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() mockLattice := mocks.NewMockLattice(c) + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, errors.New("Listregister_failed")) cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) + tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) + + err := tgManager.Delete(ctx, &tgDeleteInput) + assert.NotNil(t, err) +} + +// While deleting target group, deregister targets unsuccessfully +func Test_DeleteTG_DeRegisterTargetsUnsuccessfully(t *testing.T) { sId := "123.456.7.890" sPort := int64(80) targets := &vpclattice.TargetSummary{ @@ -734,18 +659,23 @@ func Test_DeleteTG_DeRegisterTargetsUnsuccessfully(t *testing.T) { } tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - Type: "IP", - IsDeleted: false, - LatticeID: "123", + Type: "IP", } tgDeleteInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, + Spec: tgSpec, + Status: &model.TargetGroupStatus{ + Name: "name", + Arn: "arn", + Id: "id", + }, } + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, nil) mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(deRegisterTargetsOutput, nil) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) @@ -755,14 +685,8 @@ func Test_DeleteTG_DeRegisterTargetsUnsuccessfully(t *testing.T) { assert.Equal(t, err, errors.New(LATTICE_RETRY)) } -// Case6: Delete target group fails +// Delete target group fails func Test_DeleteTG_DeRegisterTargets_DeleteTargetGroupFailed(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - sId := "123.456.7.890" sPort := int64(80) targetsList := &vpclattice.TargetSummary{ @@ -782,115 +706,33 @@ func Test_DeleteTG_DeRegisterTargets_DeleteTargetGroupFailed(t *testing.T) { deleteTargetGroupOutput := &vpclattice.DeleteTargetGroupOutput{} tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - Type: "IP", - IsDeleted: false, - LatticeID: "123", - } - tgDeleteInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, - } - mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, nil) - mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(deRegisterTargetsOutput, nil) - mockLattice.EXPECT().DeleteTargetGroupWithContext(ctx, gomock.Any()).Return(deleteTargetGroupOutput, errors.New("DeleteTG_failed")) - - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - - err := tgManager.Delete(ctx, &tgDeleteInput) - - assert.NotNil(t, err) - assert.Equal(t, err, errors.New("DeleteTG_failed")) -} - -// Case7: While deleting target group, it has non-unused status targets, return LATTICE_RETRY -func Test_DeleteTG_TargetsNonUnused(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - targetId := "123.456.7.890" - targetPort := int64(80) - targetStatus := vpclattice.TargetStatusHealthy - targets := &vpclattice.TargetSummary{ - Id: &targetId, - Port: &targetPort, - Status: &targetStatus, - } - listTargetsOutput := []*vpclattice.TargetSummary{targets} - - tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - Type: "IP", - IsDeleted: false, - LatticeID: "123", + Type: "IP", } tgDeleteInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, - Status: nil, + Spec: tgSpec, + Status: &model.TargetGroupStatus{ + Name: "name", + Arn: "arn", + Id: "id", + }, } - mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, nil) - - tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - - err := tgManager.Delete(ctx, &tgDeleteInput) - - assert.NotNil(t, err) - assert.Equal(t, err, errors.New(LATTICE_RETRY)) -} - -// Case8: While deleting target group, the vpcLatticeSess.DeleteTargetGroupWithContext() return ResourceNotFoundException, delete target group success and return err nil -func Test_DeleteTG_vpcLatticeSessReturnResourceNotFound_DeleteTargetGroupSuccessAndErrIsNil(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - sId := "123.456.7.890" - sPort := int64(80) - targets := &vpclattice.TargetSummary{ - Id: &sId, - Port: &sPort, - } - listTargetsOutput := []*vpclattice.TargetSummary{targets} - deRegisterTargetsOutput := &vpclattice.DeregisterTargetsOutput{} - deleteTargetGroupOutput := &vpclattice.DeleteTargetGroupOutput{} - - tgSpec := model.TargetGroupSpec{ - Name: "test", - Config: model.TargetGroupConfig{}, - Type: "IP", - IsDeleted: false, - LatticeID: "123", - } - tgDeleteInput := model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: tgSpec, - } mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetsOutput, nil) mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(deRegisterTargetsOutput, nil) - mockLattice.EXPECT().DeleteTargetGroupWithContext(ctx, gomock.Any()).Return(deleteTargetGroupOutput, awserr.New(vpclattice.ErrCodeResourceNotFoundException, "", nil)) + mockLattice.EXPECT().DeleteTargetGroupWithContext(ctx, gomock.Any()).Return(deleteTargetGroupOutput, errors.New("DeleteTG_failed")) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) err := tgManager.Delete(ctx, &tgDeleteInput) - assert.Nil(t, err) + assert.NotNil(t, err) } func Test_ListTG_TGsExist(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - arn := "123456789" id := "123456789" name1 := "test1" @@ -922,15 +764,20 @@ func Test_ListTG_TGsExist(t *testing.T) { Config: config2, } + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockLattice := mocks.NewMockLattice(c) mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTGOutput, nil) mockLattice.EXPECT().GetTargetGroupWithContext(ctx, gomock.Any()).Return(getTG1, nil) // assume no tags mockLattice.EXPECT().ListTagsForResourceWithContext(ctx, gomock.Any()).Return(nil, errors.New("no tags")) mockLattice.EXPECT().GetTargetGroupWithContext(ctx, gomock.Any()).Return(getTG2, nil) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) tgList, err := tgManager.List(ctx) - expect := []targetGroupOutput{ + expect := []tgListOutput{ { getTargetGroupOutput: *getTG1, targetGroupTags: nil, @@ -942,187 +789,23 @@ func Test_ListTG_TGsExist(t *testing.T) { } func Test_ListTG_NoTG(t *testing.T) { + listTGOutput := []*vpclattice.TargetGroupSummary{} + c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - listTGOutput := []*vpclattice.TargetGroupSummary{} - mockLattice.EXPECT().ListTargetGroupsAsList(ctx, gomock.Any()).Return(listTGOutput, nil) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) tgManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) tgList, err := tgManager.List(ctx) - expectTgList := []targetGroupOutput(nil) + expectTgList := []tgListOutput(nil) assert.Nil(t, err) assert.Equal(t, tgList, expectTgList) } -func Test_Get(t *testing.T) { - tests := []struct { - wantErr error - tgId string - tgArn string - tgName string - input *model.TargetGroup - wantOutput model.TargetGroupStatus - randomArn string - randomId string - randomName string - tgStatus string - tgStatusFailed string - }{ - { - wantErr: nil, - tgId: "tg-id-012345", - tgArn: "tg-arn-123456", - tgName: "tg-test-1-https-http1", - input: &model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: model.TargetGroupSpec{ - Name: "tg-test-1", - Config: model.TargetGroupConfig{ - Protocol: vpclattice.TargetGroupProtocolHttps, - ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, - }, - Type: "", - IsDeleted: false, - LatticeID: "", - }, - Status: nil, - }, - wantOutput: model.TargetGroupStatus{TargetGroupARN: "tg-arn-123456", TargetGroupID: "tg-id-012345"}, - randomArn: "random-tg-arn-12345", - randomId: "random-tg-id-12345", - randomName: "tgrandom-1", - tgStatus: vpclattice.TargetGroupStatusActive, - tgStatusFailed: vpclattice.TargetGroupStatusCreateFailed, - }, - { - wantErr: errors.New("Non existing Target Group"), - tgId: "tg-id-012345", - tgArn: "tg-arn-123456", - tgName: "tg-test-1-https-http1", - input: &model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: model.TargetGroupSpec{ - Name: "tg-test-1", - Config: model.TargetGroupConfig{ - Protocol: vpclattice.TargetGroupProtocolHttps, - ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, - }, - Type: "", - IsDeleted: false, - LatticeID: "", - }, - Status: nil, - }, - wantOutput: model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, - randomArn: "random-tg-arn-12345", - randomId: "random-tg-id-12345", - randomName: "tgrandom-1", - tgStatus: vpclattice.TargetGroupStatusCreateFailed, - tgStatusFailed: vpclattice.TargetGroupStatusCreateFailed, - }, - { - wantErr: errors.New(LATTICE_RETRY), - tgId: "tg-id-012345", - tgArn: "tg-arn-123456", - tgName: "tg-test-1-https-http1", - input: &model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: model.TargetGroupSpec{ - Name: "tg-test-1", - Config: model.TargetGroupConfig{ - Protocol: vpclattice.TargetGroupProtocolHttps, - ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, - }, - Type: "", - IsDeleted: false, - LatticeID: "", - }, - Status: nil, - }, - wantOutput: model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, - randomArn: "random-tg-arn-12345", - randomId: "random-tg-id-12345", - randomName: "tgrandom-1", - tgStatus: vpclattice.TargetGroupStatusDeleteInProgress, - tgStatusFailed: vpclattice.TargetGroupStatusDeleteFailed, - }, - { - wantErr: errors.New("Non existing Target Group"), - tgId: "tg-id-012345", - tgArn: "tg-arn-123456", - tgName: "tg-test-not-exist-https-http1", - input: &model.TargetGroup{ - ResourceMeta: core.ResourceMeta{}, - Spec: model.TargetGroupSpec{ - Name: "tg-test-2", - Config: model.TargetGroupConfig{ - Protocol: vpclattice.TargetGroupProtocolHttps, - ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, - }, - Type: "", - IsDeleted: false, - LatticeID: "", - }, - Status: nil, - }, - wantOutput: model.TargetGroupStatus{TargetGroupARN: "", TargetGroupID: ""}, - randomArn: "random-tg-arn-12345", - randomId: "random-tg-id-12345", - randomName: "tgrandom-2", - tgStatus: vpclattice.TargetGroupStatusCreateFailed, - tgStatusFailed: vpclattice.TargetGroupStatusCreateFailed, - }, - } - - for i, tt := range tests { - t.Run(fmt.Sprintf("Test_%d", i), func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockLattice := mocks.NewMockLattice(c) - cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) - - listTGinput := &vpclattice.ListTargetGroupsInput{} - listTGOutput := []*vpclattice.TargetGroupSummary{ - { - Arn: &tt.randomArn, - Id: &tt.randomId, - Name: &tt.randomName, - Status: &tt.tgStatusFailed, - Type: nil, - }, - { - Arn: &tt.tgArn, - Id: &tt.tgId, - Name: &tt.tgName, - Status: &tt.tgStatus, - Type: nil, - }} - - mockLattice.EXPECT().ListTargetGroupsAsList(ctx, listTGinput).Return(listTGOutput, nil) - - targetGroupManager := NewTargetGroupManager(gwlog.FallbackLogger, cloud) - - resp, err := targetGroupManager.Get(ctx, tt.input) - - if tt.wantErr != nil { - assert.NotNil(t, err) - assert.Equal(t, err, tt.wantErr) - } else { - assert.Nil(t, err) - assert.Equal(t, resp.TargetGroupID, tt.wantOutput.TargetGroupID) - assert.Equal(t, resp.TargetGroupARN, tt.wantOutput.TargetGroupARN) - } - }) - } -} - func Test_defaultTargetGroupManager_getDefaultHealthCheckConfig(t *testing.T) { var ( resetValue = aws.Int64(0) @@ -1218,10 +901,163 @@ func Test_defaultTargetGroupManager_getDefaultHealthCheckConfig(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - s := NewTargetGroupManager(gwlog.FallbackLogger, nil) + c := gomock.NewController(t) + defer c.Finish() + + cloud := pkg_aws.NewDefaultCloud(nil, TestCloudConfig) + + s := NewTargetGroupManager(gwlog.FallbackLogger, cloud) + if got := s.getDefaultHealthCheckConfig(tt.args.targetGroupProtocolVersion); !reflect.DeepEqual(got, tt.want) { t.Errorf("defaultTargetGroupManager.getDefaultHealthCheckConfig() = %v, want %v", got, tt.want) } }) } } + +func Test_IsTargetGroupMatch(t *testing.T) { + tests := []struct { + name string + expectedResult bool + wantErr bool + modelTg *model.TargetGroup + latticeTg *vpclattice.TargetGroupSummary + tags *model.TargetGroupTagFields + listTagsOut *vpclattice.ListTagsForResourceOutput + getTgOut *vpclattice.GetTargetGroupOutput + }{ + { + name: "port not equal", + expectedResult: false, + wantErr: false, + modelTg: &model.TargetGroup{ + Spec: model.TargetGroupSpec{ + Port: 8080, + }, + }, + latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, + tags: &model.TargetGroupTagFields{}, + }, + { + name: "tags not equal", + expectedResult: false, + wantErr: false, + modelTg: &model.TargetGroup{ + Spec: model.TargetGroupSpec{ + Port: 443, + }, + }, + latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, + tags: &model.TargetGroupTagFields{EKSClusterName: "foo"}, + }, + { + name: "fetch tags not equal", + expectedResult: false, + wantErr: false, + modelTg: &model.TargetGroup{ + Spec: model.TargetGroupSpec{ + Port: 443, + }, + }, + latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, + listTagsOut: &vpclattice.ListTagsForResourceOutput{ + Tags: map[string]*string{ + model.EKSClusterNameKey: aws.String("foo"), + }, + }, + }, + { + name: "protocol version not equal", + expectedResult: false, + wantErr: false, + modelTg: &model.TargetGroup{ + Spec: model.TargetGroupSpec{ + Port: 443, + ProtocolVersion: "HTTP1", + }, + }, + latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, + tags: &model.TargetGroupTagFields{}, + getTgOut: &vpclattice.GetTargetGroupOutput{ + Config: &vpclattice.TargetGroupConfig{ + ProtocolVersion: aws.String("HTTP2"), + }, + }, + }, + { + name: "equal with existing tags", + expectedResult: true, + wantErr: false, + modelTg: &model.TargetGroup{ + Spec: model.TargetGroupSpec{ + Port: 443, + ProtocolVersion: "HTTP1", + TargetGroupTagFields: model.TargetGroupTagFields{ + EKSClusterName: "cluster", + }, + }, + }, + latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, + tags: &model.TargetGroupTagFields{ + EKSClusterName: "cluster", + }, + getTgOut: &vpclattice.GetTargetGroupOutput{ + Config: &vpclattice.TargetGroupConfig{ + ProtocolVersion: aws.String("HTTP1"), + }, + }, + }, + { + name: "equal with fetched tags", + expectedResult: true, + wantErr: false, + modelTg: &model.TargetGroup{ + Spec: model.TargetGroupSpec{ + Port: 443, + ProtocolVersion: "HTTP1", + TargetGroupTagFields: model.TargetGroupTagFields{ + EKSClusterName: "cluster", + }, + }, + }, + latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, + listTagsOut: &vpclattice.ListTagsForResourceOutput{ + Tags: map[string]*string{ + model.EKSClusterNameKey: aws.String("cluster"), + }, + }, + getTgOut: &vpclattice.GetTargetGroupOutput{ + Config: &vpclattice.TargetGroupConfig{ + ProtocolVersion: aws.String("HTTP1"), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + + mockLattice := mocks.NewMockLattice(c) + cloud := pkg_aws.NewDefaultCloud(mockLattice, TestCloudConfig) + + if tt.listTagsOut != nil { + mockLattice.EXPECT().ListTagsForResourceWithContext(ctx, gomock.Any()).Return(tt.listTagsOut, nil) + } + if tt.getTgOut != nil { + mockLattice.EXPECT().GetTargetGroupWithContext(ctx, gomock.Any()).Return(tt.getTgOut, nil) + } + + s := NewTargetGroupManager(gwlog.FallbackLogger, cloud) + result, err := s.IsTargetGroupMatch(ctx, tt.modelTg, tt.latticeTg, tt.tags) + if tt.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tt.expectedResult, result) + }) + } +} diff --git a/pkg/deploy/lattice/target_group_synthesizer.go b/pkg/deploy/lattice/target_group_synthesizer.go index 0ee0bcd5..88f3577a 100644 --- a/pkg/deploy/lattice/target_group_synthesizer.go +++ b/pkg/deploy/lattice/target_group_synthesizer.go @@ -3,9 +3,11 @@ package lattice import ( "context" "errors" - "strings" - + "github.com/aws/aws-application-networking-k8s/pkg/gateway" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/vpclattice" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "time" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -15,26 +17,38 @@ import ( pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-application-networking-k8s/pkg/config" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) +type ActionDirective bool + +const ( + // just helps a bit with readability + PerformUpserts ActionDirective = true + DoNotPerformUpserts ActionDirective = false + PerformDeletes ActionDirective = true + DoNotPerformDeletes ActionDirective = false +) + +// helpful for testing/mocking func NewTargetGroupSynthesizer( log gwlog.Logger, cloud pkg_aws.Cloud, client client.Client, tgManager TargetGroupManager, + svcExportTgBuilder gateway.SvcExportTargetGroupModelBuilder, + svcBuilder gateway.LatticeServiceBuilder, stack core.Stack, - latticeDataStore *latticestore.LatticeDataStore, ) *TargetGroupSynthesizer { return &TargetGroupSynthesizer{ log: log, cloud: cloud, client: client, targetGroupManager: tgManager, + svcExportTgBuilder: svcExportTgBuilder, + svcBuilder: svcBuilder, stack: stack, - latticeDataStore: latticeDataStore, } } @@ -44,397 +58,342 @@ type TargetGroupSynthesizer struct { client client.Client targetGroupManager TargetGroupManager stack core.Stack - latticeDataStore *latticestore.LatticeDataStore + svcExportTgBuilder gateway.SvcExportTargetGroupModelBuilder + svcBuilder gateway.LatticeServiceBuilder } func (t *TargetGroupSynthesizer) Synthesize(ctx context.Context) error { - var ret = "" - - if err := t.SynthesizeTriggeredTargetGroup(ctx); err != nil { - ret = LATTICE_RETRY - } - - /* - * TODO: resolve bug that this might delete other Route's TG before they have chance - * to be reconcile during controller restart - */ - // This might conflict and try to delete other TGs in the middle of creation, because - // this is coming from TargetGroupStackDeployer, which can run before all rules are reconciled. - // - // Since the same cleaner logic is running from ServiceStackDeployer, we may not need this here. - // - //if err := t.SynthesizeSDKTargetGroups(ctx); err != nil { - // ret = LATTICE_RETRY - //} - - if ret != "" { - return errors.New(ret) - } else { - return nil - } + return t.synthesize(ctx, PerformUpserts, PerformDeletes) +} +func (t *TargetGroupSynthesizer) SynthesizeCreate(ctx context.Context) error { + return t.synthesize(ctx, PerformUpserts, DoNotPerformDeletes) +} +func (t *TargetGroupSynthesizer) SynthesizeDelete(ctx context.Context) error { + return t.synthesize(ctx, DoNotPerformUpserts, PerformDeletes) } -func (t *TargetGroupSynthesizer) SynthesizeTriggeredTargetGroup(ctx context.Context) error { +func (t *TargetGroupSynthesizer) synthesize(ctx context.Context, performUpserts ActionDirective, performDeletes ActionDirective) error { var resTargetGroups []*model.TargetGroup var returnErr = false - t.stack.ListResources(&resTargetGroups) - - for _, resTargetGroup := range resTargetGroups { - - // find out VPC for service import - if resTargetGroup.Spec.Config.IsServiceImport { - /* right now, TG are unique across VPC, we do NOT need to get VPC - if resTargetGroup.Spec.Config.EKSClusterName != "" { - eksSess := t.cloud.EKS() + err := t.stack.ListResources(&resTargetGroups) + if err != nil { + return err + } - input := &eks.DescribeClusterInput{ - Name: aws.String(resTargetGroup.Spec.Config.EKSClusterName), - } - result, err := eksSess.DescribeCluster(input) + if bool(performDeletes) { + for _, resTargetGroup := range resTargetGroups { + if resTargetGroup.IsDeleted { + prefix := model.TgNamePrefix(resTargetGroup.Spec) + err := t.targetGroupManager.Delete(ctx, resTargetGroup) if err != nil { - t.log.Infof("Error eks DescribeCluster %v", err) + t.log.Infof("Failed TargetGroupManager.Delete %s due to %s", prefix, err) returnErr = true - continue - } else { - t.log.Infof("Found VPCID =%s for EKS cluster %s", result.String(), resTargetGroup.Spec.Config.EKSClusterName) - resTargetGroup.Spec.Config.VpcID = *result.Cluster.ResourcesVpcConfig.VpcId - t.log.Infof("targetGroup.Spec.Config.VpcID = %s", resTargetGroup.Spec.Config.VpcID) } } - // TODO today, targetGroupManager.Create() will list all target and find out the matching one - resTargetGroup.Spec.Config.VpcID = resTargetGroup.Spec.Config.VpcID - */ - - // TODO in future, we might want to use annotation to specify lattice TG arn or ID - if resTargetGroup.Spec.IsDeleted { - //Ingnore TG delete since this is an import from elsewhere - continue - } - - tgStatus, err := t.targetGroupManager.Get(ctx, resTargetGroup) - if err != nil { - t.log.Debugf("Error getting target group: %s", err) - returnErr = true - continue - } - - // for serviceimport, the httproutename is "" - - t.latticeDataStore.AddTargetGroup(resTargetGroup.Spec.Name, - resTargetGroup.Spec.Config.VpcID, tgStatus.TargetGroupARN, tgStatus.TargetGroupID, - resTargetGroup.Spec.Config.IsServiceImport, "") - - t.log.Infof("Successfully synthesized target group %s with status %s", resTargetGroup.Spec.Name, tgStatus) - } else { - if resTargetGroup.Spec.IsDeleted { - err := t.targetGroupManager.Delete(ctx, resTargetGroup) - if err != nil { - returnErr = true - continue + } + } + if bool(performUpserts) { + for _, resTargetGroup := range resTargetGroups { + if !resTargetGroup.IsDeleted { + prefix := model.TgNamePrefix(resTargetGroup.Spec) + + tgStatus, err := t.targetGroupManager.Upsert(ctx, resTargetGroup) + if err == nil { + resTargetGroup.Status = &tgStatus } else { - t.log.Debugf("Successfully deleted target group %s", resTargetGroup.Spec.Name) - t.latticeDataStore.DelTargetGroup(resTargetGroup.Spec.Name, resTargetGroup.Spec.Config.K8SHTTPRouteName, false) - } - } else { - resTargetGroup.Spec.Config.VpcID = config.VpcID - - tgStatus, err := t.targetGroupManager.Create(ctx, resTargetGroup) - if err != nil { - t.log.Debugf("Error creating target group: %s", err) + t.log.Debugf("Failed TargetGroupManager.Upsert %s due to %s", prefix, err) returnErr = true - continue } - - t.latticeDataStore.AddTargetGroup(resTargetGroup.Spec.Name, - resTargetGroup.Spec.Config.VpcID, tgStatus.TargetGroupARN, - tgStatus.TargetGroupID, resTargetGroup.Spec.Config.IsServiceImport, - resTargetGroup.Spec.Config.K8SHTTPRouteName) - - t.log.Debugf("Successfully synthesized target group %s", resTargetGroup.Spec.Name) } } } if returnErr { - return errors.New("LATTICE-RETRY") - } else { - return nil + t.log.Infof("Error during target group synthesis, will retry") + return errors.New(LATTICE_RETRY) } + + return nil } -func (t *TargetGroupSynthesizer) SynthesizeSDKTargetGroups(ctx context.Context) error { - var staleSDKTGs []model.TargetGroup - sdkTGs, err := t.targetGroupManager.List(ctx) +// this method assumes all synthesis +func (t *TargetGroupSynthesizer) SynthesizeUnusedDelete(ctx context.Context) error { + tgsToDelete, err := t.calculateTargetGroupsToDelete(ctx) if err != nil { - t.log.Errorf("Error listing target groups: %s", err) - return nil + return err } - for _, sdkTG := range sdkTGs { - tgRouteName := "" - - if *sdkTG.getTargetGroupOutput.Config.VpcIdentifier != config.VpcID { - t.log.Debugf("Ignoring target group %s (%s) because it is configured for other VPCs", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - continue + retErr := false + for _, tg := range tgsToDelete { + modelStatus := model.TargetGroupStatus{ + Name: aws.StringValue(tg.getTargetGroupOutput.Name), + Arn: aws.StringValue(tg.getTargetGroupOutput.Arn), + Id: aws.StringValue(tg.getTargetGroupOutput.Id), } - - // does target group have K8S tags, ignore if it is not tagged - tgTags := sdkTG.targetGroupTags - if tgTags == nil || tgTags.Tags == nil { - t.log.Debugf("Ignoring target group %s (%s) because it is not tagged for K8S", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - continue + modelTg := model.TargetGroup{ + Status: &modelStatus, + IsDeleted: true, } - parentRef, ok := tgTags.Tags[model.K8SParentRefTypeKey] - if !ok || parentRef == nil { - t.log.Debugf("Ignoring target group %s (%s) because it has no K8S parentRef tag", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - continue + err := t.targetGroupManager.Delete(ctx, &modelTg) + if err != nil { + t.log.Infof("Failed TargetGroupManager.Delete %s due to %s", modelStatus.Id, err) + retErr = true } + } - srvName, ok := tgTags.Tags[model.K8SServiceNameKey] + if retErr { + return errors.New(LATTICE_RETRY) + } else { + return nil + } +} - if !ok || srvName == nil { - t.log.Debugf("Ignoring target group %s (%s) because it has no servicename tag", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - continue - } +func (t *TargetGroupSynthesizer) calculateTargetGroupsToDelete(ctx context.Context) ([]tgListOutput, error) { + latticeTgs, err := t.targetGroupManager.List(ctx) + if err != nil { + t.log.Infof("Failed TargetGroupManager.List due to %s", err) + return latticeTgs, err + } - srvNamespace, ok := tgTags.Tags[model.K8SServiceNamespaceKey] + var tgsToDelete []tgListOutput - if !ok || srvNamespace == nil { - t.log.Infof("Ignoring target group %s (%s) because it has no serviceNamespace tag", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) + // we check existing target groups to see if they are still in use - this is necessary as + // some changes to existing service exports or routes will simply create new target groups, + // for example on protocol changes + for _, latticeTg := range latticeTgs { + tagFields, controllerManaged := t.isControllerManaged(latticeTg) + if !controllerManaged { continue } - // if its parentref is service export, check the parent service export exist - // Ignore if service export exists - if *parentRef == model.K8SServiceExportType { - t.log.Infof("TargetGroup %s (%s) is referenced by ServiceExport", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) + // most importantly, is the tg in use? + if len(latticeTg.getTargetGroupOutput.ServiceArns) > 0 { + t.log.Debugf("TargetGroup %s (%s) is referenced by lattice service", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + continue + } - srvExportName := types.NamespacedName{ - Namespace: *srvNamespace, - Name: *srvName, + if tagFields.K8SParentRefType == model.ParentRefTypeSvcExport { + if t.shouldDeleteSvcExportTg(ctx, latticeTg, tagFields) { + tgsToDelete = append(tgsToDelete, latticeTg) } - srvExport := &mcsv1alpha1.ServiceExport{} - if err := t.client.Get(ctx, srvExportName, srvExport); err == nil { - t.log.Debugf("Ignoring target group %s (%s), which was triggered by serviceexport, since serviceexport object is found", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - continue + } else { + if t.shouldDeleteRouteTg(ctx, latticeTg, tagFields) { + tgsToDelete = append(tgsToDelete, latticeTg) } } + } + return tgsToDelete, nil +} - // if its parentRef is a route, check that the parent route exists - // Ignore if route does not exist - if *parentRef == model.K8SHTTPRouteType { - t.log.Infof("Target group %s (%s) is referenced by a route", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - - routeNameValue, ok := tgTags.Tags[model.K8SHTTPRouteNameKey] - tgRouteName = *routeNameValue - if !ok || routeNameValue == nil { - t.log.Infof("Ignoring target group %s (%s), which was triggered by a route, because it has no route name tag", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - continue - } - - routeNamespaceValue, ok := tgTags.Tags[model.K8SHTTPRouteNamespaceKey] - - if !ok || routeNamespaceValue == nil { - t.log.Infof("Ignoring target group %s (%s), which was triggered by a route, because it has no route namespace tag", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - continue - } +func (t *TargetGroupSynthesizer) shouldDeleteSvcExportTg( + ctx context.Context, latticeTg tgListOutput, tagFields model.TargetGroupTagFields) bool { - routeName := types.NamespacedName{ - Namespace: *routeNamespaceValue, - Name: *routeNameValue, - } + svcExportName := types.NamespacedName{ + Namespace: tagFields.K8SServiceNamespace, + Name: tagFields.K8SServiceName, + } - var route core.Route - if *sdkTG.getTargetGroupOutput.Config.ProtocolVersion == vpclattice.TargetGroupProtocolVersionGrpc { - if route, err = core.GetGRPCRoute(ctx, t.client, routeName); err != nil { - t.log.Errorf("Could not find GRPCRoute for target group %s", err) - } - } else { - if route, err = core.GetHTTPRoute(ctx, t.client, routeName); err != nil { - t.log.Errorf("Could not find HTTPRoute for target group %s", err) - } - } + t.log.Debugf("TargetGroup %s (%s) is referenced by ServiceExport", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) - if route != nil { - tgName := latticestore.TargetGroupName(*srvName, *srvNamespace) - - // We have finished rule reconciliation at this point. - // If a target group under HTTPRoute does not have any service, it is stale. - isUsed := t.isTargetGroupUsedByRoute(ctx, tgName, route) && - len(sdkTG.getTargetGroupOutput.ServiceArns) > 0 - if isUsed { - t.log.Infof("Ignoring target group %s (%s), which was triggered by a route, since route object is found", - *sdkTG.getTargetGroupOutput.Arn, *sdkTG.getTargetGroupOutput.Name) - continue - } else { - t.log.Infof("target group %s is not used by route %s-%s", tgName, route.Name(), route.Namespace()) - } - } - } - - // the routename for serviceimport is "" - if tg, err := t.latticeDataStore.GetTargetGroup(*sdkTG.getTargetGroupOutput.Name, "", true); err == nil { - t.log.Debugf("Ignoring target group %s, which was created by service import", tg.TargetGroupKey.Name) - continue + svcExport := &mcsv1alpha1.ServiceExport{} + err := t.client.Get(ctx, svcExportName, svcExport) + if err != nil { + if apierrors.IsNotFound(err) { + // if the service export does not exist, we can safely delete + t.log.Infof("Will delete TargetGroup %s (%s) - ServiceExport is not found", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return true + } else { + // skip if we have an unknown error + t.log.Infof("Received unexpected API error getting service export %s", err) + return false } + } - t.log.Debugf("Appending stale target group to stale list. Name: %s, routename: %s, ARN: %s", - *sdkTG.getTargetGroupOutput.Name, tgRouteName, *sdkTG.getTargetGroupOutput.Id) + if !svcExport.DeletionTimestamp.IsZero() { + // backing object is deleted, we can delete too + t.log.Infof("Will delete TargetGroup %s (%s) - ServiceExport has been deleted", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return true + } - staleSDKTGs = append(staleSDKTGs, model.TargetGroup{ - Spec: model.TargetGroupSpec{ - Name: *sdkTG.getTargetGroupOutput.Name, - Config: model.TargetGroupConfig{ - K8SHTTPRouteName: tgRouteName, - }, - LatticeID: *sdkTG.getTargetGroupOutput.Id, - }, - }) + // now we get to the tricky business of seeing if our unused target group actually matches + // the current state of the service and service export - the most correct way to do this is to + // reconstruct the target group spec from the service export itself, then compare fields + modelTg, err := t.svcExportTgBuilder.BuildTargetGroup(ctx, svcExport) + if err != nil { + t.log.Infof("Received error building svc export target group model %s", err) + return false + } + // tags are already validated, just need to check the other essentials + ltg := latticeTg.getTargetGroupOutput + if int64(modelTg.Spec.Port) != aws.Int64Value(ltg.Config.Port) || + modelTg.Spec.Protocol != aws.StringValue(ltg.Config.Protocol) || + modelTg.Spec.ProtocolVersion != aws.StringValue(ltg.Config.ProtocolVersion) || + modelTg.Spec.IpAddressType != aws.StringValue(ltg.Config.IpAddressType) { + + // one or more immutable fields differ from the source, so the TG is out of date + t.log.Infof("Will delete TargetGroup %s (%s) - fields differ from source service/service export", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return true } - retErr := false + t.log.Debugf("ServiceExport TargetGroup %s (%s) is up to date", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) - for _, sdkTG := range staleSDKTGs { - err := t.targetGroupManager.Delete(ctx, &sdkTG) - if err != nil && !strings.Contains(err.Error(), "TargetGroup is referenced in routing configuration, listeners or rules of service.") { - t.log.Debugf("Error deleting target group %s", err) - retErr = true - } - // continue on even when there is an err + return false +} + +func (t *TargetGroupSynthesizer) shouldDeleteRouteTg( + ctx context.Context, latticeTg tgListOutput, tagFields model.TargetGroupTagFields) bool { + + routeName := types.NamespacedName{ + Namespace: tagFields.K8SRouteNamespace, + Name: tagFields.K8SRouteName, } - if retErr { - return errors.New(LATTICE_RETRY) + var err error + var route core.Route + if *latticeTg.getTargetGroupOutput.Config.ProtocolVersion == vpclattice.TargetGroupProtocolVersionGrpc { + route, err = core.GetGRPCRoute(ctx, t.client, routeName) } else { - return nil + route, err = core.GetHTTPRoute(ctx, t.client, routeName) } -} -func (t *TargetGroupSynthesizer) isTargetGroupUsedByRoute(ctx context.Context, tgName string, route core.Route) bool { - for _, rule := range route.Spec().Rules() { - for _, backendRef := range rule.BackendRefs() { - if string(*backendRef.Kind()) != "Service" { - continue - } - namespace := route.Namespace() - if backendRef.Namespace() != nil { - namespace = string(*backendRef.Namespace()) - } - refTGName := latticestore.TargetGroupName(string(backendRef.Name()), namespace) - - if tgName == refTGName { - return true - } + if err != nil { + if apierrors.IsNotFound(err) { + // if the route does not exist, we can safely delete + t.log.Debugf("Will delete TargetGroup %s (%s) - Route is not found", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return true + } else { + // skip if we have an unknown error + t.log.Infof("Received unexpected API error getting route %s", err) + return false } } - return false -} + if !route.DeletionTimestamp().IsZero() { + t.log.Debugf("Will delete TargetGroup %s (%s) - Route is deleted", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return true + } -func (t *TargetGroupSynthesizer) PostSynthesize(ctx context.Context) error { - // nothing to do here - return nil -} + // basically rebuild everything for the route and see if one of the TGs matches + routeStack, err := t.svcBuilder.Build(ctx, route) + if err != nil { + t.log.Infof("Received error building route model %s", err) + return false + } -func (t *TargetGroupSynthesizer) SynthesizeTriggeredTargetGroupsCreation(ctx context.Context) error { var resTargetGroups []*model.TargetGroup - var returnErr = false - err := t.stack.ListResources(&resTargetGroups) + err = routeStack.ListResources(&resTargetGroups) if err != nil { - return err + t.log.Infof("Error listing stack target groups %s", err) + return false } - for _, resTargetGroup := range resTargetGroups { - if resTargetGroup.Spec.IsDeleted { - t.log.Debugf("Ignoring deletion request for target group %s", resTargetGroup.Spec.Name) + var matchFound bool + for _, modelTg := range resTargetGroups { + ltg := latticeTg.getTargetGroupOutput + latticeTgSummary := vpclattice.TargetGroupSummary{ + Arn: ltg.Arn, + CreatedAt: ltg.CreatedAt, + Id: ltg.Id, + IpAddressType: ltg.Config.IpAddressType, + LastUpdatedAt: ltg.LastUpdatedAt, + Name: ltg.Name, + Port: ltg.Config.Port, + Protocol: ltg.Config.Protocol, + ServiceArns: ltg.ServiceArns, + Status: ltg.Status, + Type: ltg.Type, + VpcIdentifier: ltg.Config.VpcIdentifier, + } + + match, err := t.targetGroupManager.IsTargetGroupMatch(ctx, modelTg, &latticeTgSummary, &tagFields) + if err != nil { + t.log.Infof("Received error during tg comparison %s", err) continue } - if resTargetGroup.Spec.Config.IsServiceImport { - tgStatus, err := t.targetGroupManager.Get(ctx, resTargetGroup) - if err != nil { - t.log.Debugf("Error getting target group %s due to %s", resTargetGroup.Spec.Name, err) - returnErr = true - continue - } + if match { + t.log.Debugf("Route TargetGroup %s (%s) is up to date", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) - // for serviceimport, the httproutename is "" - t.latticeDataStore.AddTargetGroup(resTargetGroup.Spec.Name, - resTargetGroup.Spec.Config.VpcID, tgStatus.TargetGroupARN, tgStatus.TargetGroupID, - resTargetGroup.Spec.Config.IsServiceImport, "") - - t.log.Debugf("Successfully synthesized target group %s with status %s", resTargetGroup.Spec.Name, tgStatus) - } else { // handle TargetGroup creation request that triggered by httproute with backendref k8sService creation or serviceExport creation - resTargetGroup.Spec.Config.VpcID = config.VpcID - tgStatus, err := t.targetGroupManager.Create(ctx, resTargetGroup) - if err != nil { - t.log.Debugf("Error creating target group %s due to %s", resTargetGroup.Spec.Name, err) - returnErr = true - continue - } - //In the ModelBuildTask, it should already add a tg entry in the latticeDataStore, - //in here, only UPDATE the entry with tgStatus.TargetGroupARN and tgStatus.TargetGroupID - t.latticeDataStore.AddTargetGroup(resTargetGroup.Spec.Name, - resTargetGroup.Spec.Config.VpcID, tgStatus.TargetGroupARN, - tgStatus.TargetGroupID, resTargetGroup.Spec.Config.IsServiceImport, - resTargetGroup.Spec.Config.K8SHTTPRouteName) - - t.log.Debugf("Successfully synthesized target group %s with status %s", resTargetGroup.Spec.Name, tgStatus) + matchFound = true + break } } - if returnErr { - return errors.New(LATTICE_RETRY) - } else { - return nil + if !matchFound { + t.log.Debugf("Will delete TargetGroup %s (%s) - TG is not up to date", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + + return true // safe to delete + } + + // here we just delete anything more than X minutes old - worst case we'll have to recreate + // the target group - note this case is only theoretically possible at this point + fiveMinsAgo := time.Now().Add(-time.Minute * 5) + if fiveMinsAgo.After(aws.TimeValue(latticeTg.getTargetGroupOutput.CreatedAt)) { + t.log.Debugf("Will delete TargetGroup %s (%s) - TG is more than 5 minutes old", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return true } + + return false } -func (t *TargetGroupSynthesizer) SynthesizeTriggeredTargetGroupsDeletion(ctx context.Context) error { - var resTargetGroups []*model.TargetGroup - var returnErr = false - err := t.stack.ListResources(&resTargetGroups) - if err != nil { - return err +func (t *TargetGroupSynthesizer) isControllerManaged(latticeTg tgListOutput) (model.TargetGroupTagFields, bool) { + if latticeTg.targetGroupTags == nil { + t.log.Debugf("Ignoring target group %s (%s) because tag fetch was not successful", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return model.TargetGroupTagFields{}, false } - for _, resTargetGroup := range resTargetGroups { - if !resTargetGroup.Spec.IsDeleted { - t.log.Infof("Ignoring target group %s because it is not deleted", resTargetGroup.Spec.Name) - continue - } + // TGs from earlier releases will require 1-time manual cleanup + // this method of validation only covers TGs created by this build + // of the controller forward + if aws.StringValue(latticeTg.getTargetGroupOutput.Config.VpcIdentifier) != config.VpcID { + t.log.Debugf("Ignoring target group %s (%s) because it is not configured for this VPC", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return model.TargetGroupTagFields{}, false + } - if resTargetGroup.Spec.Config.IsServiceImport { - t.log.Debugf("Deleting service import target group from local datastore %s", resTargetGroup.Spec.LatticeID) - t.latticeDataStore.DelTargetGroup(resTargetGroup.Spec.Name, resTargetGroup.Spec.Config.K8SHTTPRouteName, resTargetGroup.Spec.Config.IsServiceImport) - } else { - // For delete TargetGroup request triggered by k8s service, invoke vpc lattice api to delete it, if success, delete the tg in the datastore as well - err := t.targetGroupManager.Delete(ctx, resTargetGroup) - if err == nil { - t.latticeDataStore.DelTargetGroup(resTargetGroup.Spec.Name, resTargetGroup.Spec.Config.K8SHTTPRouteName, resTargetGroup.Spec.Config.IsServiceImport) - } else { - t.log.Debugf("Error deleting target group %s due to %s", resTargetGroup.Spec.Name, err) - returnErr = true - } - } + tagFields := model.TGTagFieldsFromTags(latticeTg.targetGroupTags.Tags) + + if tagFields.EKSClusterName != config.ClusterName { + t.log.Debugf("Ignoring target group %s (%s) because it is not configured for this Cluster", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return model.TargetGroupTagFields{}, false } - if returnErr { - return errors.New(LATTICE_RETRY) - } else { - return nil + + if tagFields.K8SParentRefType == model.ParentRefTypeInvalid || + tagFields.K8SServiceName == "" || tagFields.K8SServiceNamespace == "" { + + t.log.Infof("Ignoring target group %s (%s) as one or more required tags are missing", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return model.TargetGroupTagFields{}, false + } + + // route-based TGs should have the additional route keys + if tagFields.IsRoute() && (tagFields.K8SRouteName == "" || tagFields.K8SRouteNamespace == "") { + t.log.Infof("Ignoring route-based target group %s (%s) as one or more required tags are missing", + *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) + return model.TargetGroupTagFields{}, false } + + return tagFields, true +} + +func (t *TargetGroupSynthesizer) PostSynthesize(ctx context.Context) error { + // nothing to do here + return nil } diff --git a/pkg/deploy/lattice/target_group_synthesizer_test.go b/pkg/deploy/lattice/target_group_synthesizer_test.go index b1437065..b8834db0 100644 --- a/pkg/deploy/lattice/target_group_synthesizer_test.go +++ b/pkg/deploy/lattice/target_group_synthesizer_test.go @@ -2,1244 +2,487 @@ package lattice import ( "context" - "errors" - "fmt" - "testing" - + mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" + "github.com/aws/aws-application-networking-k8s/pkg/config" + "github.com/aws/aws-application-networking-k8s/pkg/gateway" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/vpclattice" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - - corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "net/http" + "sigs.k8s.io/controller-runtime/pkg/client" + "testing" + "time" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" - - "k8s.io/apimachinery/pkg/runtime" - clientgoscheme "k8s.io/client-go/kubernetes/scheme" - testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - - mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" - "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" - "github.com/aws/aws-application-networking-k8s/pkg/config" - "github.com/aws/aws-application-networking-k8s/pkg/gateway" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) -func Test_SynthesizeTriggeredServiceExport(t *testing.T) { - now := metav1.Now() - tests := []struct { - name string - svcExport *mcsv1alpha1.ServiceExport - tgManagerError bool - wantErrIsNil bool - }{ - { - name: "Adding a new targetgroup, ok case", - svcExport: &mcsv1alpha1.ServiceExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: "export1", - Namespace: "ns1", - }, - }, - tgManagerError: false, - wantErrIsNil: true, - }, - { - name: "Adding a new targetgroup, nok case", - svcExport: &mcsv1alpha1.ServiceExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: "export2", - Namespace: "ns1", - }, - }, - tgManagerError: true, - wantErrIsNil: false, - }, - { - name: "Deleting a targetgroup, ok case", - svcExport: &mcsv1alpha1.ServiceExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: "export3", - Namespace: "ns1", - Finalizers: []string{"gateway.k8s.aws/resources"}, - DeletionTimestamp: &now, - }, - }, - tgManagerError: false, - wantErrIsNil: true, - }, - { - name: "Deleting a targetgroup, nok case", - svcExport: &mcsv1alpha1.ServiceExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: "export3", - Namespace: "ns1", - Finalizers: []string{"gateway.k8s.aws/resources"}, - DeletionTimestamp: &now, - }, - }, - tgManagerError: true, - wantErrIsNil: false, - }, +func Test_Synthesize(t *testing.T) { + // all synthesize does is delegate to the manager + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockTGManager := NewMockTargetGroupManager(c) + + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) + tgToDelete := &model.TargetGroup{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", "tg-delete"), + IsDeleted: true, } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - k8sSchema := runtime.NewScheme() - clientgoscheme.AddToScheme(k8sSchema) - v1alpha1.AddToScheme(k8sSchema) - k8sClient := testclient.NewFakeClientWithScheme(k8sSchema) - - svc := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.svcExport.Name, - Namespace: tt.svcExport.Namespace, - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - {}, - }, - IPFamilies: []corev1.IPFamily{ - corev1.IPv4Protocol, - }, - }, - } - - k8sClient.Create(ctx, svc.DeepCopy()) - - mockTGManager := NewMockTargetGroupManager(c) - - ds := latticestore.NewLatticeDataStore() - if !tt.svcExport.DeletionTimestamp.IsZero() { - // When test serviceExport deletion, we expect latticeDataStore already had this tg entry - tgName := latticestore.TargetGroupName(tt.svcExport.Name, tt.svcExport.Namespace) - ds.AddTargetGroup(tgName, "vpc-123456789", "123456789", "tg-123", false, "") - } - - builder := gateway.NewSvcExportTargetGroupBuilder(gwlog.FallbackLogger, k8sClient, ds, nil) - - stack, tg, err := builder.Build(ctx, tt.svcExport) - assert.Nil(t, err) - - synthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, stack, ds) - - tgStatus := model.TargetGroupStatus{ - TargetGroupARN: "arn123", - TargetGroupID: "4567", - } - - if tg.Spec.IsDeleted { - if tt.tgManagerError { - mockTGManager.EXPECT().Delete(ctx, tg).Return(errors.New("ERROR")) - } else { - mockTGManager.EXPECT().Delete(ctx, tg).Return(nil) - } - } else { - if tt.tgManagerError { - mockTGManager.EXPECT().Create(ctx, tg).Return(tgStatus, errors.New("ERROR")) - } else { - mockTGManager.EXPECT().Create(ctx, tg).Return(tgStatus, nil) - } - } - - err = synthesizer.SynthesizeTriggeredTargetGroup(ctx) - - if !tt.wantErrIsNil { - assert.NotNil(t, err) - return - } - - assert.Nil(t, err) - tgName := latticestore.TargetGroupName(tt.svcExport.Name, tt.svcExport.Namespace) - dsTG, err := ds.GetTargetGroup(tgName, "", false) - - if tg.Spec.IsDeleted { - assert.NotNil(t, err) - - } else { - assert.Nil(t, err) - assert.Equal(t, dsTG.ARN, tgStatus.TargetGroupARN) - assert.Equal(t, dsTG.ID, tgStatus.TargetGroupID) - } - }) + tgToCreate := &model.TargetGroup{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", "tg-create"), + IsDeleted: false, } -} + assert.NoError(t, stack.AddResource(tgToDelete)) + assert.NoError(t, stack.AddResource(tgToCreate)) + + mockTGManager.EXPECT().Delete(ctx, tgToDelete).Return(nil) + mockTGManager.EXPECT().Upsert(ctx, tgToCreate).Return(model.TargetGroupStatus{Name: "create-name"}, nil) -type svcImportDef struct { - name string - tgARN string - tgID string - tgExist bool - mgrErr bool + synthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, nil, nil, stack) + + err := synthesizer.Synthesize(ctx) + assert.Nil(t, err) + assert.Equal(t, "create-name", tgToCreate.Status.Name) } -func Test_SynthersizeTriggeredByServiceImport(t *testing.T) { - tests := []struct { - name string - svcImportList []svcImportDef - isDeleted bool - wantErrIsNil bool - }{ - { - name: "service import triggered target group", - svcImportList: []svcImportDef{ - { - name: "service-import1", - tgARN: "service-import1-arn", - tgID: "service-import1-ID", - tgExist: true, - mgrErr: false, - }, - { - name: "service-import2", - tgARN: "service-import2-arn", - tgID: "service-import2-ID", - tgExist: true, - mgrErr: false, - }, - }, - isDeleted: false, - wantErrIsNil: true, - }, - { - name: "service import triggered target group, 1st one return err", - svcImportList: []svcImportDef{ - { - name: "service-import21", - tgExist: false, - mgrErr: true, - }, - { - name: "service-import22", - tgARN: "service-import22-arn", - tgID: "service-import22-ID", - tgExist: true, - mgrErr: false, - }, - }, - isDeleted: false, - wantErrIsNil: false, - }, - { - name: "service import triggered target group, 1st one return err", - svcImportList: []svcImportDef{ - { - name: "service-import31", - tgExist: false, - mgrErr: true, - }, - { - name: "service-import32", - tgARN: "service-import32-arn", - tgID: "service-import32-ID", - tgExist: true, - mgrErr: false, - }, - }, - isDeleted: true, - wantErrIsNil: true, +func copy(src tgListOutput) tgListOutput { + srcgto := src.getTargetGroupOutput + cp := tgListOutput{ + getTargetGroupOutput: vpclattice.GetTargetGroupOutput{ + Arn: aws.String(aws.StringValue(srcgto.Arn)), + Config: nil, + Id: aws.String(aws.StringValue(srcgto.Id)), + Name: aws.String(aws.StringValue(srcgto.Name)), + Type: aws.String(aws.StringValue(srcgto.Type)), + CreatedAt: aws.Time(aws.TimeValue(srcgto.CreatedAt)), }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockTGManager := NewMockTargetGroupManager(c) - - ds := latticestore.NewLatticeDataStore() - - stack := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - for _, tgImport := range tt.svcImportList { - - tgSpec := model.TargetGroupSpec{ - Name: tgImport.name, - Type: model.TargetGroupTypeIP, - Config: model.TargetGroupConfig{ - IsServiceImport: true, - }, - IsDeleted: tt.isDeleted, - } - - tg := model.NewTargetGroup(stack, tgImport.name, tgSpec) - fmt.Printf("tg : %v\n", tg) - - if tt.isDeleted { - continue - } - - if tgImport.mgrErr { - mockTGManager.EXPECT().Get(ctx, tg).Return(model.TargetGroupStatus{}, errors.New("tgmgr err")) - } else { - mockTGManager.EXPECT().Get(ctx, tg).Return(model.TargetGroupStatus{TargetGroupARN: tgImport.tgARN, TargetGroupID: tgImport.tgID}, nil) - } - } - synthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, stack, ds) - err := synthesizer.SynthesizeTriggeredTargetGroup(ctx) - fmt.Printf("err:%v \n", err) - - if tt.wantErrIsNil { - assert.Nil(t, err) - } else { - assert.NotNil(t, err) - } - - if !tt.isDeleted { - // check datastore - for _, tgImport := range tt.svcImportList { - if tgImport.mgrErr { - _, err := ds.GetTargetGroup(tgImport.name, "", true) - assert.NotNil(t, err) - - } else { - tg, err := ds.GetTargetGroup(tgImport.name, "", true) - assert.Nil(t, err) - assert.Equal(t, tgImport.tgARN, tg.ARN) - assert.Equal(t, tgImport.tgID, tg.ID) - - } - } - } - }) + + if srcgto.Config != nil { + cp.getTargetGroupOutput.Config = &vpclattice.TargetGroupConfig{ + IpAddressType: aws.String(aws.StringValue(srcgto.Config.IpAddressType)), + Port: aws.Int64(aws.Int64Value(srcgto.Config.Port)), + Protocol: aws.String(aws.StringValue(srcgto.Config.Protocol)), + ProtocolVersion: aws.String(aws.StringValue(srcgto.Config.ProtocolVersion)), + VpcIdentifier: aws.String(aws.StringValue(srcgto.Config.VpcIdentifier)), + } } -} -type sdkTGDef struct { - name string - id string - - isSameVPC bool - hasTags bool - hasServiceExportTypeTag bool - hasHTTPRouteTypeTag bool - serviceExportExist bool - HTTPRouteExist bool - refedByHTTPRoute bool - hasServiceArns bool - serviceNetworkManagerErr error - - expectDelete bool + srctags := src.targetGroupTags + if srctags != nil { + cp.targetGroupTags = &vpclattice.ListTagsForResourceOutput{ + Tags: make(map[string]*string), + } + for k, v := range srctags.Tags { + cp.targetGroupTags.Tags[k] = aws.String(aws.StringValue(v)) + } + } + + return cp } -func Test_SynthesizeSDKTargetGroups(t *testing.T) { - kindPtr := func(k string) *gwv1beta1.Kind { - p := gwv1beta1.Kind(k) - return &p +// we have a list of target groups with varying properties +// TGs that are not managed by the controller +// - tag fetch was unsuccessful (tags nil) +// - vpc id does not match +// - cluster name does not match +// - parent ref type is invalid +// - K8SServiceName missing +// - K8SServiceNamespace missing +// - parent ref type is a route, but K8SRouteName missing +// - parent ref type is a route, but K8SRouteNamespace missing +func Test_SynthesizeUnusedDeleteIgnoreNotManagedByController(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockTGManager := NewMockTargetGroupManager(c) + + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" + + var nonManagedTgs []tgListOutput + + tgTagFetchUnsuccessful := tgListOutput{ + getTargetGroupOutput: vpclattice.GetTargetGroupOutput{ + Arn: aws.String("tg-arn"), + Id: aws.String("tg-id"), + Name: aws.String("tg-name"), + Config: &vpclattice.TargetGroupConfig{}, + Type: aws.String("IP"), + }, + targetGroupTags: nil, } + nonManagedTgs = append(nonManagedTgs, tgTagFetchUnsuccessful) + + tgWrongVpc := copy(tgTagFetchUnsuccessful) + tgWrongVpc.targetGroupTags = &vpclattice.ListTagsForResourceOutput{Tags: make(map[string]*string)} + tgWrongVpc.getTargetGroupOutput.Config.VpcIdentifier = aws.String("another-vpc") + nonManagedTgs = append(nonManagedTgs, tgWrongVpc) + + tgWrongCluster := copy(tgWrongVpc) + tgWrongCluster.getTargetGroupOutput.Config.VpcIdentifier = aws.String("vpc-id") + tgWrongCluster.targetGroupTags.Tags[model.EKSClusterNameKey] = aws.String("another-cluster") + nonManagedTgs = append(nonManagedTgs, tgWrongCluster) + + tgInvalidParentRef := copy(tgWrongCluster) + tgInvalidParentRef.targetGroupTags.Tags[model.EKSClusterNameKey] = aws.String("cluster-name") + tgInvalidParentRef.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeInvalid)) + nonManagedTgs = append(nonManagedTgs, tgInvalidParentRef) + + tgMissingK8SServiceName := copy(tgInvalidParentRef) + tgMissingK8SServiceName.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeSvcExport)) + nonManagedTgs = append(nonManagedTgs, tgMissingK8SServiceName) + + tgMissingK8SServiceNamespace := copy(tgMissingK8SServiceName) + tgMissingK8SServiceNamespace.targetGroupTags.Tags[model.K8SServiceNameKey] = aws.String("my-service") + nonManagedTgs = append(nonManagedTgs, tgMissingK8SServiceNamespace) + + tgMissingRouteName := copy(tgMissingK8SServiceNamespace) + tgMissingRouteName.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeHTTPRoute)) + tgMissingRouteName.targetGroupTags.Tags[model.K8SServiceNamespaceKey] = aws.String("ns-1") + nonManagedTgs = append(nonManagedTgs, tgMissingRouteName) + + tgMissingRouteNamespace := copy(tgMissingRouteName) + tgMissingRouteNamespace.targetGroupTags.Tags[model.K8SRouteNameKey] = aws.String("route-name") + nonManagedTgs = append(nonManagedTgs, tgMissingRouteNamespace) + + mockTGManager.EXPECT().List(ctx).Return(nonManagedTgs, nil) + synthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, nil, nil, nil) + err := synthesizer.SynthesizeUnusedDelete(ctx) + assert.Nil(t, err) +} - config.VpcID = "current-vpc" - srvname := "test-svc1" - srvnamespace := "default" - routename := "test-route1" - routenamespace := "default" - tests := []struct { - name string - sdkTargetGroups []sdkTGDef - wantSynthesizerError error - wantDataStoreError error - wantDataStoreStatus string - }{ - - { - name: "Delete SDK TargetGroup Successfully(due to not refed by any HTTPRoutes) ", - sdkTargetGroups: []sdkTGDef{ - { - name: "sdkTG1", - id: "sdkTG1-id", - serviceNetworkManagerErr: nil, - isSameVPC: true, - hasTags: true, - hasServiceExportTypeTag: false, - hasHTTPRouteTypeTag: true, - HTTPRouteExist: false, - expectDelete: true}, +func getBaseTg() tgListOutput { + baseTg := tgListOutput{ + getTargetGroupOutput: vpclattice.GetTargetGroupOutput{ + Arn: aws.String("tg-arn"), + Id: aws.String("tg-id"), + Name: aws.String("tg-name"), + CreatedAt: aws.Time(time.Now()), + Config: &vpclattice.TargetGroupConfig{ + VpcIdentifier: aws.String("vpc-id"), + Port: aws.Int64(80), + Protocol: aws.String("HTTP"), + ProtocolVersion: aws.String("HTTP1"), + IpAddressType: aws.String("IPV4"), }, - wantSynthesizerError: nil, - wantDataStoreError: nil, - wantDataStoreStatus: "", }, + targetGroupTags: &vpclattice.ListTagsForResourceOutput{Tags: make(map[string]*string)}, + } + baseTg.targetGroupTags.Tags[model.EKSClusterNameKey] = aws.String("cluster-name") + baseTg.targetGroupTags.Tags[model.K8SServiceNameKey] = aws.String("svc") + baseTg.targetGroupTags.Tags[model.K8SServiceNamespaceKey] = aws.String("ns") + return baseTg +} - { - name: "Delete SDK TargetGroup Successfully(due to not in backend ref of a HTTPRoutes) ", - sdkTargetGroups: []sdkTGDef{ - {name: "sdkTG1", id: "sdkTG1-id", serviceNetworkManagerErr: nil, - isSameVPC: true, - hasTags: true, hasServiceExportTypeTag: false, - hasHTTPRouteTypeTag: true, HTTPRouteExist: true, refedByHTTPRoute: false, - expectDelete: true}, - }, - wantSynthesizerError: nil, - wantDataStoreError: nil, - wantDataStoreStatus: "", - }, - { - name: "Delete SDK TargetGroup since it is dangling with no service associated", - sdkTargetGroups: []sdkTGDef{ - {name: "sdkTG1", id: "sdkTG1-id", serviceNetworkManagerErr: nil, - isSameVPC: true, - hasTags: true, hasServiceExportTypeTag: false, - hasHTTPRouteTypeTag: true, HTTPRouteExist: true, refedByHTTPRoute: true, - hasServiceArns: false, - expectDelete: true}, - }, - wantSynthesizerError: nil, - wantDataStoreError: nil, - wantDataStoreStatus: "", +// Do not delete cases +// TG has service arns +// TG is service export +// - port, protocol, protocolVersion, ipaddressType, all match +// +// TG is route +// - TG matches a current TG for the route +func Test_DoNotDeleteCases(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockTGManager := NewMockTargetGroupManager(c) + mockClient := mock_client.NewMockClient(c) + mockSvcExportTgBuilder := gateway.NewMockSvcExportTargetGroupModelBuilder(c) + mockSvcBuilder := gateway.NewMockLatticeServiceBuilder(c) + + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" + + baseTg := getBaseTg() + + var noDeleteTgs []tgListOutput + + tgWithSvcArns := copy(baseTg) + tgWithSvcArns.getTargetGroupOutput.Arn = aws.String("tg-with-svcs-arn") // useful for reading logs + tgWithSvcArns.getTargetGroupOutput.ServiceArns = []*string{aws.String("svc-arn")} + noDeleteTgs = append(noDeleteTgs, tgWithSvcArns) + + tgSvcExportUpToDate := copy(baseTg) + tgSvcExportUpToDate.getTargetGroupOutput.Arn = aws.String("tg-svc-export-arn") + tgSvcExportUpToDate.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeSvcExport)) + noDeleteTgs = append(noDeleteTgs, tgSvcExportUpToDate) + + tgSvcUpToDate := copy(baseTg) + tgSvcUpToDate.getTargetGroupOutput.Arn = aws.String("tg-svc-arn") + tgSvcUpToDate.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeHTTPRoute)) + tgSvcUpToDate.targetGroupTags.Tags[model.K8SRouteNameKey] = aws.String("route") + tgSvcUpToDate.targetGroupTags.Tags[model.K8SRouteNamespaceKey] = aws.String("route-ns") + noDeleteTgs = append(noDeleteTgs, tgSvcUpToDate) + + mockTGManager.EXPECT().List(ctx).Return(noDeleteTgs, nil) + + mockClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, name types.NamespacedName, routeOrSvcExport client.Object, _ ...interface{}) error { + routeOrSvcExport.SetName("ignored-name") + routeOrSvcExport.SetNamespace("ignored-ns") + return nil }, - { - name: "No need to delete SDK TargetGroup since it is referenced by a HTTPRoutes) ", - sdkTargetGroups: []sdkTGDef{ - {name: "sdkTG1", id: "sdkTG1-id", serviceNetworkManagerErr: nil, - isSameVPC: true, - hasTags: true, hasServiceExportTypeTag: false, - hasHTTPRouteTypeTag: true, HTTPRouteExist: true, refedByHTTPRoute: true, - hasServiceArns: true, - expectDelete: false}, + ).AnyTimes() + + baseModelTg := model.TargetGroup{ + Spec: model.TargetGroupSpec{ + VpcId: "vpc-id", + Type: "IP", + Port: 80, + Protocol: "HTTP", + ProtocolVersion: "HTTP1", + IpAddressType: "IPV4", + TargetGroupTagFields: model.TargetGroupTagFields{ + EKSClusterName: "cluster-name", + K8SServiceName: "svc", + K8SServiceNamespace: "ns", }, - wantSynthesizerError: nil, - wantDataStoreError: nil, - wantDataStoreStatus: "", - }, - { - name: "No need to delete SDK TargetGroup , no K8S tags ", - sdkTargetGroups: []sdkTGDef{ - {name: "sdkTG1", id: "sdkTG1-id", serviceNetworkManagerErr: nil, - isSameVPC: true, - hasTags: false, hasServiceExportTypeTag: false, - hasHTTPRouteTypeTag: false, HTTPRouteExist: false, refedByHTTPRoute: false, - expectDelete: false}, - }, - wantSynthesizerError: nil, - wantDataStoreError: nil, - wantDataStoreStatus: "", - }, - { - name: "No need to delete SDK TargetGroup , different VPC", - sdkTargetGroups: []sdkTGDef{ - {name: "sdkTG1", id: "sdkTG1-id", serviceNetworkManagerErr: nil, - isSameVPC: false, - hasTags: false, hasServiceExportTypeTag: false, - hasHTTPRouteTypeTag: false, HTTPRouteExist: false, refedByHTTPRoute: false, - expectDelete: false}, - }, - wantSynthesizerError: nil, - wantDataStoreError: nil, - wantDataStoreStatus: "", }, + } + svcExportModelTg := baseModelTg + svcExportModelTg.Spec.TargetGroupTagFields.K8SParentRefType = model.ParentRefTypeSvcExport - { - name: "delete SDK TargetGroup due not referenced by any serviceexport", - sdkTargetGroups: []sdkTGDef{ - {name: "sdkTG1", id: "sdkTG1-id", serviceNetworkManagerErr: nil, - isSameVPC: true, - hasTags: true, hasServiceExportTypeTag: true, serviceExportExist: false, - hasHTTPRouteTypeTag: false, HTTPRouteExist: false, refedByHTTPRoute: false, - expectDelete: true}, - }, - wantSynthesizerError: nil, - wantDataStoreError: nil, - wantDataStoreStatus: "", - }, + mockSvcExportTgBuilder.EXPECT().BuildTargetGroup(ctx, gomock.Any()).Return(&svcExportModelTg, nil) - { - name: "no need to delete SDK TargetGroup since it is referenced by an serviceexport", - sdkTargetGroups: []sdkTGDef{ - {name: "sdkTG1", id: "sdkTG1-id", serviceNetworkManagerErr: nil, - isSameVPC: true, - hasTags: true, hasServiceExportTypeTag: true, serviceExportExist: true, - hasHTTPRouteTypeTag: false, HTTPRouteExist: false, refedByHTTPRoute: false, - expectDelete: false}, - }, - wantSynthesizerError: nil, - wantDataStoreError: nil, - wantDataStoreStatus: "", - }, - } + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) + svcModelTg := baseModelTg + svcModelTg.ResourceMeta = core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", "tg-id") + svcModelTg.Spec.TargetGroupTagFields.K8SParentRefType = model.ParentRefTypeHTTPRoute + svcModelTg.Spec.TargetGroupTagFields.K8SRouteName = "route" + svcModelTg.Spec.TargetGroupTagFields.K8SRouteNamespace = "route-ns" + stack.AddResource(&svcModelTg) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.Background() - - ds := latticestore.NewLatticeDataStore() - - mockTGManager := NewMockTargetGroupManager(c) - sdkTGReturned := []targetGroupOutput{} - mockK8sClient := mock_client.NewMockClient(c) - - if len(tt.sdkTargetGroups) > 0 { - for _, sdkTG := range tt.sdkTargetGroups { - name := sdkTG.name - id := sdkTG.id - vpc := "" - if sdkTG.isSameVPC { - vpc = config.VpcID - - } else { - vpc = config.VpcID + "other VPC" - } - - var tagsOutput *vpclattice.ListTagsForResourceOutput - var tags = make(map[string]*string) - - tags["non-k8e"] = aws.String("not-k8s-related") - - if sdkTG.hasTags == false { - - tagsOutput = nil - } else { - - tagsOutput = &vpclattice.ListTagsForResourceOutput{ - Tags: tags, - } - - } - - if sdkTG.hasServiceExportTypeTag { - tags[model.K8SParentRefTypeKey] = aws.String(model.K8SServiceExportType) - } - - if sdkTG.hasHTTPRouteTypeTag { - tags[model.K8SParentRefTypeKey] = aws.String(model.K8SHTTPRouteType) - } - - tags[model.K8SServiceNameKey] = aws.String(srvname) - tags[model.K8SServiceNamespaceKey] = aws.String(srvnamespace) - tags[model.K8SHTTPRouteNameKey] = aws.String(routename) - tags[model.K8SHTTPRouteNamespaceKey] = aws.String(routenamespace) - - serviceArns := []*string{} - if sdkTG.hasServiceArns { - serviceArns = append(serviceArns, aws.String("dummy")) - } - - sdkTGReturned = append(sdkTGReturned, - targetGroupOutput{ - getTargetGroupOutput: vpclattice.GetTargetGroupOutput{ - Name: &name, - Id: &id, - Arn: aws.String("tg-ARN"), - Config: &vpclattice.TargetGroupConfig{ - VpcIdentifier: &vpc, - ProtocolVersion: aws.String(vpclattice.TargetGroupProtocolVersionHttp1), - }, - ServiceArns: serviceArns, - }, - targetGroupTags: tagsOutput, - }, - ) - - fmt.Printf("sdkTGReturned :%v \n", sdkTGReturned[0].targetGroupTags) - - tgSpec := model.TargetGroup{ - Spec: model.TargetGroupSpec{ - Name: sdkTG.name, - LatticeID: sdkTG.id, - }, - } - if sdkTG.hasHTTPRouteTypeTag { - tgSpec.Spec.Config.K8SHTTPRouteName = routename - } - - if sdkTG.HTTPRouteExist { - - if sdkTG.refedByHTTPRoute { - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, name types.NamespacedName, httpRoute *gwv1beta1.HTTPRoute, arg3 ...interface{}) error { - httpRoute.Name = routename - httpRoute.Namespace = routenamespace - backendNamespace := gwv1beta1.Namespace(routenamespace) - - httpRoute.Spec.Rules = []gwv1beta1.HTTPRouteRule{ - { - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: gwv1beta1.BackendRef{ - BackendObjectReference: gwv1beta1.BackendObjectReference{ - Kind: kindPtr("Service"), - Name: gwv1beta1.ObjectName(srvname), - Namespace: &backendNamespace, - }, - }, - }, - }, - }, - } - - return nil - }, - ) - - } else { - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, name types.NamespacedName, httpRoute *gwv1beta1.HTTPRoute, arg3 ...interface{}) error { - httpRoute.Name = routename - httpRoute.Namespace = routenamespace - return nil - - }, - ) - - } - } else { - if sdkTG.hasHTTPRouteTypeTag { - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, name types.NamespacedName, httpRoute *gwv1beta1.HTTPRoute, arg3 ...interface{}) error { - - return errors.New("no httproute") - - }, - ) - } - } - - if sdkTG.hasServiceExportTypeTag { - if sdkTG.serviceExportExist { - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, name types.NamespacedName, svcexport *mcsv1alpha1.ServiceExport, arg3 ...interface{}) error { - - return nil - - }, - ) - - } else { - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, name types.NamespacedName, svcexport *mcsv1alpha1.ServiceExport, arg3 ...interface{}) error { - - return errors.New("no serviceexport") - - }, - ) - - } - } - - if sdkTG.expectDelete { - mockTGManager.EXPECT().Delete(ctx, &tgSpec).Return(sdkTG.serviceNetworkManagerErr) - } - - } - } - fmt.Printf("sdkTGReturnd %v len %v\n", sdkTGReturned, len(sdkTGReturned)) - - mockTGManager.EXPECT().List(ctx).Return(sdkTGReturned, nil) - - tgSynthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, mockK8sClient, mockTGManager, nil, ds) - err := tgSynthesizer.SynthesizeSDKTargetGroups(ctx) - - assert.Equal(t, tt.wantSynthesizerError, err) - }) - } -} + mockTGManager.EXPECT().IsTargetGroupMatch(ctx, gomock.Any(), gomock.Any(), gomock.Any()).Return(true, nil) -type svcDef struct { - name string - tgARN string - tgID string - mgrErr bool -} + mockSvcBuilder.EXPECT().Build(ctx, gomock.Any()).Return(stack, nil) -func Test_SynthesizeTriggeredService(t *testing.T) { - tests := []struct { - name string - svcList []svcDef - isDeleted bool - wantErrIsNil bool - }{ - { - name: "service triggered target group", - svcList: []svcDef{ - { - name: "service11", - tgARN: "service11-arn", - tgID: "service11-ID", - mgrErr: false, - }, - { - name: "service12", - tgARN: "service12-arn", - tgID: "service12-ID", - mgrErr: false, - }, - }, - isDeleted: false, - wantErrIsNil: true, - }, - { - name: "service triggered target group", - svcList: []svcDef{ - { - name: "service21", - tgARN: "service21-arn", - tgID: "service21-ID", - mgrErr: true, - }, - { - name: "service22", - tgARN: "service22-arn", - tgID: "service22-ID", - mgrErr: false, - }, - }, - isDeleted: false, - wantErrIsNil: false, - }, - { - name: "service triggered target group", - svcList: []svcDef{ - { - name: "service31", - tgARN: "service31-arn", - tgID: "service31-ID", - mgrErr: true, - }, - { - name: "service32", - tgARN: "service32-arn", - tgID: "service32-ID", - mgrErr: false, - }, - }, - isDeleted: true, - wantErrIsNil: false, - }, - } + synthesizer := NewTargetGroupSynthesizer( + gwlog.FallbackLogger, nil, mockClient, mockTGManager, mockSvcExportTgBuilder, mockSvcBuilder, stack) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockTGManager := NewMockTargetGroupManager(c) - - ds := latticestore.NewLatticeDataStore() - - stack := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - for _, svc := range tt.svcList { - tgSpec := model.TargetGroupSpec{ - Name: svc.name, - Type: model.TargetGroupTypeIP, - Config: model.TargetGroupConfig{ - IsServiceImport: false, - }, - IsDeleted: tt.isDeleted, - } - - tg := model.NewTargetGroup(stack, svc.name, tgSpec) - fmt.Printf("tg : %v\n", tg) - - if !tt.isDeleted { - - if svc.mgrErr { - mockTGManager.EXPECT().Create(ctx, tg).Return(model.TargetGroupStatus{}, errors.New("tgmgr err")) - } else { - mockTGManager.EXPECT().Create(ctx, tg).Return(model.TargetGroupStatus{TargetGroupARN: svc.tgARN, TargetGroupID: svc.tgID}, nil) - } - } else { - if svc.mgrErr { - mockTGManager.EXPECT().Delete(ctx, tg).Return(errors.New("tgmgr err")) - } else { - mockTGManager.EXPECT().Delete(ctx, tg).Return(nil) - } - } - } - - synthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, stack, ds) - err := synthesizer.SynthesizeTriggeredTargetGroup(ctx) - fmt.Printf("err:%v \n", err) - - if tt.wantErrIsNil { - assert.Nil(t, err) - } else { - assert.NotNil(t, err) - } - - if !tt.isDeleted { - // check datastore - for _, tg := range tt.svcList { - if tg.mgrErr { - //TODO, test routename - _, err := ds.GetTargetGroup(tg.name, "", false) - assert.NotNil(t, err) - - } else { - //TODO, test routename - dsTG, err := ds.GetTargetGroup(tg.name, "", false) - assert.Nil(t, err) - assert.Equal(t, tg.tgARN, dsTG.ARN) - assert.Equal(t, tg.tgID, dsTG.ID) - - } - } - } - }) - } + err := synthesizer.SynthesizeUnusedDelete(ctx) + assert.Nil(t, err) } -func Test_SynthesizeTriggeredTargetGroupsCreation_TriggeredByServiceExport(t *testing.T) { - now := metav1.Now() - tests := []struct { - name string - svcExport *mcsv1alpha1.ServiceExport - tgManagerError bool - wantErrIsNil bool - }{ - { - name: "Creating ServiceExport request trigger targetgroup creation, ok case", - svcExport: &mcsv1alpha1.ServiceExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: "export1", - Namespace: "ns1", - }, +func Test_DeleteServiceExport_DeleteCases(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockTGManager := NewMockTargetGroupManager(c) + mockClient := mock_client.NewMockClient(c) + mockSvcExportTgBuilder := gateway.NewMockSvcExportTargetGroupModelBuilder(c) + + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" + + baseTg := getBaseTg() + + var deleteTgs []tgListOutput + tgSvcExport := copy(baseTg) + tgSvcExport.getTargetGroupOutput.Arn = aws.String("tg-svc-export-arn") + tgSvcExport.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeSvcExport)) + deleteTgs = append(deleteTgs, tgSvcExport) + + t.Run("Service Export does not exist", func(t *testing.T) { + mockTGManager.EXPECT().List(ctx).Return(deleteTgs, nil) + + // the important bit below - svc export get returns does-not-exist + mockClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return( + &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Code: http.StatusNotFound, + Reason: metav1.StatusReasonNotFound, + }, + }) + mockTGManager.EXPECT().Delete(ctx, gomock.Any()).Return(nil) + + synthesizer := NewTargetGroupSynthesizer( + gwlog.FallbackLogger, nil, mockClient, mockTGManager, mockSvcExportTgBuilder, nil, nil) + + err := synthesizer.SynthesizeUnusedDelete(ctx) + assert.Nil(t, err) + }) + + t.Run("Service Export deleted", func(t *testing.T) { + mockTGManager.EXPECT().List(ctx).Return(deleteTgs, nil) + + // the important bit below - svc export get returns deleted + mockClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, name types.NamespacedName, svcExport client.Object, _ ...interface{}) error { + now := metav1.Now() + svcExport.SetName("svc") + svcExport.SetNamespace("ns") + svcExport.SetDeletionTimestamp(&now) + return nil }, - tgManagerError: false, - wantErrIsNil: true, - }, - { - name: "Creating ServiceExport request trigger targetgroup creation, not ok case", - svcExport: &mcsv1alpha1.ServiceExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: "export2", - Namespace: "ns1", + ) + mockTGManager.EXPECT().Delete(ctx, gomock.Any()).Return(nil) + + synthesizer := NewTargetGroupSynthesizer( + gwlog.FallbackLogger, nil, mockClient, mockTGManager, mockSvcExportTgBuilder, nil, nil) + + err := synthesizer.SynthesizeUnusedDelete(ctx) + assert.Nil(t, err) + }) + + t.Run("Service Export model differs", func(t *testing.T) { + modelTg := model.TargetGroup{ + Spec: model.TargetGroupSpec{ + VpcId: "vpc-id", + Type: "IP", + Port: 8080, // <-- important bit, port has changed + Protocol: "HTTP", + ProtocolVersion: "HTTP1", + IpAddressType: "IPV4", + TargetGroupTagFields: model.TargetGroupTagFields{ + EKSClusterName: "cluster-name", + K8SServiceName: "svc", + K8SServiceNamespace: "ns", + K8SParentRefType: model.ParentRefTypeSvcExport, }, }, - tgManagerError: true, - wantErrIsNil: false, - }, - { - name: "SynthesizeTriggeredTargetGroupsCreation() should ignore any targetgroup deletion request", - svcExport: &mcsv1alpha1.ServiceExport{ - ObjectMeta: metav1.ObjectMeta{ - Name: "export3", - Namespace: "ns1", - Finalizers: []string{"gateway.k8s.aws/resources"}, - DeletionTimestamp: &now, - }, + } + + mockClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, name types.NamespacedName, svcExport client.Object, _ ...interface{}) error { + svcExport.SetName("svc") + svcExport.SetNamespace("ns") + return nil }, - tgManagerError: false, - wantErrIsNil: true, - }, - } + ) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - k8sSchema := runtime.NewScheme() - clientgoscheme.AddToScheme(k8sSchema) - v1alpha1.AddToScheme(k8sSchema) - k8sClient := testclient.NewFakeClientWithScheme(k8sSchema) - - svc := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: tt.svcExport.Name, - Namespace: tt.svcExport.Namespace, - }, - Spec: corev1.ServiceSpec{ - Ports: []corev1.ServicePort{ - {}, - }, - IPFamilies: []corev1.IPFamily{ - corev1.IPv4Protocol, - }, - }, - } - - k8sClient.Create(ctx, svc.DeepCopy()) - - mockTGManager := NewMockTargetGroupManager(c) - - ds := latticestore.NewLatticeDataStore() - if !tt.svcExport.DeletionTimestamp.IsZero() { - // When test serviceExport deletion, we expect latticeDataStore already had this tg entry - tgName := latticestore.TargetGroupName(tt.svcExport.Name, tt.svcExport.Namespace) - ds.AddTargetGroup(tgName, "vpc-123456789", "arn123", "4567", false, "") - } - builder := gateway.NewSvcExportTargetGroupBuilder(gwlog.FallbackLogger, k8sClient, ds, nil) - - stack, tg, err := builder.Build(ctx, tt.svcExport) - assert.Nil(t, err) - - synthersizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, stack, ds) - - tgStatus := model.TargetGroupStatus{ - TargetGroupARN: "arn123", - TargetGroupID: "4567", - } - - if !tg.Spec.IsDeleted { - if tt.tgManagerError { - mockTGManager.EXPECT().Create(ctx, tg).Return(tgStatus, errors.New("ERROR")) - } else { - mockTGManager.EXPECT().Create(ctx, tg).Return(tgStatus, nil) - } - } - err = synthersizer.SynthesizeTriggeredTargetGroupsCreation(ctx) - if !tt.wantErrIsNil { - assert.NotNil(t, err) - return - } - - assert.Nil(t, err) - tgName := latticestore.TargetGroupName(tt.svcExport.Name, tt.svcExport.Namespace) - dsTG, err := ds.GetTargetGroup(tgName, "", false) - assert.Nil(t, err) - if !tg.Spec.IsDeleted { - assert.Equal(t, dsTG.ARN, tgStatus.TargetGroupARN) - assert.Equal(t, dsTG.ID, tgStatus.TargetGroupID) - } - }) - } -} + mockSvcExportTgBuilder.EXPECT().BuildTargetGroup(ctx, gomock.Any()).Return(&modelTg, nil) -func Test_SynthesizeTriggeredTargetGroupsDeletion_TriggeredByServiceImport(t *testing.T) { - tests := []struct { - name string - svcImportList []svcImportDef - }{ - - { - name: "Ignore all target group deletion request triggered by httproute deletion with backendref service import", - svcImportList: []svcImportDef{ - { - name: "service-import31", - tgExist: false, - mgrErr: true, - }, - { - name: "service-import32", - tgARN: "service-import32-arn", - tgID: "service-import32-ID", - tgExist: true, - mgrErr: false, - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockTGManager := NewMockTargetGroupManager(c) - ds := latticestore.NewLatticeDataStore() - - for _, svcImport := range tt.svcImportList { - if svcImport.tgExist { - ds.AddTargetGroup(svcImport.name, "my-vpc", svcImport.tgARN, svcImport.tgID, true, "") - } - } - stack := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - synthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, stack, ds) - - err := synthesizer.SynthesizeTriggeredTargetGroupsDeletion(ctx) - assert.Nil(t, err) - - for _, svcImport := range tt.svcImportList { - if svcImport.tgExist { - // synthesizer.SynthesizeTriggeredTargetGroupsDeletion() should ignore all serviceImport triggered TG deletion, - // so the previouly existed TG should still exist in datastore - tgDataStore, err := ds.GetTargetGroup(svcImport.name, "", true) - assert.Nil(t, err) - assert.Equal(t, svcImport.tgID, tgDataStore.ID) - assert.Equal(t, svcImport.tgARN, tgDataStore.ARN) - } - } - }) - } -} + mockTGManager.EXPECT().List(ctx).Return(deleteTgs, nil) + mockTGManager.EXPECT().Delete(ctx, gomock.Any()).Return(nil) -func Test_SynthesizeTriggeredTargetGroupsCreation_TriggeredByK8sService(t *testing.T) { - tests := []struct { - name string - svcList []svcDef - isDeleted bool - wantErrIsNil bool - }{ - { - name: "httproute creation request with backendref k8sService triggered target group creation, ok case", - svcList: []svcDef{ - { - name: "service11", - tgARN: "service11-arn", - tgID: "service11-ID", - mgrErr: false, - }, - { - name: "service12", - tgARN: "service12-arn", - tgID: "service12-ID", - mgrErr: false, - }, - }, - isDeleted: false, - wantErrIsNil: true, - }, - { - name: "httproute creation request with backendref k8sService triggered target group creation, mgrErr", - svcList: []svcDef{ - { - name: "service21", - tgARN: "service21-arn", - tgID: "service21-ID", - mgrErr: true, - }, - { - name: "service22", - tgARN: "service22-arn", - tgID: "service22-ID", - mgrErr: false, - }, - }, - isDeleted: false, - wantErrIsNil: false, - }, - { - name: "SynthesizeTriggeredTargetGroupsCreation should ignore any target group deletion request", - svcList: []svcDef{ - { - name: "service31", - tgARN: "service31-arn", - tgID: "service31-ID", - mgrErr: false, - }, - { - name: "service32", - tgARN: "service32-arn", - tgID: "service32-ID", - mgrErr: false, - }, - }, - isDeleted: true, - wantErrIsNil: true, - }, - } + synthesizer := NewTargetGroupSynthesizer( + gwlog.FallbackLogger, nil, mockClient, mockTGManager, mockSvcExportTgBuilder, nil, nil) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockTGManager := NewMockTargetGroupManager(c) - - ds := latticestore.NewLatticeDataStore() - - stack := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - for _, svc := range tt.svcList { - tgSpec := model.TargetGroupSpec{ - Name: svc.name, - Type: model.TargetGroupTypeIP, - Config: model.TargetGroupConfig{ - IsServiceImport: false, - }, - IsDeleted: tt.isDeleted, - } - - tg := model.NewTargetGroup(stack, svc.name, tgSpec) - fmt.Printf("tg : %v\n", tg) - - if !tt.isDeleted { - - if svc.mgrErr { - mockTGManager.EXPECT().Create(ctx, tg).Return(model.TargetGroupStatus{}, errors.New("tgmgr err")) - } else { - mockTGManager.EXPECT().Create(ctx, tg).Return(model.TargetGroupStatus{TargetGroupARN: svc.tgARN, TargetGroupID: svc.tgID}, nil) - } - } - } - - synthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, stack, ds) - var err error - - err = synthesizer.SynthesizeTriggeredTargetGroupsCreation(ctx) - - fmt.Printf("err:%v \n", err) - - if tt.wantErrIsNil { - assert.Nil(t, err) - } else { - assert.NotNil(t, err) - } - - if !tt.isDeleted { - // check datastore - for _, tg := range tt.svcList { - if tg.mgrErr { - //TODO, test routename - _, err := ds.GetTargetGroup(tg.name, "", false) - assert.NotNil(t, err) - - } else { - //TODO, test routename - dsTG, err := ds.GetTargetGroup(tg.name, "", false) - assert.Nil(t, err) - assert.Equal(t, tg.tgARN, dsTG.ARN) - assert.Equal(t, tg.tgID, dsTG.ID) - - } - } - } - }) - } + err := synthesizer.SynthesizeUnusedDelete(ctx) + assert.Nil(t, err) + }) } -func Test_SynthesizeTriggeredTargetGroupsDeletion_TriggeredByK8sService(t *testing.T) { - tests := []struct { - name string - svcList []svcDef - isDeleted bool - wantErrIsNil bool - }{ - { - name: "httproute with backendref k8sService deletion request triggering target group deletion, ok case", - svcList: []svcDef{ - { - name: "service11", - tgARN: "service11-arn", - tgID: "service11-ID", - mgrErr: false, - }, - { - name: "service12", - tgARN: "service12-arn", - tgID: "service12-ID", - mgrErr: false, - }, +func Test_DeleteRoute_DeleteCases(t *testing.T) { + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() + mockTGManager := NewMockTargetGroupManager(c) + mockClient := mock_client.NewMockClient(c) + mockSvcBuilder := gateway.NewMockLatticeServiceBuilder(c) + + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" + + baseTg := getBaseTg() + + var deleteTgs []tgListOutput + tgSvc := copy(baseTg) + tgSvc.getTargetGroupOutput.Arn = aws.String("tg-svc-arn") + tgSvc.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeHTTPRoute)) + tgSvc.targetGroupTags.Tags[model.K8SRouteNameKey] = aws.String("route") + tgSvc.targetGroupTags.Tags[model.K8SRouteNamespaceKey] = aws.String("route-ns") + deleteTgs = append(deleteTgs, tgSvc) + + t.Run("Route does not exist", func(t *testing.T) { + mockTGManager.EXPECT().List(ctx).Return(deleteTgs, nil) + + // the important bit below - svc export get returns does-not-exist + mockClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return( + &apierrors.StatusError{ + ErrStatus: metav1.Status{ + Code: http.StatusNotFound, + Reason: metav1.StatusReasonNotFound, + }, + }) + mockTGManager.EXPECT().Delete(ctx, gomock.Any()).Return(nil) + + synthesizer := NewTargetGroupSynthesizer( + gwlog.FallbackLogger, nil, mockClient, mockTGManager, nil, mockSvcBuilder, nil) + + err := synthesizer.SynthesizeUnusedDelete(ctx) + assert.Nil(t, err) + }) + + t.Run("Route deleted", func(t *testing.T) { + mockTGManager.EXPECT().List(ctx).Return(deleteTgs, nil) + + // the important bit below - svc export get returns deleted + mockClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, name types.NamespacedName, route client.Object, _ ...interface{}) error { + now := metav1.Now() + route.SetName("route-name") + route.SetNamespace("route-ns") + route.SetDeletionTimestamp(&now) + return nil }, - isDeleted: true, - wantErrIsNil: true, - }, - { - name: "httproute with backendref k8sService deletion request triggering target group deletion, mgrErr", - svcList: []svcDef{ - { - name: "service21", - tgARN: "service21-arn", - tgID: "service21-ID", - mgrErr: true, - }, - { - name: "service22", - tgARN: "service22-arn", - tgID: "service22-ID", - mgrErr: false, + ) + mockTGManager.EXPECT().Delete(ctx, gomock.Any()).Return(nil) + + synthesizer := NewTargetGroupSynthesizer( + gwlog.FallbackLogger, nil, mockClient, mockTGManager, nil, mockSvcBuilder, nil) + + err := synthesizer.SynthesizeUnusedDelete(ctx) + assert.Nil(t, err) + }) + + t.Run("Route model differs", func(t *testing.T) { + // current logic requires we return at least one tg + // but match logic is deferred to tgManager.IsTargetGroupMatch + svcModelTg := model.TargetGroup{ + Spec: model.TargetGroupSpec{ + VpcId: "vpc-id", + Type: "IP", + Port: 8080, + Protocol: "HTTP", + ProtocolVersion: "HTTP1", + IpAddressType: "IPV4", + TargetGroupTagFields: model.TargetGroupTagFields{ + EKSClusterName: "cluster-name", + K8SServiceName: "svc", + K8SServiceNamespace: "ns", + K8SRouteName: "route-name", + K8SRouteNamespace: "route-ns", + K8SParentRefType: model.ParentRefTypeSvcExport, }, }, - isDeleted: true, - wantErrIsNil: false, - }, - { - name: "SynthesizeTriggeredTargetGroupsDeletion should ignore target group creation request", - svcList: []svcDef{ - { - name: "service31", - tgARN: "service31-arn", - tgID: "service31-ID", - mgrErr: true, - }, - { - name: "service32", - tgARN: "service32-arn", - tgID: "service32-ID", - mgrErr: false, - }, + } + + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) + svcModelTg.ResourceMeta = core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", "tg-id") + stack.AddResource(&svcModelTg) + + mockSvcBuilder.EXPECT().Build(ctx, gomock.Any()).Return(stack, nil) + + mockClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( + func(ctx context.Context, name types.NamespacedName, svcOrSvcExport client.Object, _ ...interface{}) error { + svcOrSvcExport.SetName("svc") + svcOrSvcExport.SetNamespace("ns") + return nil }, - isDeleted: false, - wantErrIsNil: true, - }, - } + ) - httpRouteName := "my-http-route" - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockTGManager := NewMockTargetGroupManager(c) - - ds := latticestore.NewLatticeDataStore() - - stack := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - for _, svc := range tt.svcList { - tgSpec := model.TargetGroupSpec{ - Name: svc.name, - Type: model.TargetGroupTypeIP, - Config: model.TargetGroupConfig{ - K8SHTTPRouteName: httpRouteName, - IsServiceImport: false, - }, - IsDeleted: tt.isDeleted, - } - - tg := model.NewTargetGroup(stack, svc.name, tgSpec) - fmt.Printf("tg : %v\n", tg) - - if tt.isDeleted { - ds.AddTargetGroup(tg.Spec.Name, tg.Spec.Config.VpcID, svc.tgARN, svc.tgID, tg.Spec.Config.IsServiceImport, httpRouteName) - - if svc.mgrErr { - mockTGManager.EXPECT().Delete(ctx, tg).Return(errors.New("tgmgr err")) - } else { - mockTGManager.EXPECT().Delete(ctx, tg).Return(nil) - } - } - } - - synthesizer := NewTargetGroupSynthesizer(gwlog.FallbackLogger, nil, nil, mockTGManager, stack, ds) - var err error - err = synthesizer.SynthesizeTriggeredTargetGroupsDeletion(ctx) - if tt.wantErrIsNil { - assert.Nil(t, err) - } else { - assert.NotNil(t, err) - } - fmt.Printf("err:%v \n", err) - - for _, svc := range tt.svcList { - if tt.isDeleted { - tgDateStore, err := ds.GetTargetGroup(svc.name, httpRouteName, false) - - if svc.mgrErr { - assert.Nil(t, err, "targetGroup should still exist since targetGroupManager.Delete return error") - assert.Equal(t, svc.tgID, tgDateStore.ID) - assert.Equal(t, svc.tgID, tgDateStore.ID) - - } else { - _, err := ds.GetTargetGroup(svc.name, httpRouteName, false) - assert.Equal(t, err, errors.New(latticestore.DATASTORE_TG_NOT_EXIST), - "targetGroup %v should be deleted in the datastore if targetGroupManager.Delete() success", svc.name) - - } - } - } - }) - } + mockTGManager.EXPECT().List(ctx).Return(deleteTgs, nil) + // important bit below, return false for match + // this is actually what decides if the tgs are a match or not + mockTGManager.EXPECT().IsTargetGroupMatch(ctx, gomock.Any(), gomock.Any(), gomock.Any()). + Return(false, nil) + mockTGManager.EXPECT().Delete(ctx, gomock.Any()).Return(nil) + + synthesizer := NewTargetGroupSynthesizer( + gwlog.FallbackLogger, nil, mockClient, mockTGManager, nil, mockSvcBuilder, nil) + + err := synthesizer.SynthesizeUnusedDelete(ctx) + assert.Nil(t, err) + }) } + +// TODO: Error cases should not delete diff --git a/pkg/deploy/lattice/targets_manager.go b/pkg/deploy/lattice/targets_manager.go index 8dddf07e..6a0159df 100644 --- a/pkg/deploy/lattice/targets_manager.go +++ b/pkg/deploy/lattice/targets_manager.go @@ -3,125 +3,130 @@ package lattice import ( "context" "errors" - + "fmt" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/vpclattice" pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" ) type TargetsManager interface { - Create(ctx context.Context, targets *model.Targets) error + Update(ctx context.Context, modelTargets *model.Targets, modelTg *model.TargetGroup) error } type defaultTargetsManager struct { - log gwlog.Logger - cloud pkg_aws.Cloud - datastore *latticestore.LatticeDataStore + log gwlog.Logger + cloud pkg_aws.Cloud } func NewTargetsManager( log gwlog.Logger, cloud pkg_aws.Cloud, - datastore *latticestore.LatticeDataStore, ) *defaultTargetsManager { return &defaultTargetsManager{ - log: log, - cloud: cloud, - datastore: datastore, + log: log, + cloud: cloud, } } -// Create will try to register targets to the target group -// return Retry when: -// -// Target group does not exist -// nonempty unsuccessfully registered targets list -// otherwise: -// nil -func (s *defaultTargetsManager) Create(ctx context.Context, targets *model.Targets) error { - s.log.Debugf("Creating targets for target group %s-%s", targets.Spec.Name, targets.Spec.Namespace) - - // Need to find TargetGroup ID from datastore - tgName := latticestore.TargetGroupName(targets.Spec.Name, targets.Spec.Namespace) - tg, err := s.datastore.GetTargetGroup(tgName, targets.Spec.RouteName, false) // isServiceImport=false - if err != nil { - s.log.Debugf("Failed to Create targets, service %s-%s was not found, will retry later", - targets.Spec.Name, targets.Spec.Namespace) - return errors.New(LATTICE_RETRY) +func (s *defaultTargetsManager) Update(ctx context.Context, modelTargets *model.Targets, modelTg *model.TargetGroup) error { + if modelTg.Status == nil || modelTg.Status.Id == "" { + return errors.New("model target group is missing id") } + if modelTargets.Spec.StackTargetGroupId != modelTg.ID() { + return fmt.Errorf("target group ID %s does not match target reference ID %s", + modelTg.ID(), modelTargets.Spec.StackTargetGroupId) + } + + s.log.Debugf("Creating targets for target group %s", modelTg.Status.Id) - vpcLatticeSess := s.cloud.Lattice() + lattice := s.cloud.Lattice() listTargetsInput := vpclattice.ListTargetsInput{ - TargetGroupIdentifier: &tg.ID, + TargetGroupIdentifier: &modelTg.Status.Id, } - var delTargetsList []*vpclattice.Target - listTargetsOutput, err := vpcLatticeSess.ListTargetsAsList(ctx, &listTargetsInput) + listTargetsOutput, err := lattice.ListTargetsAsList(ctx, &listTargetsInput) if err != nil { return err } - for _, sdkT := range listTargetsOutput { - // check if sdkT is in input target list - isStale := true - for _, t := range targets.Spec.TargetIPList { - if (aws.StringValue(sdkT.Id) == t.TargetIP) && (aws.Int64Value(sdkT.Port) == t.Port) { - isStale = false - break - } - } - if isStale { - delTargetsList = append(delTargetsList, &vpclattice.Target{Id: sdkT.Id, Port: sdkT.Port}) - } - } - - if len(delTargetsList) > 0 { - deRegisterTargetsInput := vpclattice.DeregisterTargetsInput{ - TargetGroupIdentifier: &tg.ID, - Targets: delTargetsList, - } - _, err := vpcLatticeSess.DeregisterTargetsWithContext(ctx, &deRegisterTargetsInput) - if err != nil { - s.log.Errorf("Deregistering targets for target group %s failed due to %s", tg.ID, err) - } - } + s.deregisterStaleTargets(ctx, modelTargets, modelTg, listTargetsOutput) + return s.registerTargets(ctx, modelTargets, modelTg) +} - // TODO following should be done at model level - var targetList []*vpclattice.Target - for _, target := range targets.Spec.TargetIPList { - port := target.Port - targetIP := target.TargetIP +func (s *defaultTargetsManager) registerTargets( + ctx context.Context, + modelTargets *model.Targets, + modelTg *model.TargetGroup, +) error { + var latticeTargets []*vpclattice.Target + for _, modelTarget := range modelTargets.Spec.TargetList { + port := modelTarget.Port + targetIP := modelTarget.TargetIP t := vpclattice.Target{ Id: &targetIP, Port: &port, } - targetList = append(targetList, &t) + latticeTargets = append(latticeTargets, &t) } // No targets to register - if len(targetList) == 0 { + if len(latticeTargets) == 0 { return nil } registerRouteInput := vpclattice.RegisterTargetsInput{ - TargetGroupIdentifier: &tg.ID, - Targets: targetList, + TargetGroupIdentifier: &modelTg.Status.Id, + Targets: latticeTargets, } - resp, err := vpcLatticeSess.RegisterTargetsWithContext(ctx, ®isterRouteInput) + resp, err := s.cloud.Lattice().RegisterTargetsWithContext(ctx, ®isterRouteInput) if err != nil { - return err + return fmt.Errorf("Failed RegisterTargets %s due to %s", modelTg.Status.Id, err) } - isTargetRegisteredUnsuccessful := len(resp.Unsuccessful) > 0 - if isTargetRegisteredUnsuccessful { - s.log.Debugf("Failed to register targets for target group %s, will retry later", tg.ID) + if len(resp.Unsuccessful) > 0 { + s.log.Infof("Failed RegisterTargets (Unsuccessful=%d) %s, will retry", + len(resp.Unsuccessful), modelTg.Status.Id) return errors.New(LATTICE_RETRY) } - s.log.Debugf("Successfully registered targets for target group %s", tg.ID) + s.log.Infof("Success RegisterTargets %d, %s", len(resp.Successful), modelTg.Status.Id) return nil } + +func (s *defaultTargetsManager) deregisterStaleTargets( + ctx context.Context, + modelTargets *model.Targets, + modelTg *model.TargetGroup, + listTargetsOutput []*vpclattice.TargetSummary, +) { + var targetsToDeregister []*vpclattice.Target + for _, latticeTarget := range listTargetsOutput { + isStale := true + for _, t := range modelTargets.Spec.TargetList { + if (aws.StringValue(latticeTarget.Id) == t.TargetIP) && (aws.Int64Value(latticeTarget.Port) == t.Port) { + isStale = false + break + } + } + + if isStale { + targetsToDeregister = append(targetsToDeregister, &vpclattice.Target{Id: latticeTarget.Id, Port: latticeTarget.Port}) + } + } + + if len(targetsToDeregister) > 0 { + deregisterTargetsInput := vpclattice.DeregisterTargetsInput{ + TargetGroupIdentifier: &modelTg.Status.Id, + Targets: targetsToDeregister, + } + _, err := s.cloud.Lattice().DeregisterTargetsWithContext(ctx, &deregisterTargetsInput) + if err != nil { + s.log.Infof("Failed DeregisterTargets %s due to %s", modelTg.Status.Id, err) + } else { + s.log.Infof("Success DeregisterTargets %s", modelTg.Status.Id) + } + } +} diff --git a/pkg/deploy/lattice/targets_manager_mock.go b/pkg/deploy/lattice/targets_manager_mock.go index cbdba262..2e26310a 100644 --- a/pkg/deploy/lattice/targets_manager_mock.go +++ b/pkg/deploy/lattice/targets_manager_mock.go @@ -35,16 +35,16 @@ func (m *MockTargetsManager) EXPECT() *MockTargetsManagerMockRecorder { return m.recorder } -// Create mocks base method. -func (m *MockTargetsManager) Create(ctx context.Context, targets *lattice.Targets) error { +// Update mocks base method. +func (m *MockTargetsManager) Update(ctx context.Context, modelTargets *lattice.Targets, modelTg *lattice.TargetGroup) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", ctx, targets) + ret := m.ctrl.Call(m, "Update", ctx, modelTargets, modelTg) ret0, _ := ret[0].(error) return ret0 } -// Create indicates an expected call of Create. -func (mr *MockTargetsManagerMockRecorder) Create(ctx, targets interface{}) *gomock.Call { +// Update indicates an expected call of Update. +func (mr *MockTargetsManagerMockRecorder) Update(ctx, modelTargets, modelTg interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockTargetsManager)(nil).Create), ctx, targets) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Update", reflect.TypeOf((*MockTargetsManager)(nil).Update), ctx, modelTargets, modelTg) } diff --git a/pkg/deploy/lattice/targets_manager_test.go b/pkg/deploy/lattice/targets_manager_test.go index 08017404..b5ddfec6 100644 --- a/pkg/deploy/lattice/targets_manager_test.go +++ b/pkg/deploy/lattice/targets_manager_test.go @@ -3,6 +3,7 @@ package lattice import ( "context" "errors" + "github.com/aws/aws-sdk-go/aws" "testing" "github.com/aws/aws-sdk-go/service/vpclattice" @@ -11,269 +12,276 @@ import ( mocks_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" mocks "github.com/aws/aws-application-networking-k8s/pkg/aws/services" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" ) -/* -case1: register targets Successfully -case2: target group does not exist -case3: failed register targets -case4: register targets Unsuccessfully -*/ - -func Test_RegisterTargets_RegisterSuccessfully(t *testing.T) { +func TestTargetsManager(t *testing.T) { targets := model.Target{ - TargetIP: "123.456.78", + TargetIP: "192.0.2.10", Port: int64(8080), } targetsSpec := model.TargetsSpec{ - Name: "test", - TargetGroupID: "123456789", - TargetIPList: []model.Target{targets}, + StackTargetGroupId: "tg-stack-id", + TargetList: []model.Target{targets}, + } + modelTargets := model.Targets{ + Spec: targetsSpec, } - createInput := model.Targets{ - ResourceMeta: core.ResourceMeta{}, - Spec: targetsSpec, + + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) + modelTg := model.TargetGroup{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", "tg-stack-id"), + Status: &model.TargetGroupStatus{ + Name: "tg-name", + Arn: "tg-arn", + Id: "tg-id", + }, } - id := "123456789" - ip := "123.456.78" - port := int64(8080) targetInput := &vpclattice.Target{ - Id: &ip, - Port: &port, + Id: aws.String(targets.TargetIP), + Port: aws.Int64(targets.Port), } registerTargetsInput := &vpclattice.RegisterTargetsInput{ - TargetGroupIdentifier: &id, + TargetGroupIdentifier: aws.String("tg-id"), Targets: []*vpclattice.Target{targetInput}, } - tgCreateOutput := &vpclattice.RegisterTargetsOutput{} - listTargetOutput := []*vpclattice.TargetSummary{} + registerTargetsOutput := &vpclattice.RegisterTargetsOutput{} + var emptyListTargetOutput []*vpclattice.TargetSummary - latticeDataStore := latticestore.NewLatticeDataStore() - tgName := latticestore.TargetGroupName("test", "") - //TODO routename - latticeDataStore.AddTargetGroup(tgName, "vpc-123456789", "123456789", "123456789", false, "") c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() mockCloud := mocks_aws.NewMockCloud(c) mockLattice := mocks.NewMockLattice(c) - - mockLattice.EXPECT().RegisterTargetsWithContext(ctx, registerTargetsInput).Return(tgCreateOutput, nil) - mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetOutput, nil) mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() - targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud, latticeDataStore) - err := targetsManager.Create(ctx, &createInput) + t.Run("success - no current targets", func(t *testing.T) { + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(emptyListTargetOutput, nil) + mockLattice.EXPECT().RegisterTargetsWithContext(ctx, registerTargetsInput).Return(registerTargetsOutput, nil) - assert.Nil(t, err) -} + targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err := targetsManager.Update(ctx, &modelTargets, &modelTg) -// Target group does not exist, should return Retry -func Test_RegisterTargets_TGNotExist(t *testing.T) { - targetsSpec := model.TargetsSpec{ - Name: "test", - TargetGroupID: "123456789", - } - createInput := model.Targets{ - ResourceMeta: core.ResourceMeta{}, - Spec: targetsSpec, - } + assert.Nil(t, err) + }) - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - latticeDataStore := latticestore.NewLatticeDataStore() - mockCloud := mocks_aws.NewMockCloud(c) - - targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud, latticeDataStore) - err := targetsManager.Create(ctx, &createInput) + t.Run("success - deregister targets, no target overlap", func(t *testing.T) { + existingTarget := &vpclattice.TargetSummary{ + Id: aws.String("192.0.2.250"), + Port: aws.Int64(80), + } + existingTargets := []*vpclattice.TargetSummary{existingTarget} - assert.NotNil(t, err) - assert.Equal(t, err, errors.New(LATTICE_RETRY)) -} - -// case3: api call to register target fails -func Test_RegisterTargets_Registerfailed(t *testing.T) { - sId := "123.456.7.890" - sPort := int64(80) - targetsList := &vpclattice.TargetSummary{ - Id: &sId, - Port: &sPort, - } - targetsSuccessful := &vpclattice.Target{ - Id: &sId, - Port: &sPort, - } - successful := []*vpclattice.Target{targetsSuccessful} - deRegisterTargetsOutput := &vpclattice.DeregisterTargetsOutput{ - Successful: successful, - } - - listTargetOutput := []*vpclattice.TargetSummary{targetsList} - - targetsSpec := model.TargetsSpec{ - Name: "test", - Namespace: "", - TargetGroupID: "123456789", - TargetIPList: []model.Target{ + deregisterTargets := []*vpclattice.Target{ { - TargetIP: "123.456.7.891", - Port: sPort, + Id: existingTarget.Id, + Port: existingTarget.Port, }, - }, - } - - planToRegister := model.Targets{ - ResourceMeta: core.ResourceMeta{}, - Spec: targetsSpec, - } - - registerTargetsOutput := &vpclattice.RegisterTargetsOutput{} - - latticeDataStore := latticestore.NewLatticeDataStore() - tgName := latticestore.TargetGroupName("test", "") - // routename - latticeDataStore.AddTargetGroup(tgName, "vpc-123456789", "123456789", "123456789", false, "") - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := mocks.NewMockLattice(c) - - mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetOutput, nil) - mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(deRegisterTargetsOutput, nil) - mockLattice.EXPECT().RegisterTargetsWithContext(ctx, gomock.Any()).Return(registerTargetsOutput, errors.New("Register_Targets_Failed")) - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() - - targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud, latticeDataStore) - err := targetsManager.Create(ctx, &planToRegister) - - assert.NotNil(t, err) - assert.Equal(t, err, errors.New("Register_Targets_Failed")) -} - -// case4: register targets Unsuccessfully -func Test_RegisterTargets_RegisterUnsuccessfully(t *testing.T) { - sId := "123.456.7.890" - sPort := int64(80) - targetsList := &vpclattice.TargetSummary{ - Id: &sId, - Port: &sPort, - } - targetsSuccessful := &vpclattice.Target{ - Id: &sId, - Port: &sPort, - } - successful := []*vpclattice.Target{targetsSuccessful} - deRegisterTargetsOutput := &vpclattice.DeregisterTargetsOutput{ - Successful: successful, - } - listTargetOutput := []*vpclattice.TargetSummary{targetsList} - - tgId := "123456789" - deRegisterTargetsInput := &vpclattice.DeregisterTargetsInput{ - TargetGroupIdentifier: &tgId, - Targets: []*vpclattice.Target{targetsSuccessful}, - } - - targetToRegister := model.Target{ - TargetIP: "123.456.78", - Port: int64(8080), - } - targetsSpec := model.TargetsSpec{ - Name: "test", - Namespace: "", - TargetGroupID: tgId, - TargetIPList: []model.Target{targetToRegister}, - } - planToRegister := model.Targets{ - ResourceMeta: core.ResourceMeta{}, - Spec: targetsSpec, - } - - ip := "123.456.78" - port := int64(8080) - targetToRegisterInput := &vpclattice.Target{ - Id: &ip, - Port: &port, - } - registerTargetsInput := vpclattice.RegisterTargetsInput{ - TargetGroupIdentifier: &tgId, - Targets: []*vpclattice.Target{targetToRegisterInput}, - } - - unsuccessfulId := "123.456.78" - unsuccessfulPort := int64(8080) - targetsUnsuccessful := &vpclattice.TargetFailure{ - Id: &unsuccessfulId, - Port: &unsuccessfulPort, - } - unsuccessful := []*vpclattice.TargetFailure{targetsUnsuccessful} - - registerTargetsOutput := &vpclattice.RegisterTargetsOutput{ - Unsuccessful: unsuccessful, - } - - latticeDataStore := latticestore.NewLatticeDataStore() - tgName := latticestore.TargetGroupName("test", "") - //routename - latticeDataStore.AddTargetGroup(tgName, "vpc-123456789", "123456789", "123456789", false, "") - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := mocks.NewMockLattice(c) - - mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetOutput, nil) - mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, deRegisterTargetsInput).Return(deRegisterTargetsOutput, nil) - mockLattice.EXPECT().RegisterTargetsWithContext(ctx, ®isterTargetsInput).Return(registerTargetsOutput, nil) - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() - - targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud, latticeDataStore) - err := targetsManager.Create(ctx, &planToRegister) - - assert.NotNil(t, err) - assert.Equal(t, err, errors.New(LATTICE_RETRY)) -} + } + deregisterInput := &vpclattice.DeregisterTargetsInput{ + TargetGroupIdentifier: aws.String(modelTg.Status.Id), + Targets: deregisterTargets, + } + deregisterOutput := &vpclattice.DeregisterTargetsOutput{ + Successful: deregisterTargets, + } + + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(existingTargets, nil) + mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, deregisterInput).Return(deregisterOutput, nil) + mockLattice.EXPECT().RegisterTargetsWithContext(ctx, registerTargetsInput).Return(registerTargetsOutput, nil) + + targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err := targetsManager.Update(ctx, &modelTargets, &modelTg) + + assert.Nil(t, err) + }) + + t.Run("failures", func(t *testing.T) { + // error on ListTargetsAsList + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(nil, errors.New("List_Targets_Failed")) + + targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err := targetsManager.Update(ctx, &modelTargets, &modelTg) + assert.NotNil(t, err) + + // error on RegisterTargetsWithContext + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(emptyListTargetOutput, nil) + mockLattice.EXPECT().RegisterTargetsWithContext(ctx, gomock.Any()).Return(registerTargetsOutput, errors.New("Register_Targets_Failed")) + + targetsManager = NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err = targetsManager.Update(ctx, &modelTargets, &modelTg) + assert.NotNil(t, err) + + // error on DeregisterTargetsWithContext - currently this DOES NOT cause overall failure + existingTarget := &vpclattice.TargetSummary{ + Id: aws.String("192.0.2.250"), + Port: aws.Int64(80), + } + existingTargets := []*vpclattice.TargetSummary{existingTarget} + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(existingTargets, nil) + mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, gomock.Any()).Return(nil, errors.New("Deregister_Targets_Failed")) + mockLattice.EXPECT().RegisterTargetsWithContext(ctx, gomock.Any()).Return(registerTargetsOutput, nil) + + targetsManager = NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err = targetsManager.Update(ctx, &modelTargets, &modelTg) + assert.Nil(t, err) + }) + + t.Run("basic validation", func(t *testing.T) { + targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud) + + missingStatusTg := model.TargetGroup{} + err := targetsManager.Update(ctx, &modelTargets, &missingStatusTg) + assert.NotNil(t, err) + + missingStatusTg = model.TargetGroup{Status: &model.TargetGroupStatus{}} + err = targetsManager.Update(ctx, &modelTargets, &missingStatusTg) + assert.NotNil(t, err) + + mismatchedId := "not-the-same-stack-id" + mismatchedTg := model.TargetGroup{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", mismatchedId), + Status: &model.TargetGroupStatus{Id: "tg-id"}, + } + err = targetsManager.Update(ctx, &modelTargets, &mismatchedTg) + assert.NotNil(t, err) + }) + + t.Run("register unsuccessful returns error", func(t *testing.T) { + unsuccessful := []*vpclattice.TargetFailure{ + { + Id: aws.String(targets.TargetIP), + Port: aws.Int64(targets.Port), + }, + } + unsuccessfulRTO := &vpclattice.RegisterTargetsOutput{ + Unsuccessful: unsuccessful, + } + + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(emptyListTargetOutput, nil) + mockLattice.EXPECT().RegisterTargetsWithContext(ctx, registerTargetsInput).Return(unsuccessfulRTO, nil) + + targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err := targetsManager.Update(ctx, &modelTargets, &modelTg) + + assert.NotNil(t, err) + }) + + t.Run("no targets does not register", func(t *testing.T) { + emptyTargetsSpec := model.TargetsSpec{ + StackTargetGroupId: "tg-stack-id", + TargetList: []model.Target{}, + } + emptyModelTargets := model.Targets{ + Spec: emptyTargetsSpec, + } + + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(emptyListTargetOutput, nil) + + targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err := targetsManager.Update(ctx, &emptyModelTargets, &modelTg) + + assert.Nil(t, err) + }) + + t.Run("overlapping target sets does the right thing", func(t *testing.T) { + mt1 := model.Target{ + TargetIP: "192.0.2.10", + Port: int64(8080), + } + mt2 := model.Target{ + TargetIP: "192.0.2.20", + Port: int64(8080), + } + mt3 := model.Target{ + TargetIP: "192.0.2.30", + Port: int64(8080), + } + + existingTargets := []*vpclattice.TargetSummary{ + { + Id: aws.String(mt1.TargetIP), + Port: aws.Int64(mt1.Port), + }, + { + Id: aws.String(mt2.TargetIP), + Port: aws.Int64(mt2.Port), + }, + } -func Test_RegisterTargets_NoTargets_NoCallRegisterTargets(t *testing.T) { - planToRegister := model.Targets{ - ResourceMeta: core.ResourceMeta{}, - Spec: model.TargetsSpec{ - Name: "test", - Namespace: "", - TargetGroupID: "123456789", - TargetIPList: []model.Target{}, - }, - } + newTargets := model.Targets{ + Spec: model.TargetsSpec{ + StackTargetGroupId: "tg-stack-id", + TargetList: []model.Target{mt2, mt3}, + }, + } - latticeDataStore := latticestore.NewLatticeDataStore() - tgName := latticestore.TargetGroupName("test", "") + deregisterTargets := []*vpclattice.Target{ + { + Id: aws.String(mt1.TargetIP), + Port: aws.Int64(mt1.Port), + }, + } + + deregisterInput := &vpclattice.DeregisterTargetsInput{ + TargetGroupIdentifier: aws.String(modelTg.Status.Id), + Targets: deregisterTargets, + } + deregisterOutput := &vpclattice.DeregisterTargetsOutput{ + Successful: deregisterTargets, + } + + registerInput := &vpclattice.RegisterTargetsInput{ + TargetGroupIdentifier: aws.String("tg-id"), + Targets: []*vpclattice.Target{ + {Id: aws.String(mt2.TargetIP), Port: aws.Int64(mt2.Port)}, + {Id: aws.String(mt3.TargetIP), Port: aws.Int64(mt3.Port)}, + }, + } - listTargetOutput := []*vpclattice.TargetSummary{} + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(existingTargets, nil) + mockLattice.EXPECT().RegisterTargetsWithContext(ctx, registerInput).Return(registerTargetsOutput, nil) + mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, deregisterInput).Return(deregisterOutput, nil) - // routename - latticeDataStore.AddTargetGroup(tgName, "vpc-123456789", "123456789", "123456789", false, "") + targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err := targetsManager.Update(ctx, &newTargets, &modelTg) - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := mocks.NewMockLattice(c) + assert.Nil(t, err) - mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(listTargetOutput, nil) - // Expect not to call RegisterTargets - mockLattice.EXPECT().RegisterTargetsWithContext(ctx, gomock.Any()).MaxTimes(0) - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() + }) - targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud, latticeDataStore) - err := targetsManager.Create(ctx, &planToRegister) + t.Run("port difference handled correctly", func(t *testing.T) { + existingTarget := &vpclattice.TargetSummary{ + Id: aws.String(targets.TargetIP), + Port: aws.Int64(targets.Port + 1), // <-- the important bit + } + existingTargets := []*vpclattice.TargetSummary{existingTarget} - assert.Nil(t, err) + deregisterTargets := []*vpclattice.Target{ + { + Id: existingTarget.Id, + Port: existingTarget.Port, + }, + } + deregisterInput := &vpclattice.DeregisterTargetsInput{ + TargetGroupIdentifier: aws.String(modelTg.Status.Id), + Targets: deregisterTargets, + } + deregisterOutput := &vpclattice.DeregisterTargetsOutput{ + Successful: deregisterTargets, + } + + mockLattice.EXPECT().ListTargetsAsList(ctx, gomock.Any()).Return(existingTargets, nil) + mockLattice.EXPECT().DeregisterTargetsWithContext(ctx, deregisterInput).Return(deregisterOutput, nil) + mockLattice.EXPECT().RegisterTargetsWithContext(ctx, registerTargetsInput).Return(registerTargetsOutput, nil) + + targetsManager := NewTargetsManager(gwlog.FallbackLogger, mockCloud) + err := targetsManager.Update(ctx, &modelTargets, &modelTg) + + assert.Nil(t, err) + }) } diff --git a/pkg/deploy/lattice/targets_synthesizer.go b/pkg/deploy/lattice/targets_synthesizer.go index b81782ac..9af03664 100644 --- a/pkg/deploy/lattice/targets_synthesizer.go +++ b/pkg/deploy/lattice/targets_synthesizer.go @@ -2,10 +2,10 @@ package lattice import ( "context" + "errors" "fmt" pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -16,23 +16,20 @@ func NewTargetsSynthesizer( cloud pkg_aws.Cloud, tgManager TargetsManager, stack core.Stack, - latticeDataStore *latticestore.LatticeDataStore, ) *targetsSynthesizer { return &targetsSynthesizer{ - log: log, - cloud: cloud, - targetsManager: tgManager, - stack: stack, - latticeDataStore: latticeDataStore, + log: log, + cloud: cloud, + targetsManager: tgManager, + stack: stack, } } type targetsSynthesizer struct { - log gwlog.Logger - cloud pkg_aws.Cloud - targetsManager TargetsManager - stack core.Stack - latticeDataStore *latticestore.LatticeDataStore + log gwlog.Logger + cloud pkg_aws.Cloud + targetsManager TargetsManager + stack core.Stack } func (t *targetsSynthesizer) Synthesize(ctx context.Context) error { @@ -41,37 +38,30 @@ func (t *targetsSynthesizer) Synthesize(ctx context.Context) error { if err != nil { t.log.Errorf("Failed to list targets due to %s", err) } - return t.SynthesizeTargets(ctx, resTargets) -} -func (t *targetsSynthesizer) SynthesizeTargets(ctx context.Context, resTargets []*model.Targets) error { for _, targets := range resTargets { - err := t.targetsManager.Create(ctx, targets) + resTg, err := t.stack.GetResource(targets.Spec.StackTargetGroupId, &model.TargetGroup{}) if err != nil { - return fmt.Errorf("failed to synthesize targets due to %s", err) + return err } - tgName := latticestore.TargetGroupName(targets.Spec.Name, targets.Spec.Namespace) - var targetList []latticestore.Target - for _, target := range targets.Spec.TargetIPList { - targetList = append(targetList, latticestore.Target{ - TargetIP: target.TargetIP, - TargetPort: target.Port, - }) + tg, ok := resTg.(*model.TargetGroup) + if !ok { + return errors.New("unexpected type conversion failure for target group stack object") } - err = t.latticeDataStore.UpdateTargetsForTargetGroup(tgName, targets.Spec.RouteName, targetList) + err = t.targetsManager.Update(ctx, targets, tg) if err != nil { - t.log.Errorf("Failed to update targets for target group %s due to %s", tgName, err) + identifier := model.TgNamePrefix(tg.Spec) + if tg.Status != nil && tg.Status.Id != "" { + identifier = tg.Status.Id + } + return fmt.Errorf("failed to synthesize targets %s due to %s", identifier, err) } } return nil } -func (t *targetsSynthesizer) synthesizeSDKTargets(ctx context.Context) error { - return nil -} - func (t *targetsSynthesizer) PostSynthesize(ctx context.Context) error { // nothing to do here return nil diff --git a/pkg/deploy/lattice/targets_synthesizer_test.go b/pkg/deploy/lattice/targets_synthesizer_test.go index d1b90705..b36a1842 100644 --- a/pkg/deploy/lattice/targets_synthesizer_test.go +++ b/pkg/deploy/lattice/targets_synthesizer_test.go @@ -2,121 +2,62 @@ package lattice import ( "context" - "fmt" - "testing" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - - "k8s.io/apimachinery/pkg/types" - - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "testing" ) func Test_SynthesizeTargets(t *testing.T) { - tests := []struct { - name string - srvExportName string - srvExportNamespace string - targetList []model.Target - expectedTargetList []latticestore.Target - }{ + + targetList := []model.Target{ + { + TargetIP: "10.10.1.1", + Port: 8675, + }, + { + TargetIP: "10.10.1.1", + Port: 309, + }, { - name: "Add all endpoints to build spec", - srvExportName: "export1", - srvExportNamespace: "ns1", - targetList: []model.Target{ - { - TargetIP: "10.10.1.1", - Port: 8675, - }, - { - TargetIP: "10.10.1.1", - Port: 309, - }, - { - TargetIP: "10.10.1.2", - Port: 8675, - }, - { - TargetIP: "10.10.1.2", - Port: 309, - }, - }, - expectedTargetList: []latticestore.Target{ - { - TargetIP: "10.10.1.1", - TargetPort: 8675, - }, - { - TargetIP: "10.10.1.1", - TargetPort: 309, - }, - { - TargetIP: "10.10.1.2", - TargetPort: 8675, - }, - { - TargetIP: "10.10.1.2", - TargetPort: 309, - }, - }, + TargetIP: "10.10.1.2", + Port: 8675, + }, + { + TargetIP: "10.10.1.2", + Port: 309, }, } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - ds := latticestore.NewLatticeDataStore() - - tgName := latticestore.TargetGroupName(tt.srvExportName, tt.srvExportNamespace) - // TODO routename - err := ds.AddTargetGroup(tgName, "", "", "", false, "") - assert.Nil(t, err) - ds.SetTargetGroupByServiceExport(tgName, false, true) - - mockTargetsManager := NewMockTargetsManager(c) - tgNamespacedName := types.NamespacedName{ - Namespace: tt.srvExportNamespace, - Name: tt.srvExportName, - } - - stack := core.NewDefaultStack(core.StackID(tgNamespacedName)) - - synthesizer := NewTargetsSynthesizer(gwlog.FallbackLogger, nil, mockTargetsManager, stack, ds) + c := gomock.NewController(t) + defer c.Finish() + ctx := context.TODO() - targetsSpec := model.TargetsSpec{ - Name: tt.srvExportName, - Namespace: tt.srvExportNamespace, - TargetIPList: tt.targetList, - } - modelTarget := model.Targets{ - Spec: targetsSpec, - } + mockTargetsManager := NewMockTargetsManager(c) - resTargetsList := []*model.Targets{} + stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) - resTargetsList = append(resTargetsList, &modelTarget) - - mockTargetsManager.EXPECT().Create(ctx, gomock.Any()).Return(nil) - - err = synthesizer.SynthesizeTargets(ctx, resTargetsList) - assert.Nil(t, err) + modelTg := model.TargetGroup{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", "tg-stack-id"), + Status: &model.TargetGroupStatus{ + Name: "tg-name", + Arn: "tg-arn", + Id: "tg-id", + }, + } + assert.NoError(t, stack.AddResource(&modelTg)) - // TODO routename - dsTG, err := ds.GetTargetGroup(tgName, "", false) - assert.Equal(t, tt.expectedTargetList, dsTG.EndPoints) + targetsSpec := model.TargetsSpec{ + StackTargetGroupId: modelTg.ID(), + TargetList: targetList, + } + model.NewTargets(stack, targetsSpec) - assert.Nil(t, err) - fmt.Printf("dsTG: %v \n", dsTG) + mockTargetsManager.EXPECT().Update(ctx, gomock.Any(), gomock.Any()).Return(nil) - assert.Nil(t, err) - }) - } + synthesizer := NewTargetsSynthesizer(gwlog.FallbackLogger, nil, mockTargetsManager, stack) + err := synthesizer.Synthesize(ctx) + assert.Nil(t, err) } diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index da36d658..ff5a780e 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -2,7 +2,7 @@ package deploy import ( "context" - + "github.com/aws/aws-application-networking-k8s/pkg/gateway" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" "sigs.k8s.io/controller-runtime/pkg/client" @@ -10,21 +10,13 @@ import ( pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-application-networking-k8s/pkg/deploy/externaldns" "github.com/aws/aws-application-networking-k8s/pkg/deploy/lattice" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) -// StackDeployer will deploy a resource stack into AWS and K8S. type StackDeployer interface { - // Deploy a resource stack. Deploy(ctx context.Context, stack core.Stack) error } -//var _ StackDeployer = &defaultStackDeployer{} - -// TODO, later might have a single stack, righ now will have -// dedicated stack for serviceNetwork/service/targetgroup type serviceNetworkStackDeployer struct { log gwlog.Logger cloud pkg_aws.Cloud @@ -47,7 +39,6 @@ func NewServiceNetworkStackDeployer(log gwlog.Logger, cloud pkg_aws.Cloud, k8sCl } // Deploy a resource stack - func deploy(ctx context.Context, stack core.Stack, synthesizers []ResourceSynthesizer) error { for _, synthesizer := range synthesizers { if err := synthesizer.Synthesize(ctx); err != nil { @@ -80,69 +71,77 @@ type latticeServiceStackDeployer struct { listenerManager lattice.ListenerManager ruleManager lattice.RuleManager dnsEndpointManager externaldns.DnsEndpointManager - latticeDataStore *latticestore.LatticeDataStore + svcExportTgBuilder gateway.SvcExportTargetGroupModelBuilder + svcBuilder gateway.LatticeServiceBuilder } func NewLatticeServiceStackDeploy( log gwlog.Logger, cloud pkg_aws.Cloud, k8sClient client.Client, - latticeDataStore *latticestore.LatticeDataStore, ) *latticeServiceStackDeployer { return &latticeServiceStackDeployer{ log: log, cloud: cloud, k8sClient: k8sClient, - latticeServiceManager: lattice.NewServiceManager(cloud, latticeDataStore), + latticeServiceManager: lattice.NewServiceManager(log, cloud), targetGroupManager: lattice.NewTargetGroupManager(log, cloud), - targetsManager: lattice.NewTargetsManager(log, cloud, latticeDataStore), - listenerManager: lattice.NewListenerManager(log, cloud, latticeDataStore), - ruleManager: lattice.NewRuleManager(log, cloud, latticeDataStore), + targetsManager: lattice.NewTargetsManager(log, cloud), + listenerManager: lattice.NewListenerManager(log, cloud), + ruleManager: lattice.NewRuleManager(log, cloud), dnsEndpointManager: externaldns.NewDnsEndpointManager(log, k8sClient), - latticeDataStore: latticeDataStore, + svcExportTgBuilder: gateway.NewSvcExportTargetGroupBuilder(log, k8sClient), + svcBuilder: gateway.NewLatticeServiceBuilder(log, k8sClient), } } func (d *latticeServiceStackDeployer) Deploy(ctx context.Context, stack core.Stack) error { - targetGroupSynthesizer := lattice.NewTargetGroupSynthesizer(d.log, d.cloud, d.k8sClient, d.targetGroupManager, stack, d.latticeDataStore) - targetsSynthesizer := lattice.NewTargetsSynthesizer(d.log, d.cloud, d.targetsManager, stack, d.latticeDataStore) - serviceSynthesizer := lattice.NewServiceSynthesizer(d.log, d.latticeServiceManager, d.dnsEndpointManager, stack, d.latticeDataStore) - listenerSynthesizer := lattice.NewListenerSynthesizer(d.log, d.listenerManager, stack, d.latticeDataStore) - ruleSynthesizer := lattice.NewRuleSynthesizer(d.log, d.ruleManager, stack, d.latticeDataStore) + targetGroupSynthesizer := lattice.NewTargetGroupSynthesizer(d.log, d.cloud, d.k8sClient, d.targetGroupManager, d.svcExportTgBuilder, d.svcBuilder, stack) + targetsSynthesizer := lattice.NewTargetsSynthesizer(d.log, d.cloud, d.targetsManager, stack) + serviceSynthesizer := lattice.NewServiceSynthesizer(d.log, d.latticeServiceManager, d.dnsEndpointManager, stack) + listenerSynthesizer := lattice.NewListenerSynthesizer(d.log, d.listenerManager, stack) + ruleSynthesizer := lattice.NewRuleSynthesizer(d.log, d.ruleManager, d.targetGroupManager, stack) //Handle targetGroups creation request - if err := targetGroupSynthesizer.SynthesizeTriggeredTargetGroupsCreation(ctx); err != nil { + if err := targetGroupSynthesizer.SynthesizeCreate(ctx); err != nil { + d.log.Infof("Error during tg synthesis %s", err) return err } //Handle targets "reconciliation" request (register intend-to-be-registered targets and deregister intend-to-be-registered targets) if err := targetsSynthesizer.Synthesize(ctx); err != nil { + d.log.Infof("Error during target synthesis %s", err) return err } // Handle latticeService "reconciliation" request if err := serviceSynthesizer.Synthesize(ctx); err != nil { + d.log.Infof("Error during service synthesis %s", err) return err } //Handle latticeService listeners "reconciliation" request if err := listenerSynthesizer.Synthesize(ctx); err != nil { + d.log.Infof("Error during listener synthesis %s", err) return err } //Handle latticeService listener's rules "reconciliation" request if err := ruleSynthesizer.Synthesize(ctx); err != nil { + d.log.Infof("Error during rule synthesis %s", err) return err } //Handle targetGroup deletion request - if err := targetGroupSynthesizer.SynthesizeTriggeredTargetGroupsDeletion(ctx); err != nil { + if err := targetGroupSynthesizer.SynthesizeDelete(ctx); err != nil { + d.log.Infof("Error during tg delete synthesis %s", err) return err } // Do garbage collection for not-in-use targetGroups //TODO: run SynthesizeSDKTargetGroups(ctx) as a global garbage collector scheduled backgroud task (i.e., run it as a goroutine in main.go) - if err := targetGroupSynthesizer.SynthesizeSDKTargetGroups(ctx); err != nil { + if err := targetGroupSynthesizer.SynthesizeUnusedDelete(ctx); err != nil { + d.log.Infof("Error during tg unused delete synthesis %s", err) return err } @@ -154,7 +153,8 @@ type latticeTargetGroupStackDeployer struct { cloud pkg_aws.Cloud k8sclient client.Client targetGroupManager lattice.TargetGroupManager - latticeDatastore *latticestore.LatticeDataStore + svcExportTgBuilder gateway.SvcExportTargetGroupModelBuilder + svcBuilder gateway.LatticeServiceBuilder } // triggered by service export @@ -162,79 +162,25 @@ func NewTargetGroupStackDeploy( log gwlog.Logger, cloud pkg_aws.Cloud, k8sClient client.Client, - latticeDataStore *latticestore.LatticeDataStore, ) *latticeTargetGroupStackDeployer { return &latticeTargetGroupStackDeployer{ log: log, cloud: cloud, k8sclient: k8sClient, targetGroupManager: lattice.NewTargetGroupManager(log, cloud), - latticeDatastore: latticeDataStore, + svcExportTgBuilder: gateway.NewSvcExportTargetGroupBuilder(log, k8sClient), + svcBuilder: gateway.NewLatticeServiceBuilder(log, k8sClient), } } func (d *latticeTargetGroupStackDeployer) Deploy(ctx context.Context, stack core.Stack) error { synthesizers := []ResourceSynthesizer{ - lattice.NewTargetGroupSynthesizer(d.log, d.cloud, d.k8sclient, d.targetGroupManager, stack, d.latticeDatastore), - lattice.NewTargetsSynthesizer(d.log, d.cloud, lattice.NewTargetsManager(d.log, d.cloud, d.latticeDatastore), stack, d.latticeDatastore), + lattice.NewTargetGroupSynthesizer(d.log, d.cloud, d.k8sclient, d.targetGroupManager, d.svcExportTgBuilder, d.svcBuilder, stack), + lattice.NewTargetsSynthesizer(d.log, d.cloud, lattice.NewTargetsManager(d.log, d.cloud), stack), } return deploy(ctx, stack, synthesizers) } -type latticeTargetsStackDeployer struct { - log gwlog.Logger - k8sClient client.Client - stack core.Stack - targetsManager lattice.TargetsManager - latticeDataStore *latticestore.LatticeDataStore -} - -func NewTargetsStackDeployer( - log gwlog.Logger, - cloud pkg_aws.Cloud, - k8sClient client.Client, - latticeDataStore *latticestore.LatticeDataStore, -) *latticeTargetsStackDeployer { - return &latticeTargetsStackDeployer{ - k8sClient: k8sClient, - targetsManager: lattice.NewTargetsManager(log, cloud, latticeDataStore), - latticeDataStore: latticeDataStore, - } -} - -func (d *latticeTargetsStackDeployer) Deploy(ctx context.Context, stack core.Stack) error { - var resTargets []*model.Targets - - d.stack = stack - - err := d.stack.ListResources(&resTargets) - if err != nil { - d.log.Errorf("Failed to list targets due to %s", err) - } - - for _, targets := range resTargets { - err := d.targetsManager.Create(ctx, targets) - if err == nil { - tgName := latticestore.TargetGroupName(targets.Spec.Name, targets.Spec.Namespace) - - var targetList []latticestore.Target - for _, target := range targetList { - t := latticestore.Target{ - TargetIP: target.TargetIP, - TargetPort: target.TargetPort, - } - targetList = append(targetList, t) - } - err := d.latticeDataStore.UpdateTargetsForTargetGroup(tgName, targets.Spec.RouteName, targetList) - if err != nil { - d.log.Errorf("Failed to update targets for target group %s due to %s", tgName, err) - } - } - - } - return nil -} - type accessLogSubscriptionStackDeployer struct { log gwlog.Logger k8sClient client.Client diff --git a/pkg/deploy/stack_deployer_test.go b/pkg/deploy/stack_deployer_test.go deleted file mode 100644 index 3c3393f1..00000000 --- a/pkg/deploy/stack_deployer_test.go +++ /dev/null @@ -1,255 +0,0 @@ -package deploy - -import ( - "context" - "testing" - - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/service/vpclattice" - - "github.com/aws/aws-application-networking-k8s/pkg/aws/services" - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/assert" - "k8s.io/apimachinery/pkg/types" - - mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" - mocks_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-application-networking-k8s/pkg/deploy/lattice" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - - "github.com/aws/aws-application-networking-k8s/pkg/deploy/externaldns" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" -) - -func Test_latticeServiceStackDeployer_createAllResources(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - - mockClient := mock_client.NewMockClient(c) - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := services.NewMockLattice(c) - mockServiceManager := lattice.NewMockServiceManager(c) - mockTargetGroupManager := lattice.NewMockTargetGroupManager(c) - mockListenerManager := lattice.NewMockListenerManager(c) - mockRuleManager := lattice.NewMockRuleManager(c) - mockDnsManager := externaldns.NewMockDnsEndpointManager(c) - mockTargetsManager := lattice.NewMockTargetsManager(c) - mockLatticeDataStore := latticestore.NewLatticeDataStore() - - ctx := context.TODO() - - s := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - stackService := model.NewLatticeService(s, "fake-service", model.ServiceSpec{}) - model.NewTargetGroup(s, "fake-targetGroup", model.TargetGroupSpec{}) - model.NewTargets(s, "fake-target", model.TargetsSpec{}) - model.NewListener(s, "fake-listener", 8080, "HTTP", "service1", "default", model.DefaultAction{}) - model.NewRule(s, "fake-rule", "fake-rule", "default", 80, "HTTP", model.RuleAction{}, model.RuleSpec{}) - - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String(stackService.LatticeServiceName()), - Id: aws.String("fake-service"), - }, nil).AnyTimes() - - mockListenerManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockRuleManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() - - mockTargetGroupManager.EXPECT().List(gomock.Any()).AnyTimes() - mockListenerManager.EXPECT().List(gomock.Any(), gomock.Any()).AnyTimes() - - mockServiceManager.EXPECT().Create(gomock.Any(), gomock.Any()) - mockTargetGroupManager.EXPECT().Create(gomock.Any(), gomock.Any()).AnyTimes() - mockTargetsManager.EXPECT().Create(gomock.Any(), gomock.Any()) - mockListenerManager.EXPECT().Create(gomock.Any(), gomock.Any()) - mockRuleManager.EXPECT().Create(gomock.Any(), gomock.Any()) - mockDnsManager.EXPECT().Create(gomock.Any(), gomock.Any()) - - deployer := &latticeServiceStackDeployer{ - log: gwlog.FallbackLogger, - cloud: mockCloud, - k8sClient: mockClient, - latticeServiceManager: mockServiceManager, - targetGroupManager: mockTargetGroupManager, - listenerManager: mockListenerManager, - ruleManager: mockRuleManager, - targetsManager: mockTargetsManager, - dnsEndpointManager: mockDnsManager, - latticeDataStore: mockLatticeDataStore, - } - - err := deployer.Deploy(ctx, s) - - assert.Nil(t, err) -} - -func Test_latticeServiceStackDeployer_CreateJustService(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - - mockClient := mock_client.NewMockClient(c) - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := services.NewMockLattice(c) - mockServiceManager := lattice.NewMockServiceManager(c) - mockTargetGroupManager := lattice.NewMockTargetGroupManager(c) - mockTargetsManager := lattice.NewMockTargetsManager(c) - mockListenerManager := lattice.NewMockListenerManager(c) - mockRuleManager := lattice.NewMockRuleManager(c) - mockDnsManager := externaldns.NewMockDnsEndpointManager(c) - mockLatticeDataStore := latticestore.NewLatticeDataStore() - - ctx := context.TODO() - - mockTargetGroupManager.EXPECT().List(gomock.Any()) - mockListenerManager.EXPECT().List(gomock.Any(), gomock.Any()) - - mockServiceManager.EXPECT().Create(gomock.Any(), gomock.Any()) - mockDnsManager.EXPECT().Create(gomock.Any(), gomock.Any()) - - s := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - stackService := model.NewLatticeService(s, "fake-service", model.ServiceSpec{}) - - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String(stackService.LatticeServiceName()), - Id: aws.String("fake-service"), - }, nil).AnyTimes() - - mockListenerManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockRuleManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() - - deployer := &latticeServiceStackDeployer{ - log: gwlog.FallbackLogger, - cloud: mockCloud, - k8sClient: mockClient, - latticeServiceManager: mockServiceManager, - targetGroupManager: mockTargetGroupManager, - targetsManager: mockTargetsManager, - listenerManager: mockListenerManager, - ruleManager: mockRuleManager, - dnsEndpointManager: mockDnsManager, - latticeDataStore: mockLatticeDataStore, - } - - err := deployer.Deploy(ctx, s) - - assert.Nil(t, err) -} - -func Test_latticeServiceStackDeployer_DeleteService(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - - mockClient := mock_client.NewMockClient(c) - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := services.NewMockLattice(c) - mockServiceManager := lattice.NewMockServiceManager(c) - mockTargetGroupManager := lattice.NewMockTargetGroupManager(c) - mockListenerManager := lattice.NewMockListenerManager(c) - mockRuleManager := lattice.NewMockRuleManager(c) - mockTargetsManager := lattice.NewMockTargetsManager(c) - mockLatticeDataStore := latticestore.NewLatticeDataStore() - - ctx := context.TODO() - - s := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - stackService := model.NewLatticeService(s, "fake-service", model.ServiceSpec{ - IsDeleted: true, - }) - - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String(stackService.LatticeServiceName()), - Id: aws.String("fake-service"), - }, nil).AnyTimes() - - mockListenerManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockRuleManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() - - mockTargetGroupManager.EXPECT().List(gomock.Any()).AnyTimes() - mockListenerManager.EXPECT().List(gomock.Any(), gomock.Any()).AnyTimes() - - mockServiceManager.EXPECT().Delete(gomock.Any(), gomock.Any()) - - deployer := &latticeServiceStackDeployer{ - log: gwlog.FallbackLogger, - cloud: mockCloud, - k8sClient: mockClient, - latticeServiceManager: mockServiceManager, - targetGroupManager: mockTargetGroupManager, - listenerManager: mockListenerManager, - ruleManager: mockRuleManager, - targetsManager: mockTargetsManager, - latticeDataStore: mockLatticeDataStore, - } - - err := deployer.Deploy(ctx, s) - - assert.Nil(t, err) -} - -func Test_latticeServiceStackDeployer_DeleteAllResources(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - - mockClient := mock_client.NewMockClient(c) - mockCloud := mocks_aws.NewMockCloud(c) - mockLattice := services.NewMockLattice(c) - mockServiceManager := lattice.NewMockServiceManager(c) - mockTargetGroupManager := lattice.NewMockTargetGroupManager(c) - mockListenerManager := lattice.NewMockListenerManager(c) - mockRuleManager := lattice.NewMockRuleManager(c) - mockTargetsManager := lattice.NewMockTargetsManager(c) - mockLatticeDataStore := latticestore.NewLatticeDataStore() - - ctx := context.TODO() - - s := core.NewDefaultStack(core.StackID(types.NamespacedName{Namespace: "tt", Name: "name"})) - - stackService := model.NewLatticeService(s, "fake-service", model.ServiceSpec{ - IsDeleted: true, - }) - model.NewTargetGroup(s, "fake-targetGroup", model.TargetGroupSpec{ - IsDeleted: true, - }) - - mockLattice.EXPECT().FindService(gomock.Any(), gomock.Any()).Return( - &vpclattice.ServiceSummary{ - Name: aws.String(stackService.LatticeServiceName()), - Id: aws.String("fake-service"), - }, nil).AnyTimes() - - mockListenerManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockRuleManager.EXPECT().Cloud().Return(mockCloud).AnyTimes() - mockCloud.EXPECT().Lattice().Return(mockLattice).AnyTimes() - - mockTargetGroupManager.EXPECT().List(gomock.Any()).AnyTimes() - mockListenerManager.EXPECT().List(gomock.Any(), gomock.Any()).AnyTimes() - - mockServiceManager.EXPECT().Delete(gomock.Any(), gomock.Any()) - mockTargetGroupManager.EXPECT().Delete(gomock.Any(), gomock.Any()) - - deployer := &latticeServiceStackDeployer{ - log: gwlog.FallbackLogger, - cloud: mockCloud, - k8sClient: mockClient, - latticeServiceManager: mockServiceManager, - targetGroupManager: mockTargetGroupManager, - listenerManager: mockListenerManager, - ruleManager: mockRuleManager, - targetsManager: mockTargetsManager, - latticeDataStore: mockLatticeDataStore, - } - - err := deployer.Deploy(ctx, s) - - assert.Nil(t, err) -} diff --git a/pkg/gateway/model_build_lattice_service.go b/pkg/gateway/model_build_lattice_service.go index 6ff4f16e..d1b2b359 100644 --- a/pkg/gateway/model_build_lattice_service.go +++ b/pkg/gateway/model_build_lattice_service.go @@ -2,8 +2,9 @@ package gateway import ( "context" - "errors" "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "sigs.k8s.io/controller-runtime/pkg/client" @@ -13,57 +14,55 @@ import ( "github.com/aws/aws-application-networking-k8s/pkg/k8s" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" - - pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" ) type LatticeServiceBuilder interface { - Build(ctx context.Context, httpRoute core.Route) (core.Stack, *model.Service, error) + Build(ctx context.Context, httpRoute core.Route) (core.Stack, error) } type LatticeServiceModelBuilder struct { log gwlog.Logger client client.Client - defaultTags map[string]string - datastore *latticestore.LatticeDataStore - cloud pkg_aws.Cloud + brTgBuilder BackendRefTargetGroupModelBuilder } func NewLatticeServiceBuilder( log gwlog.Logger, client client.Client, - datastore *latticestore.LatticeDataStore, - cloud pkg_aws.Cloud, ) *LatticeServiceModelBuilder { + brTgBuilder := NewBackendRefTargetGroupBuilder(log, client) + return &LatticeServiceModelBuilder{ - log: log, - client: client, - datastore: datastore, - cloud: cloud, + log: log, + client: client, + brTgBuilder: brTgBuilder, } } func (b *LatticeServiceModelBuilder) Build( ctx context.Context, route core.Route, -) (core.Stack, *model.Service, error) { +) (core.Stack, error) { stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(route.K8sObject()))) + if b.brTgBuilder == nil { + b.log.Debugf("brTgBuilder is nil, initializing") + b.brTgBuilder = NewBackendRefTargetGroupBuilder(b.log, b.client) + } + task := &latticeServiceModelBuildTask{ - log: b.log, - route: route, - stack: stack, - client: b.client, - tgByResID: make(map[string]*model.TargetGroup), - datastore: b.datastore, + log: b.log, + route: route, + stack: stack, + client: b.client, + brTgBuilder: b.brTgBuilder, } if err := task.run(ctx); err != nil { - return stack, task.latticeService, errors.New("LATTICE_RETRY") + return task.stack, err } - return task.stack, task.latticeService, nil + return task.stack, nil } func (t *latticeServiceModelBuildTask) run(ctx context.Context) error { @@ -72,35 +71,35 @@ func (t *latticeServiceModelBuildTask) run(ctx context.Context) error { } func (t *latticeServiceModelBuildTask) buildModel(ctx context.Context) error { - if err := t.buildLatticeService(ctx); err != nil { - return fmt.Errorf("failed to build lattice service due to %w", err) - } - - if err := t.buildTargetGroupsForRoute(ctx, t.client); err != nil { - return fmt.Errorf("failed to build target group due to %w", err) + modelSvc, err := t.buildLatticeService(ctx) + if err != nil { + return err } - if !t.route.DeletionTimestamp().IsZero() { - t.log.Debugf("Ignoring building lattice service on delete for route %s-%s", t.route.Name(), t.route.Namespace()) - return nil - } - - if err := t.buildTargetsForRoute(ctx); err != nil { - t.log.Debugf("failed to build targets due to %s", err) - } - - if err := t.buildListeners(ctx); err != nil { + err = t.buildListeners(ctx, modelSvc.ID()) + if err != nil { return fmt.Errorf("failed to build listener due to %w", err) } - if err := t.buildRules(ctx); err != nil { - return fmt.Errorf("failed to build rule due to %w", err) + var modelListeners []*model.Listener + err = t.stack.ListResources(&modelListeners) + if err != nil { + return err + } + t.log.Debugf("Building rules for %d listeners", len(modelListeners)) + for _, modelListener := range modelListeners { + // building rules will also build target groups and targets as needed + // even on delete we try to build everything we may then need to remove + err = t.buildRules(ctx, modelListener.ID()) + if err != nil { + return fmt.Errorf("failed to build rules due to %w", err) + } } return nil } -func (t *latticeServiceModelBuildTask) buildLatticeService(ctx context.Context) error { +func (t *latticeServiceModelBuildTask) buildLatticeService(ctx context.Context) (*model.Service, error) { routeType := core.HttpRouteType if _, ok := t.route.(*core.GRPCRoute); ok { routeType = core.GrpcRouteType @@ -132,28 +131,65 @@ func (t *latticeServiceModelBuildTask) buildLatticeService(ctx context.Context) spec.CustomerDomainName = "" } - if t.route.DeletionTimestamp().IsZero() { - spec.IsDeleted = false - } else { - spec.IsDeleted = true + certArn, err := t.getACMCertArn(ctx) + if err != nil { + return nil, err + } + spec.CustomerCertARN = certArn + + svc, err := model.NewLatticeService(t.stack, spec) + if err != nil { + return nil, err } - serviceResourceName := fmt.Sprintf("%s-%s", t.route.Name(), t.route.Namespace()) + t.log.Debugf("Added service %s to the stack (ID %s)", svc.Spec.LatticeServiceName(), svc.ID()) + svc.IsDeleted = !t.route.DeletionTimestamp().IsZero() + return svc, nil +} - t.latticeService = model.NewLatticeService(t.stack, serviceResourceName, spec) +// returns empty string if not found +func (t *latticeServiceModelBuildTask) getACMCertArn(ctx context.Context) (string, error) { + gw, err := t.getGateway(ctx) + if err != nil { + if apierrors.IsNotFound(err) && !t.route.DeletionTimestamp().IsZero() { + return "", nil // ok if we're deleting the route + } + return "", err + } - return nil + for _, parentRef := range t.route.Spec().ParentRefs() { + if parentRef.Name != t.route.Spec().ParentRefs()[0].Name { + // when a service is associate to multiple service network(s), all listener config MUST be same + // so here we are only using the 1st gateway + t.log.Debugf("Ignore ParentRef of different gateway %s-%s", parentRef.Name, parentRef.Namespace) + continue + } + + if parentRef.SectionName == nil { + continue + } + + for _, section := range gw.Spec.Listeners { + if section.Name == *parentRef.SectionName && section.TLS != nil { + if section.TLS.Mode != nil && *section.TLS.Mode == gwv1beta1.TLSModeTerminate { + curCertARN, ok := section.TLS.Options[awsCustomCertARN] + if ok { + t.log.Debugf("Found certification %s under section %s", curCertARN, section.Name) + return string(curCertARN), nil + } + } + break + } + } + } + + return "", nil } type latticeServiceModelBuildTask struct { - log gwlog.Logger - route core.Route - client client.Client - latticeService *model.Service - tgByResID map[string]*model.TargetGroup - listenerByResID map[string]*model.Listener - rulesByResID map[string]*model.Rule - stack core.Stack - datastore *latticestore.LatticeDataStore - cloud pkg_aws.Cloud + log gwlog.Logger + route core.Route + client client.Client + stack core.Stack + brTgBuilder BackendRefTargetGroupModelBuilder } diff --git a/pkg/gateway/model_build_lattice_service_mock.go b/pkg/gateway/model_build_lattice_service_mock.go new file mode 100644 index 00000000..c8591199 --- /dev/null +++ b/pkg/gateway/model_build_lattice_service_mock.go @@ -0,0 +1,51 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./pkg/gateway/model_build_lattice_service.go + +// Package gateway is a generated GoMock package. +package gateway + +import ( + context "context" + reflect "reflect" + + core "github.com/aws/aws-application-networking-k8s/pkg/model/core" + gomock "github.com/golang/mock/gomock" +) + +// MockLatticeServiceBuilder is a mock of LatticeServiceBuilder interface. +type MockLatticeServiceBuilder struct { + ctrl *gomock.Controller + recorder *MockLatticeServiceBuilderMockRecorder +} + +// MockLatticeServiceBuilderMockRecorder is the mock recorder for MockLatticeServiceBuilder. +type MockLatticeServiceBuilderMockRecorder struct { + mock *MockLatticeServiceBuilder +} + +// NewMockLatticeServiceBuilder creates a new mock instance. +func NewMockLatticeServiceBuilder(ctrl *gomock.Controller) *MockLatticeServiceBuilder { + mock := &MockLatticeServiceBuilder{ctrl: ctrl} + mock.recorder = &MockLatticeServiceBuilderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockLatticeServiceBuilder) EXPECT() *MockLatticeServiceBuilderMockRecorder { + return m.recorder +} + +// Build mocks base method. +func (m *MockLatticeServiceBuilder) Build(ctx context.Context, httpRoute core.Route) (core.Stack, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Build", ctx, httpRoute) + ret0, _ := ret[0].(core.Stack) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Build indicates an expected call of Build. +func (mr *MockLatticeServiceBuilderMockRecorder) Build(ctx, httpRoute interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*MockLatticeServiceBuilder)(nil).Build), ctx, httpRoute) +} diff --git a/pkg/gateway/model_build_lattice_service_test.go b/pkg/gateway/model_build_lattice_service_test.go index 502440e2..56d387e7 100644 --- a/pkg/gateway/model_build_lattice_service_test.go +++ b/pkg/gateway/model_build_lattice_service_test.go @@ -2,26 +2,19 @@ package gateway import ( "context" - "fmt" + "github.com/aws/aws-application-networking-k8s/pkg/k8s" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - "testing" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - "k8s.io/apimachinery/pkg/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - - "github.com/aws/aws-application-networking-k8s/pkg/k8s" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "testing" ) func Test_LatticeServiceModelBuild(t *testing.T) { @@ -33,6 +26,11 @@ func Test_LatticeServiceModelBuild(t *testing.T) { var weight2 = int32(90) var namespace = gwv1beta1.Namespace("default") + namespacePtr := func(ns string) *gwv1beta1.Namespace { + p := gwv1beta1.Namespace(ns) + return &p + } + var backendRef1 = gwv1beta1.BackendRef{ BackendObjectReference: gwv1beta1.BackendObjectReference{ Name: "targetgroup1", @@ -50,26 +48,38 @@ func Test_LatticeServiceModelBuild(t *testing.T) { Weight: &weight2, } + tlsSectionName := gwv1beta1.SectionName("tls") + tlsModeTerminate := gwv1beta1.TLSModeTerminate + tests := []struct { name string + gw gwv1beta1.Gateway route core.Route - wantError error wantErrIsNil bool - wantName string - wantRouteType core.RouteType wantIsDeleted bool + expected model.ServiceSpec }{ { - name: "Add LatticeService with hostname", + name: "Add LatticeService with hostname", + wantIsDeleted: false, + wantErrIsNil: true, + gw: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway1", + Namespace: "default", + }, + }, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Name: "service1", + Name: "service1", + Namespace: "test", }, Spec: gwv1beta1.HTTPRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ ParentRefs: []gwv1beta1.ParentReference{ { - Name: "gateway1", + Name: "gateway1", + Namespace: namespacePtr("default"), }, }, }, @@ -79,41 +89,61 @@ func Test_LatticeServiceModelBuild(t *testing.T) { }, }, }), - - wantError: nil, - wantName: "service1", - wantRouteType: core.HttpRouteType, - wantIsDeleted: false, - wantErrIsNil: true, + expected: model.ServiceSpec{ + Name: "service1", + Namespace: "test", + CustomerDomainName: "test1.test.com", + RouteType: core.HttpRouteType, + ServiceNetworkNames: []string{"gateway1"}, + }, }, { - name: "Add LatticeService", + name: "Add LatticeService", + wantIsDeleted: false, + wantErrIsNil: true, + gw: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway1", + Namespace: "default", + }, + }, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Name: "service1", + Name: "service1", + Namespace: "default", }, Spec: gwv1beta1.HTTPRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ ParentRefs: []gwv1beta1.ParentReference{ { - Name: "gateway1", + Name: "gateway1", + Namespace: namespacePtr("default"), }, }, }, }, }), - - wantError: nil, - wantName: "service1", - wantRouteType: core.HttpRouteType, - wantIsDeleted: false, - wantErrIsNil: true, + expected: model.ServiceSpec{ + Name: "service1", + Namespace: "default", + RouteType: core.HttpRouteType, + ServiceNetworkNames: []string{"gateway1"}, + }, }, { - name: "Add LatticeService with GRPCRoute", + name: "Add LatticeService with GRPCRoute", + wantIsDeleted: false, + wantErrIsNil: true, + gw: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway1", + Namespace: "test", + }, + }, route: core.NewGRPCRoute(gwv1alpha2.GRPCRoute{ ObjectMeta: metav1.ObjectMeta{ - Name: "service1", + Name: "service1", + Namespace: "test", }, Spec: gwv1alpha2.GRPCRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ @@ -125,19 +155,38 @@ func Test_LatticeServiceModelBuild(t *testing.T) { }, }, }), - wantError: nil, - wantName: "service1", - wantRouteType: core.GrpcRouteType, - wantIsDeleted: false, - wantErrIsNil: true, + expected: model.ServiceSpec{ + Name: "service1", + Namespace: "test", + RouteType: core.GrpcRouteType, + ServiceNetworkNames: []string{"gateway1"}, + }, }, { - name: "Delete LatticeService", + name: "Delete LatticeService", + wantIsDeleted: true, + wantErrIsNil: true, + gw: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway2", + Namespace: "ns1", + }, + Spec: gwv1beta1.GatewaySpec{ + Listeners: []gwv1beta1.Listener{ + { + Name: httpSectionName, + Port: 80, + Protocol: "HTTP", + }, + }, + }, + }, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service2", + Namespace: "ns1", Finalizers: []string{"gateway.k8s.aws/resources"}, - DeletionTimestamp: &now, + DeletionTimestamp: &now, // <- the important bit }, Spec: gwv1beta1.HTTPRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ @@ -162,12 +211,173 @@ func Test_LatticeServiceModelBuild(t *testing.T) { }, }, }), - - wantError: nil, - wantName: "service2", - wantRouteType: core.HttpRouteType, - wantIsDeleted: true, + expected: model.ServiceSpec{ + Name: "service2", + Namespace: "ns1", + RouteType: core.HttpRouteType, + ServiceNetworkNames: []string{"gateway2"}, + }, + }, + { + name: "Service with customer Cert ARN", + wantIsDeleted: false, wantErrIsNil: true, + gw: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway1", + Namespace: "default", + }, + Spec: gwv1beta1.GatewaySpec{ + Listeners: []gwv1beta1.Listener{ + { + Name: "tls", + Port: 443, + Protocol: "HTTPS", + TLS: &gwv1beta1.GatewayTLSConfig{ + Mode: &tlsModeTerminate, + CertificateRefs: nil, + Options: map[gwv1beta1.AnnotationKey]gwv1beta1.AnnotationValue{ + "application-networking.k8s.aws/certificate-arn": "cert-arn", + }, + }, + }, + }, + }, + }, + route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service1", + Namespace: "default", + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Name: "gateway1", + Namespace: namespacePtr("default"), + SectionName: &tlsSectionName, + }, + }, + }, + }, + }), + expected: model.ServiceSpec{ + Name: "service1", + Namespace: "default", + RouteType: core.HttpRouteType, + CustomerCertARN: "cert-arn", + ServiceNetworkNames: []string{"gateway1"}, + }, + }, + { + name: "GW does not exist", + gw: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway1", + Namespace: "default", + }, + }, + route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service1", + Namespace: "default", + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Name: "not-a-real-gateway", + Namespace: namespacePtr("default"), + }, + }, + }, + }, + }), + wantErrIsNil: false, + }, + { + name: "Service with TLS section but no cert arn", + wantIsDeleted: false, + wantErrIsNil: true, + gw: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway1", + Namespace: "default", + }, + Spec: gwv1beta1.GatewaySpec{ + Listeners: []gwv1beta1.Listener{ + { + Name: "tls", + Port: 443, + Protocol: "HTTPS", + TLS: &gwv1beta1.GatewayTLSConfig{ + Mode: &tlsModeTerminate, + CertificateRefs: nil, + }, + }, + }, + }, + }, + route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service1", + Namespace: "default", + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Name: "gateway1", + Namespace: namespacePtr("default"), + SectionName: &tlsSectionName, + }, + }, + }, + }, + }), + expected: model.ServiceSpec{ + Name: "service1", + Namespace: "default", + RouteType: core.HttpRouteType, + ServiceNetworkNames: []string{"gateway1"}, + }, + }, + { + name: "Multiple service networks", + wantIsDeleted: false, + wantErrIsNil: true, + gw: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway1", + Namespace: "default", + }, + }, + route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "service1", + Namespace: "default", + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{ + { + Name: "gateway1", + Namespace: namespacePtr("default"), + }, + { + Name: "gateway2", + Namespace: namespacePtr("ns2"), + }, + }, + }, + }, + }), + expected: model.ServiceSpec{ + Name: "service1", + Namespace: "default", + RouteType: core.HttpRouteType, + ServiceNetworkNames: []string{"gateway1", "gateway2"}, + }, }, } @@ -179,55 +389,34 @@ func Test_LatticeServiceModelBuild(t *testing.T) { k8sSchema := runtime.NewScheme() clientgoscheme.AddToScheme(k8sSchema) + gwv1beta1.AddToScheme(k8sSchema) k8sClient := testclient.NewFakeClientWithScheme(k8sSchema) - ds := latticestore.NewLatticeDataStore() - - //builder := NewLatticeServiceBuilder(k8sClient, ds, nil) + assert.NoError(t, k8sClient.Create(ctx, tt.gw.DeepCopy())) stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.route.K8sObject()))) task := &latticeServiceModelBuildTask{ - log: gwlog.FallbackLogger, - route: tt.route, - stack: stack, - client: k8sClient, - tgByResID: make(map[string]*model.TargetGroup), - datastore: ds, + log: gwlog.FallbackLogger, + route: tt.route, + stack: stack, + client: k8sClient, } - err := task.buildLatticeService(ctx) - - fmt.Printf("task.latticeService.Spec %v, err: %v\n", task.latticeService.Spec, err) - - if tt.wantIsDeleted { - assert.Equal(t, true, task.latticeService.Spec.IsDeleted) - // make sure no rules and listener are built - var resRules []*model.Rule - stack.ListResources(&resRules) - assert.Equal(t, len(resRules), 0) - - var resListener []*model.Listener - stack.ListResources(&resListener) - assert.Equal(t, len(resListener), 0) - - } else { - assert.Equal(t, false, task.latticeService.Spec.IsDeleted) - assert.Equal(t, tt.route.Name(), task.latticeService.Spec.Name) - assert.Equal(t, tt.route.Namespace(), task.latticeService.Spec.Namespace) - - if len(tt.route.Spec().Hostnames()) > 0 { - assert.Equal(t, string(tt.route.Spec().Hostnames()[0]), task.latticeService.Spec.CustomerDomainName) - } else { - assert.Equal(t, "", task.latticeService.Spec.CustomerDomainName) - } + svc, err := task.buildLatticeService(ctx) + if !tt.wantErrIsNil { + assert.NotNil(t, err) + return } + assert.Nil(t, err) - if tt.wantErrIsNil { - assert.Nil(t, err) + assert.Equal(t, tt.wantIsDeleted, svc.IsDeleted) - } else { - assert.NotNil(t, err) - } + assert.Equal(t, tt.expected.Name, svc.Spec.Name) + assert.Equal(t, tt.expected.Namespace, svc.Spec.Namespace) + assert.Equal(t, tt.expected.CustomerCertARN, svc.Spec.CustomerCertARN) + assert.Equal(t, tt.expected.CustomerDomainName, svc.Spec.CustomerDomainName) + assert.Equal(t, tt.expected.RouteType, svc.Spec.RouteType) + assert.Equal(t, tt.expected.ServiceNetworkNames, svc.Spec.ServiceNetworkNames) }) } } diff --git a/pkg/gateway/model_build_listener.go b/pkg/gateway/model_build_listener.go index 40a780b1..f6fb12ac 100644 --- a/pkg/gateway/model_build_listener.go +++ b/pkg/gateway/model_build_listener.go @@ -4,8 +4,8 @@ import ( "context" "errors" "fmt" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/types" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -18,31 +18,23 @@ const ( func (t *latticeServiceModelBuildTask) extractListenerInfo( ctx context.Context, parentRef gwv1beta1.ParentReference, -) (int64, string, string, error) { +) (int64, string, error) { protocol := gwv1beta1.HTTPProtocolType if parentRef.SectionName != nil { t.log.Debugf("Listener parentRef SectionName is %s", *parentRef.SectionName) } t.log.Debugf("Building Listener for Route %s-%s", t.route.Name(), t.route.Namespace()) - var gwNamespace = t.route.Namespace() - if t.route.Spec().ParentRefs()[0].Namespace != nil { - gwNamespace = string(*t.route.Spec().ParentRefs()[0].Namespace) - } - - var listenerPort = 0 - gw := &gwv1beta1.Gateway{} - gwName := types.NamespacedName{ - Namespace: gwNamespace, - Name: string(t.route.Spec().ParentRefs()[0].Name), - } - - if err := t.client.Get(ctx, gwName, gw); err != nil { - return 0, "", "", fmt.Errorf("failed to build Listener due to unknow http parent ref, Name %s, err %w", gwName, err) + gw, err := t.getGateway(ctx) + if err != nil { + if apierrors.IsNotFound(err) && !t.route.DeletionTimestamp().IsZero() { + return 0, string(protocol), nil // ok if we're deleting the route + } + return 0, "", err } - var certARN = "" // go through parent find out the matching section name + var listenerPort int if parentRef.SectionName != nil { found := false for _, section := range gw.Spec.Listeners { @@ -50,36 +42,48 @@ func (t *latticeServiceModelBuildTask) extractListenerInfo( listenerPort = int(section.Port) protocol = section.Protocol found = true - - if section.TLS != nil { - if section.TLS.Mode != nil && *section.TLS.Mode == gwv1beta1.TLSModeTerminate { - curCertARN, ok := section.TLS.Options[awsCustomCertARN] - if ok { - t.log.Debugf("Found certification %s under section %s", curCertARN, section.Name) - certARN = string(curCertARN) - } - } - } break } } if !found { - return 0, "", "", fmt.Errorf("error building listener, no matching sectionName in parentRef for Name %s, Section %s", parentRef.Name, *parentRef.SectionName) + return 0, "", fmt.Errorf("error building listener, no matching sectionName in parentRef for Name %s, Section %s", parentRef.Name, *parentRef.SectionName) } } else { // use 1st listener port - // TODO check no listener if len(gw.Spec.Listeners) == 0 { - return 0, "", "", errors.New("error building listener, there is NO listeners on GW") + return 0, "", errors.New("error building listener, there is NO listeners on GW") } + listenerPort = int(gw.Spec.Listeners[0].Port) + protocol = gw.Spec.Listeners[0].Protocol } - return int64(listenerPort), string(protocol), certARN, nil + return int64(listenerPort), string(protocol), nil +} + +func (t *latticeServiceModelBuildTask) getGateway(ctx context.Context) (*gwv1beta1.Gateway, error) { + var gwNamespace = t.route.Namespace() + if t.route.Spec().ParentRefs()[0].Namespace != nil { + gwNamespace = string(*t.route.Spec().ParentRefs()[0].Namespace) + } + gw := &gwv1beta1.Gateway{} + gwName := types.NamespacedName{ + Namespace: gwNamespace, + Name: string(t.route.Spec().ParentRefs()[0].Name), + } + + if err := t.client.Get(ctx, gwName, gw); err != nil { + return nil, fmt.Errorf("failed to get gateway, name %s, err %w", gwName, err) + } + return gw, nil } -func (t *latticeServiceModelBuildTask) buildListeners(ctx context.Context) error { +func (t *latticeServiceModelBuildTask) buildListeners(ctx context.Context, stackSvcId string) error { + if len(t.route.Spec().ParentRefs()) == 0 { + t.log.Debugf("No ParentRefs on route %s-%s, nothing to do", t.route.Name(), t.route.Namespace()) + } + for _, parentRef := range t.route.Spec().ParentRefs() { if parentRef.Name != t.route.Spec().ParentRefs()[0].Name { // when a service is associate to multiple service network(s), all listener config MUST be same @@ -88,45 +92,27 @@ func (t *latticeServiceModelBuildTask) buildListeners(ctx context.Context) error continue } - port, protocol, certARN, err := t.extractListenerInfo(ctx, parentRef) + port, protocol, err := t.extractListenerInfo(ctx, parentRef) if err != nil { return err } - if t.latticeService != nil { - t.latticeService.Spec.CustomerCertARN = certARN + spec := model.ListenerSpec{ + StackServiceId: stackSvcId, + K8SRouteName: t.route.Name(), + K8SRouteNamespace: t.route.Namespace(), + Port: port, + Protocol: protocol, } - t.log.Debugf("Building Listener: found matching listner Port %d", port) - - if len(t.route.Spec().Rules()) == 0 { - return fmt.Errorf("error building listener, there are no rules for route %s-%s", - t.route.Name(), t.route.Namespace()) - } - - rule := t.route.Spec().Rules()[0] - - if len(rule.BackendRefs()) == 0 { - return fmt.Errorf("error building listener, there are no backend refs for route %s-%s", - t.route.Name(), t.route.Namespace()) - } - - backendRef := rule.BackendRefs()[0] - targetGroupName := string(backendRef.Name()) - - var targetGroupNamespace = t.route.Namespace() - if backendRef.Namespace() != nil { - targetGroupNamespace = string(*backendRef.Namespace()) - } - - action := model.DefaultAction{ - BackendServiceName: targetGroupName, - BackendServiceNamespace: targetGroupNamespace, + modelListener, err := model.NewListener(t.stack, spec) + if err != nil { + return err } - listenerResourceName := fmt.Sprintf("%s-%s-%d-%s", t.route.Name(), t.route.Namespace(), port, protocol) - t.log.Infof("Creating new listener with name %s", listenerResourceName) - model.NewListener(t.stack, listenerResourceName, port, protocol, t.route.Name(), t.route.Namespace(), action) + t.log.Debugf("Added listener %s-%s to the stack (ID %s)", + modelListener.Spec.K8SRouteName, modelListener.Spec.K8SRouteNamespace, modelListener.ID()) + modelListener.IsDeleted = !t.route.DeletionTimestamp().IsZero() } return nil diff --git a/pkg/gateway/model_build_listener_test.go b/pkg/gateway/model_build_listener_test.go index 2929a5e7..7c53d4ff 100644 --- a/pkg/gateway/model_build_listener_test.go +++ b/pkg/gateway/model_build_listener_test.go @@ -3,25 +3,17 @@ package gateway import ( "context" "errors" - "fmt" + mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" + "github.com/aws/aws-application-networking-k8s/pkg/k8s" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - "testing" - "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - "k8s.io/apimachinery/pkg/types" - - mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" - - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - - "github.com/aws/aws-application-networking-k8s/pkg/k8s" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "testing" ) // PortNumberPtr translates an int to a *PortNumber @@ -34,32 +26,22 @@ func Test_ListenerModelBuild(t *testing.T) { var httpSectionName gwv1beta1.SectionName = "http" var missingSectionName gwv1beta1.SectionName = "miss" var serviceKind gwv1beta1.Kind = "Service" - var serviceimportKind gwv1beta1.Kind = "ServiceImport" var backendRef = gwv1beta1.BackendRef{ BackendObjectReference: gwv1beta1.BackendObjectReference{ Name: "targetgroup1", Kind: &serviceKind, }, } - var backendServiceImportRef = gwv1beta1.BackendRef{ - BackendObjectReference: gwv1beta1.BackendObjectReference{ - Name: "targetgroup1", - Kind: &serviceimportKind, - }, - } tests := []struct { name string gwListenerPort gwv1beta1.PortNumber - gwListenerProtocol gwv1beta1.ProtocolType route core.Route wantErrIsNil bool k8sGetGatewayCall bool k8sGatewayReturnOK bool tlsTerminate bool - noTLSOption bool - wrongTLSOption bool - certARN string + expectedSpec []model.ListenerSpec }{ { name: "listener, default service action", @@ -67,74 +49,7 @@ func Test_ListenerModelBuild(t *testing.T) { wantErrIsNil: true, k8sGetGatewayCall: true, k8sGatewayReturnOK: true, - route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", - Namespace: "default", - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gw1", - SectionName: &httpSectionName, - }, - }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ - { - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef, - }, - }, - }, - }, - }, - }), - }, - { - name: "listener, tls with cert arn", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, - tlsTerminate: true, - certARN: "test-cert-ARN", - route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", - Namespace: "default", - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gw1", - SectionName: &httpSectionName, - }, - }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ - { - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef, - }, - }, - }, - }, - }, - }), - }, - { - name: "listener, tls mode is not terminate", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, tlsTerminate: false, - certARN: "test-cert-ARN", route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -160,15 +75,23 @@ func Test_ListenerModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.ListenerSpec{ + { + StackServiceId: "svc-id", + K8SRouteName: "service1", + K8SRouteNamespace: "default", + Port: 80, + Protocol: "HTTP", + }, + }, }, { - name: "listener, with wrong annotation", - gwListenerPort: *PortNumberPtr(80), + name: "tls listener", + gwListenerPort: *PortNumberPtr(443), wantErrIsNil: true, k8sGetGatewayCall: true, k8sGatewayReturnOK: true, - tlsTerminate: false, - certARN: "test-cert-ARN", + tlsTerminate: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -194,43 +117,20 @@ func Test_ListenerModelBuild(t *testing.T) { }, }, }), - }, - { - name: "listener, default serviceimport action", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, - route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", - Namespace: "default", + expectedSpec: []model.ListenerSpec{ + { + StackServiceId: "svc-id", + K8SRouteName: "service1", + K8SRouteNamespace: "default", + Port: 443, + Protocol: "HTTPS", }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gw1", - SectionName: &httpSectionName, - }, - }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ - { - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendServiceImportRef, - }, - }, - }, - }, - }, - }), + }, }, { - name: "no parentref ", + name: "no parentref", gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, + wantErrIsNil: true, k8sGetGatewayCall: false, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ @@ -252,6 +152,7 @@ func Test_ListenerModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.ListenerSpec{}, // empty list }, { name: "No k8sgateway object", @@ -286,7 +187,7 @@ func Test_ListenerModelBuild(t *testing.T) { }), }, { - name: "no section name ", + name: "no section name", gwListenerPort: *PortNumberPtr(80), wantErrIsNil: false, k8sGetGatewayCall: true, @@ -328,106 +229,62 @@ func Test_ListenerModelBuild(t *testing.T) { mockK8sClient := mock_client.NewMockClient(c) if tt.k8sGetGatewayCall { - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).DoAndReturn( func(ctx context.Context, gwName types.NamespacedName, gw *gwv1beta1.Gateway, arg3 ...interface{}) error { + if !tt.k8sGatewayReturnOK { + return errors.New("unknown k8s object") + } + listener := gwv1beta1.Listener{ + Port: tt.gwListenerPort, + Protocol: "HTTP", + Name: httpSectionName, + } - if tt.k8sGatewayReturnOK { - listener := gwv1beta1.Listener{ - Port: tt.gwListenerPort, - Protocol: "HTTP", - Name: *tt.route.Spec().ParentRefs()[0].SectionName, - } - - if tt.tlsTerminate { - mode := gwv1beta1.TLSModeTerminate - var tlsConfig gwv1beta1.GatewayTLSConfig - - if tt.noTLSOption { - tlsConfig = gwv1beta1.GatewayTLSConfig{ - Mode: &mode, - } - - } else { - - tlsConfig = gwv1beta1.GatewayTLSConfig{ - Mode: &mode, - Options: make(map[gwv1beta1.AnnotationKey]gwv1beta1.AnnotationValue), - } - - if tt.wrongTLSOption { - tlsConfig.Options["wrong-annotation"] = gwv1beta1.AnnotationValue(tt.certARN) - - } else { - tlsConfig.Options[awsCustomCertARN] = gwv1beta1.AnnotationValue(tt.certARN) - } - } - listener.TLS = &tlsConfig - + if tt.tlsTerminate { + listener.Protocol = "HTTPS" + mode := gwv1beta1.TLSModeTerminate + listener.TLS = &gwv1beta1.GatewayTLSConfig{ + Mode: &mode, } - gw.Spec.Listeners = append(gw.Spec.Listeners, listener) - return nil - } else { - return errors.New("unknown k8s object") } + + gw.Spec.Listeners = append(gw.Spec.Listeners, listener) + return nil }, ) } - ds := latticestore.NewLatticeDataStore() - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.route.K8sObject()))) task := &latticeServiceModelBuildTask{ - log: gwlog.FallbackLogger, - route: tt.route, - stack: stack, - client: mockK8sClient, - listenerByResID: make(map[string]*model.Listener), - datastore: ds, + log: gwlog.FallbackLogger, + route: tt.route, + client: mockK8sClient, + stack: stack, } - service := model.Service{} - task.latticeService = &service - - err := task.buildListeners(ctx) - - fmt.Printf("task.buildListeners err: %v \n", err) + err := task.buildListeners(ctx, "svc-id") if !tt.wantErrIsNil { - // TODO why following is failing???? - //assert.Equal(t, err!=nil, true) - //assert.Error(t, err) - fmt.Printf("task.buildListeners tt : %v err: %v %v\n", tt.name, err, err != nil) + assert.NotNil(t, err) return - } else { - assert.NoError(t, err) } - fmt.Printf("listeners %v\n", task.listenerByResID) - fmt.Printf("task : %v stack %v\n", task, stack) - var resListener []*model.Listener + assert.NoError(t, err) + var resListener []*model.Listener stack.ListResources(&resListener) - fmt.Printf("resListener :%v \n", resListener) - assert.Equal(t, resListener[0].Spec.Port, int64(tt.gwListenerPort)) - assert.Equal(t, resListener[0].Spec.Name, tt.route.Name()) - assert.Equal(t, resListener[0].Spec.Namespace, tt.route.Namespace()) - assert.Equal(t, resListener[0].Spec.Protocol, "HTTP") + assert.Equal(t, len(tt.expectedSpec), len(resListener)) - assert.Equal(t, resListener[0].Spec.DefaultAction.BackendServiceName, - string(tt.route.Spec().Rules()[0].BackendRefs()[0].Name())) - if ns := tt.route.Spec().Rules()[0].BackendRefs()[0].Namespace(); ns != nil { - assert.Equal(t, resListener[0].Spec.DefaultAction.BackendServiceNamespace, *ns) - } else { - assert.Equal(t, resListener[0].Spec.DefaultAction.BackendServiceNamespace, tt.route.Namespace()) - } + for i, expected := range tt.expectedSpec { + actual := resListener[i].Spec - if tt.tlsTerminate && !tt.noTLSOption && !tt.wrongTLSOption { - assert.Equal(t, task.latticeService.Spec.CustomerCertARN, tt.certARN) - } else { - assert.Equal(t, task.latticeService.Spec.CustomerCertARN, "") + assert.Equal(t, expected.StackServiceId, actual.StackServiceId) + assert.Equal(t, expected.K8SRouteName, actual.K8SRouteName) + assert.Equal(t, expected.K8SRouteNamespace, actual.K8SRouteNamespace) + assert.Equal(t, expected.Port, actual.Port) + assert.Equal(t, expected.Protocol, actual.Protocol) } }) } diff --git a/pkg/gateway/model_build_rule.go b/pkg/gateway/model_build_rule.go index 7e055a52..c0f692b9 100644 --- a/pkg/gateway/model_build_rule.go +++ b/pkg/gateway/model_build_rule.go @@ -4,6 +4,9 @@ import ( "context" "errors" "fmt" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" "github.com/aws/aws-application-networking-k8s/pkg/model/core" @@ -26,35 +29,20 @@ const ( LATTICE_MAX_HEADER_MATCHES = 5 ) -func (t *latticeServiceModelBuildTask) buildRules(ctx context.Context) error { - var ruleID = 1 - for _, parentRef := range t.route.Spec().ParentRefs() { - if parentRef.Name != t.route.Spec().ParentRefs()[0].Name { - // when a service is associate to multiple service network(s), all listener config MUST be same - // so here we are only using the 1st gateway - t.log.Debugf("Ignore parentref of different gateway %s-%s", parentRef.Name, *parentRef.Namespace) - continue - } +func (t *latticeServiceModelBuildTask) buildRules(ctx context.Context, stackListenerId string) error { + t.log.Debugf("Processing %d rules", len(t.route.Spec().Rules())) - port, protocol, _, err := t.extractListenerInfo(ctx, parentRef) - if err != nil { - return err + for i, rule := range t.route.Spec().Rules() { + ruleSpec := model.RuleSpec{ + StackListenerId: stackListenerId, + Priority: int64(i + 1), } - for _, rule := range t.route.Spec().Rules() { - var ruleSpec model.RuleSpec - - if len(rule.Matches()) > 1 { - // only support 1 match today - return errors.New(LATTICE_NO_SUPPORT_FOR_MULTIPLE_MATCHES) - } - - if len(rule.Matches()) == 0 { - t.log.Debugf("Continue next rule, no matches specified in current rule") - continue - } - + if len(rule.Matches()) > 1 { // only support 1 match today + return errors.New(LATTICE_NO_SUPPORT_FOR_MULTIPLE_MATCHES) + } else if len(rule.Matches()) > 0 { + t.log.Debugf("Processing rule match") match := rule.Matches()[0] switch m := match.(type) { @@ -73,16 +61,27 @@ func (t *latticeServiceModelBuildTask) buildRules(ctx context.Context) error { if err := t.updateRuleSpecWithHeaderMatches(match, &ruleSpec); err != nil { return err } + } - tgList := t.getTargetGroupsForRuleAction(rule) + ruleTgList, err := t.getTargetGroupsForRuleAction(ctx, rule) + if err != nil { + return err + } + + ruleSpec.Action = model.RuleAction{ + TargetGroups: ruleTgList, + } - ruleIDName := fmt.Sprintf("rule-%d", ruleID) - ruleAction := model.RuleAction{ - TargetGroups: tgList, + // don't bother adding rules on delete, these will be removed automatically with the owning route/lattice service + // target groups will still be present and removed as needed + if t.route.DeletionTimestamp().IsZero() { + stackRule, err := model.NewRule(t.stack, ruleSpec) + if err != nil { + return err } - model.NewRule(t.stack, ruleIDName, t.route.Name(), t.route.Namespace(), port, - protocol, ruleAction, ruleSpec) - ruleID++ + t.log.Debugf("Added rule %d to the stack (ID %s)", stackRule.Spec.Priority, stackRule.ID()) + } else { + t.log.Debugf("Skipping adding rule %d to the stack since the route is deleted", ruleSpec.Priority) } } @@ -90,7 +89,14 @@ func (t *latticeServiceModelBuildTask) buildRules(ctx context.Context) error { } func (t *latticeServiceModelBuildTask) updateRuleSpecForHttpRoute(m *core.HTTPRouteMatch, ruleSpec *model.RuleSpec) error { - if m.Path() != nil && m.Path().Type != nil { + hasPath := m.Path() != nil + hasType := hasPath && m.Path().Type != nil + + if hasPath && !hasType { + return errors.New("type is required on path match") + } + + if hasPath { t.log.Debugf("Examining pathmatch type %s value %s for for httproute %s-%s ", *m.Path().Type, *m.Path().Value, t.route.Name(), t.route.Namespace()) @@ -131,7 +137,7 @@ func (t *latticeServiceModelBuildTask) updateRuleSpecForHttpRoute(m *core.HTTPRo func (t *latticeServiceModelBuildTask) updateRuleSpecForGrpcRoute(m *core.GRPCRouteMatch, ruleSpec *model.RuleSpec) error { t.log.Debugf("Building rule with GRPCRouteMatch, %+v", *m) - ruleSpec.Method = string(gwv1beta1.HTTPMethodPost) + ruleSpec.Method = string(gwv1beta1.HTTPMethodPost) // GRPC is always POST method := m.Method() // VPC Lattice doesn't support suffix/regex matching, so we can't support method match without service if method.Service == nil && method.Method != nil { @@ -167,12 +173,10 @@ func (t *latticeServiceModelBuildTask) updateRuleSpecWithHeaderMatches(match cor return errors.New(LATTICE_EXCEED_MAX_HEADER_MATCHES) } - ruleSpec.NumOfHeaderMatches = len(match.Headers()) - t.log.Debugf("Examining match headers for route %s-%s", t.route.Name(), t.route.Namespace()) - for i, header := range match.Headers() { - t.log.Debugf("Examining match.Header: i = %d header.Type %s", i, *header.Type()) + for _, header := range match.Headers() { + t.log.Debugf("Examining match.Header: header.Type %s", *header.Type()) if header.Type() != nil && *header.Type() != gwv1beta1.HeaderMatchExact { t.log.Debugf("Unsupported header matchtype %s for httproute %s-%s", *header.Type(), t.route.Name(), t.route.Namespace()) @@ -182,48 +186,80 @@ func (t *latticeServiceModelBuildTask) updateRuleSpecWithHeaderMatches(match cor matchType := vpclattice.HeaderMatchType{ Exact: aws.String(header.Value()), } - ruleSpec.MatchedHeaders[i].Match = &matchType headerName := header.Name() - ruleSpec.MatchedHeaders[i].Name = &headerName + + headerMatch := vpclattice.HeaderMatch{} + headerMatch.Match = &matchType + headerMatch.Name = &headerName + + ruleSpec.MatchedHeaders = append(ruleSpec.MatchedHeaders, headerMatch) } + return nil } -func (t *latticeServiceModelBuildTask) getTargetGroupsForRuleAction(rule core.RouteRule) []*model.RuleTargetGroup { +func (t *latticeServiceModelBuildTask) getTargetGroupsForRuleAction(ctx context.Context, rule core.RouteRule) ([]*model.RuleTargetGroup, error) { var tgList []*model.RuleTargetGroup for _, backendRef := range rule.BackendRefs() { - ruleTG := model.RuleTargetGroup{} - if string(*backendRef.Kind()) == "Service" { - namespace := t.route.Namespace() - if backendRef.Namespace() != nil { - namespace = string(*backendRef.Namespace()) - } - ruleTG.Name = string(backendRef.Name()) - ruleTG.Namespace = namespace - ruleTG.RouteName = t.route.Name() - ruleTG.IsServiceImport = false - if backendRef.Weight() != nil { - ruleTG.Weight = int64(*backendRef.Weight()) - } + ruleTG := model.RuleTargetGroup{ + Weight: 1, // default value according to spec + } + if backendRef.Weight() != nil { + ruleTG.Weight = int64(*backendRef.Weight()) + } + + namespace := t.route.Namespace() + if backendRef.Namespace() != nil { + namespace = string(*backendRef.Namespace()) } + t.log.Debugf("Processing %s backendRef %s-%s", string(*backendRef.Kind()), backendRef.Name(), namespace) + if string(*backendRef.Kind()) == "ServiceImport" { - ruleTG.Name = string(backendRef.Name()) - ruleTG.Namespace = t.route.Namespace() - if backendRef.Namespace() != nil { - ruleTG.Namespace = string(*backendRef.Namespace()) + // there needs to be a pre-existing target group, we fetch all the fields + // needed to identify it + svcImportTg := model.SvcImportTargetGroup{ + K8SServiceNamespace: namespace, + K8SServiceName: string(backendRef.Name()), + } + + // if there's a matching top-level service import, we can get additional fields + svcImportName := types.NamespacedName{ + Namespace: namespace, + Name: string(backendRef.Name()), + } + svcImport := &mcsv1alpha1.ServiceImport{} + if err := t.client.Get(ctx, svcImportName, svcImport); err != nil { + if !apierrors.IsNotFound(err) { + return nil, err + } + } + if svcImport != nil { + vpc, ok := svcImport.Annotations["multicluster.x-k8s.io/aws-vpc"] + if ok { + svcImportTg.VpcId = vpc + } + + eksCluster, ok := svcImport.Annotations["multicluster.x-k8s.io/aws-eks-cluster-name"] + if ok { + svcImportTg.EKSClusterName = eksCluster + } } - // the routeName for serviceimport is always "" - ruleTG.RouteName = "" - ruleTG.IsServiceImport = true + ruleTG.SvcImportTG = &svcImportTg + } - if backendRef.Weight() != nil { - ruleTG.Weight = int64(*backendRef.Weight()) + if string(*backendRef.Kind()) == "Service" { + // generate the actual target group model for the backendRef + _, tg, err := t.brTgBuilder.Build(ctx, t.route, backendRef, t.stack) + if err != nil { + return nil, err } + ruleTG.StackTargetGroupId = tg.ID() } tgList = append(tgList, &ruleTG) } - return tgList + + return tgList, nil } diff --git a/pkg/gateway/model_build_rule_test.go b/pkg/gateway/model_build_rule_test.go index 5aee27bc..3feb9080 100644 --- a/pkg/gateway/model_build_rule_test.go +++ b/pkg/gateway/model_build_rule_test.go @@ -2,36 +2,48 @@ package gateway import ( "context" - "errors" "fmt" + "github.com/aws/aws-application-networking-k8s/pkg/k8s" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - "k8s.io/utils/pointer" - "testing" - + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/vpclattice" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - - "github.com/aws/aws-sdk-go/service/vpclattice" - + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "k8s.io/utils/pointer" + "reflect" + testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" + "testing" +) - "k8s.io/apimachinery/pkg/types" - - mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" +type dummyTgBuilder struct { + i int +} - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" +func (d *dummyTgBuilder) Build(ctx context.Context, route core.Route, backendRef core.BackendRef, stack core.Stack) (core.Stack, *model.TargetGroup, error) { + // just need to provide a TG with an ID + id := fmt.Sprintf("tg-%d", d.i) + d.i++ - "github.com/aws/aws-application-networking-k8s/pkg/k8s" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" -) + tg := &model.TargetGroup{ + ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", id), + } + stack.AddResource(tg) + return stack, tg, nil +} func Test_RuleModelBuild(t *testing.T) { var httpSectionName gwv1beta1.SectionName = "http" var serviceKind gwv1beta1.Kind = "Service" - var serviceimportKind gwv1beta1.Kind = "ServiceImport" + var serviceImportKind gwv1beta1.Kind = "ServiceImport" var weight1 = int32(10) var weight2 = int32(90) var namespace = gwv1beta1.Namespace("testnamespace") @@ -42,7 +54,14 @@ func Test_RuleModelBuild(t *testing.T) { var httpGet = gwv1beta1.HTTPMethodGet var httpPost = gwv1beta1.HTTPMethodPost var k8sPathMatchExactType = gwv1beta1.PathMatchExact + var k8sPathMatchPrefix = gwv1beta1.PathMatchPathPrefix var k8sMethodMatchExactType = gwv1alpha2.GRPCMethodMatchExact + var k8sHeaderExactType = gwv1beta1.HeaderMatchExact + var hdr1 = "env1" + var hdr1Value = "test1" + var hdr2 = "env2" + var hdr2Value = "test2" + var backendRef1 = gwv1beta1.BackendRef{ BackendObjectReference: gwv1beta1.BackendObjectReference{ Name: "targetgroup1", @@ -53,7 +72,7 @@ func Test_RuleModelBuild(t *testing.T) { var backendRef2 = gwv1beta1.BackendRef{ BackendObjectReference: gwv1beta1.BackendObjectReference{ Name: "targetgroup2", - Kind: &serviceimportKind, + Kind: &serviceImportKind, }, Weight: &weight2, } @@ -61,7 +80,7 @@ func Test_RuleModelBuild(t *testing.T) { BackendObjectReference: gwv1beta1.BackendObjectReference{ Name: "targetgroup2", Namespace: &namespace, - Kind: &serviceimportKind, + Kind: &serviceImportKind, }, Weight: &weight2, } @@ -69,31 +88,26 @@ func Test_RuleModelBuild(t *testing.T) { BackendObjectReference: gwv1beta1.BackendObjectReference{ Name: "targetgroup2", Namespace: &namespace2, - Kind: &serviceimportKind, + Kind: &serviceImportKind, }, Weight: &weight2, } var backendServiceImportRef = gwv1beta1.BackendRef{ BackendObjectReference: gwv1beta1.BackendObjectReference{ Name: "targetgroup1", - Kind: &serviceimportKind, + Kind: &serviceImportKind, }, } tests := []struct { - name string - gwListenerPort gwv1beta1.PortNumber - route core.Route - wantErrIsNil bool - k8sGetGatewayCall bool - k8sGatewayReturnOK bool + name string + route core.Route + wantErrIsNil bool + expectedSpec []model.RuleSpec }{ { - name: "rule, default service action", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, + name: "rule, default service action", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -119,13 +133,23 @@ func Test_RuleModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.RuleSpec{ // note priority is only calculated at synthesis b/c it requires access to existing rules + { + StackListenerId: "listener-id", + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(weight1), + }, + }, + }, + }, + }, }, { - name: "rule, default serviceimport action", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, + name: "rule, default serviceimport action", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -151,13 +175,26 @@ func Test_RuleModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendServiceImportRef.Name), + K8SServiceNamespace: "default", + }, + Weight: 1, + }, + }, + }, + }, + }, }, { - name: "rule, weighted target group", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, + name: "rule, weighted target group", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -186,13 +223,30 @@ func Test_RuleModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(weight1), + }, + { + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendRef2.Name), + K8SServiceNamespace: "default", + }, + Weight: int64(weight2), + }, + }, + }, + }, + }, }, { - name: "rule, path based target group", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, + name: "rule, path based target group", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -211,7 +265,6 @@ func Test_RuleModelBuild(t *testing.T) { { Matches: []gwv1beta1.HTTPRouteMatch{ { - Path: &gwv1beta1.HTTPPathMatch{ Type: &k8sPathMatchExactType, Value: &path1, @@ -227,9 +280,8 @@ func Test_RuleModelBuild(t *testing.T) { { Matches: []gwv1beta1.HTTPRouteMatch{ { - Path: &gwv1beta1.HTTPPathMatch{ - + Type: &k8sPathMatchPrefix, Value: &path2, }, }, @@ -243,13 +295,41 @@ func Test_RuleModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: path1, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(weight1), + }, + }, + }, + }, + { + StackListenerId: "listener-id", + PathMatchPrefix: true, + PathMatchValue: path2, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendRef2.Name), + K8SServiceNamespace: "default", + }, + Weight: int64(weight2), + }, + }, + }, + }, + }, }, { - name: "rule, method based", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, + name: "rule, method based", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -292,13 +372,39 @@ func Test_RuleModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + Method: string(httpGet), + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(weight1), + }, + }, + }, + }, + { + StackListenerId: "listener-id", + Method: string(httpPost), + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendRef2.Name), + K8SServiceNamespace: "default", + }, + Weight: int64(weight2), + }, + }, + }, + }, + }, }, { - name: "rule, different namespace combination", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, + name: "rule, different namespace combination", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -319,6 +425,7 @@ func Test_RuleModelBuild(t *testing.T) { { Path: &gwv1beta1.HTTPPathMatch{ Value: &path1, + Type: &k8sPathMatchExactType, }, }, }, @@ -333,6 +440,7 @@ func Test_RuleModelBuild(t *testing.T) { { Path: &gwv1beta1.HTTPPathMatch{ Value: &path2, + Type: &k8sPathMatchExactType, }, }, }, @@ -347,6 +455,7 @@ func Test_RuleModelBuild(t *testing.T) { { Path: &gwv1beta1.HTTPPathMatch{ Value: &path3, + Type: &k8sPathMatchExactType, }, }, }, @@ -359,13 +468,57 @@ func Test_RuleModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: path1, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(weight1), + }, + }, + }, + }, + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: path2, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendRef1Namespace1.Name), + K8SServiceNamespace: string(*backendRef1Namespace1.Namespace), + }, + Weight: int64(weight2), + }, + }, + }, + }, + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: path3, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendRef1Namespace2.Name), + K8SServiceNamespace: string(*backendRef1Namespace2.Namespace), + }, + Weight: int64(weight2), + }, + }, + }, + }, + }, }, { - name: "rule, default service import action for GRPCRoute", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, + name: "rule, default service import action for GRPCRoute", + wantErrIsNil: true, route: core.NewGRPCRoute(gwv1alpha2.GRPCRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -391,13 +544,26 @@ func Test_RuleModelBuild(t *testing.T) { }, }, }), + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendServiceImportRef.Name), + K8SServiceNamespace: "default", + }, + Weight: 1, + }, + }, + }, + }, + }, }, { - name: "rule, gRPC routes with methods and multiple namespaces", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - k8sGetGatewayCall: true, - k8sGatewayReturnOK: true, + name: "rule, gRPC routes with methods and multiple namespaces", + wantErrIsNil: true, route: core.NewGRPCRoute(gwv1alpha2.GRPCRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -464,220 +630,60 @@ func Test_RuleModelBuild(t *testing.T) { }, }, }), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - c := gomock.NewController(t) - defer c.Finish() - ctx := context.TODO() - - mockK8sClient := mock_client.NewMockClient(c) - - if tt.k8sGetGatewayCall { - - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, gwName types.NamespacedName, gw *gwv1beta1.Gateway, arg3 ...interface{}) error { - - if tt.k8sGatewayReturnOK { - gw.Spec.Listeners = append(gw.Spec.Listeners, gwv1beta1.Listener{ - Port: tt.gwListenerPort, - Name: *tt.route.Spec().ParentRefs()[0].SectionName, - }) - return nil - } else { - return errors.New("unknown k8s object") - } - }, - ) - } - - ds := latticestore.NewLatticeDataStore() - - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.route.K8sObject()))) - - task := &latticeServiceModelBuildTask{ - log: gwlog.FallbackLogger, - route: tt.route, - stack: stack, - client: mockK8sClient, - listenerByResID: make(map[string]*model.Listener), - datastore: ds, - } - - err := task.buildRules(ctx) - - assert.NoError(t, err) - - var resRules []*model.Rule - - stack.ListResources(&resRules) - - if len(resRules) > 0 { - fmt.Printf("resRules :%v \n", *resRules[0]) - } - - var i = 1 - for _, resRule := range resRules { - - fmt.Sscanf(resRule.Spec.RuleID, "rule-%d", &i) - - assert.Equal(t, resRule.Spec.ListenerPort, int64(tt.gwListenerPort)) - // Defer this to dedicate rule check assert.Equal(t, resRule.Spec.PathMatchValue, tt.route.) - assert.Equal(t, resRule.Spec.ServiceName, tt.route.Name()) - assert.Equal(t, resRule.Spec.ServiceNamespace, tt.route.Namespace()) - - var j = 0 - for _, tg := range resRule.Spec.Action.TargetGroups { - - assert.Equal(t, gwv1beta1.ObjectName(tg.Name), tt.route.Spec().Rules()[i-1].BackendRefs()[j].Name()) - if tt.route.Spec().Rules()[i-1].BackendRefs()[j].Namespace() != nil { - assert.Equal(t, gwv1beta1.Namespace(tg.Namespace), *tt.route.Spec().Rules()[i-1].BackendRefs()[j].Namespace()) - } else { - assert.Equal(t, tg.Namespace, tt.route.Namespace()) - } - - if *tt.route.Spec().Rules()[i-1].BackendRefs()[j].Kind() == "ServiceImport" { - assert.Equal(t, tg.IsServiceImport, true) - } else { - assert.Equal(t, tg.IsServiceImport, false) - } - j++ - } - } - }) - } -} - -func Test_HeadersRuleBuild(t *testing.T) { - var httpSectionName gwv1beta1.SectionName = "http" - var serviceKind gwv1beta1.Kind = "Service" - - var namespace = gwv1beta1.Namespace("default") - var path1 = "/ver1" - var k8sPathMatchExactType = gwv1beta1.PathMatchExact - var k8sPathMatchPrefixType = gwv1beta1.PathMatchPathPrefix - var k8sMethodMatchExactType = gwv1alpha2.GRPCMethodMatchExact - - var k8sHeaderExactType = gwv1beta1.HeaderMatchExact - var hdr1 = "env1" - var hdr1Value = "test1" - var hdr2 = "env2" - var hdr2Value = "test2" - - var backendRef1 = gwv1beta1.BackendRef{ - BackendObjectReference: gwv1beta1.BackendObjectReference{ - Name: "targetgroup1", - Namespace: &namespace, - Kind: &serviceKind, - }, - } - - tests := []struct { - name string - gwListenerPort gwv1beta1.PortNumber - route core.Route - expectedRuleSpec model.RuleSpec - wantErrIsNil bool - samerule bool - }{ - { - name: "PathMatchExact Match", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - - route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", - Namespace: "default", - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: "/service/method1", + Method: string(httpPost), + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ { - Name: "gw1", - SectionName: &httpSectionName, + StackTargetGroupId: "tg-0", + Weight: int64(weight1), }, }, }, - Rules: []gwv1beta1.HTTPRouteRule{ - { - Matches: []gwv1beta1.HTTPRouteMatch{ - { - - Path: &gwv1beta1.HTTPPathMatch{ - Type: &k8sPathMatchExactType, - Value: &path1, - }, - }, - }, - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef1, + }, + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: "/service/method2", + Method: string(httpPost), + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendRef1Namespace1.Name), + K8SServiceNamespace: string(*backendRef1Namespace1.Namespace), }, + Weight: int64(weight2), }, }, }, }, - }), - expectedRuleSpec: model.RuleSpec{ - PathMatchExact: true, - PathMatchValue: path1, - }, - }, - - { - name: "PathMatchPrefix", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - - route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", - Namespace: "default", - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: "/service/method3", + Method: string(httpPost), + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ { - Name: "gw1", - SectionName: &httpSectionName, - }, - }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ - { - Matches: []gwv1beta1.HTTPRouteMatch{ - { - - Path: &gwv1beta1.HTTPPathMatch{ - Type: &k8sPathMatchPrefixType, - Value: &path1, - }, - }, - }, - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef1, + SvcImportTG: &model.SvcImportTargetGroup{ + K8SServiceName: string(backendRef1Namespace2.Name), + K8SServiceNamespace: string(*backendRef1Namespace2.Namespace), }, + Weight: int64(weight2), }, }, }, }, - }), - expectedRuleSpec: model.RuleSpec{ - PathMatchPrefix: true, - PathMatchValue: path1, }, }, { - name: "1 header match", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - + name: "1 header match", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -696,11 +702,6 @@ func Test_HeadersRuleBuild(t *testing.T) { { Matches: []gwv1beta1.HTTPRouteMatch{ { - - // Path: &gwv1beta1.HTTPPathMatch{ - // Type: &k8sPathMatchPrefixType, - // Value: &path1, - // }, Headers: []gwv1beta1.HTTPHeaderMatch{ { Type: &k8sHeaderExactType, @@ -719,29 +720,31 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - NumOfHeaderMatches: 1, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + MatchedHeaders: []vpclattice.HeaderMatch{ + { + Name: &hdr1, + Match: &vpclattice.HeaderMatchType{ + Exact: &hdr1Value, + }, + }, + }, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(*backendRef1.Weight), + }, + }, }, - - {}, - {}, - {}, - {}, }, }, }, { - name: "2 header match", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - + name: "2 header match", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -783,33 +786,37 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + MatchedHeaders: []vpclattice.HeaderMatch{ + { + Name: &hdr1, + Match: &vpclattice.HeaderMatchType{ + Exact: &hdr1Value, + }, + }, + { + Name: &hdr2, + Match: &vpclattice.HeaderMatchType{ + Exact: &hdr2Value, + }, + }, }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(*backendRef1.Weight), + }, + }, }, - - {}, - {}, - {}, }, }, }, { - name: "2 header match , path exact", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - + name: "2 header match with path exact", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -856,35 +863,39 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - PathMatchExact: true, - PathMatchValue: path1, - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: path1, + MatchedHeaders: []vpclattice.HeaderMatch{ + { + Name: &hdr1, + Match: &vpclattice.HeaderMatchType{ + Exact: &hdr1Value, + }, + }, + { + Name: &hdr2, + Match: &vpclattice.HeaderMatchType{ + Exact: &hdr2Value, + }, + }, }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(*backendRef1.Weight), + }, + }, }, - - {}, - {}, - {}, }, }, }, { - name: "2 header match , path prefix", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - + name: "2 header match with path prefix", + wantErrIsNil: true, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -905,7 +916,7 @@ func Test_HeadersRuleBuild(t *testing.T) { { Path: &gwv1beta1.HTTPPathMatch{ - Type: &k8sPathMatchPrefixType, + Type: &k8sPathMatchPrefix, Value: &path1, }, Headers: []gwv1beta1.HTTPHeaderMatch{ @@ -931,103 +942,39 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - PathMatchPrefix: true, - PathMatchValue: path1, - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, - }, - { - name: "2 header match", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - - route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "service1", - Namespace: "default", - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gw1", - SectionName: &httpSectionName, + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchPrefix: true, + PathMatchValue: path1, + MatchedHeaders: []vpclattice.HeaderMatch{ + { + Name: &hdr1, + Match: &vpclattice.HeaderMatchType{ + Exact: &hdr1Value, }, }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ { - Matches: []gwv1beta1.HTTPRouteMatch{ - { - Headers: []gwv1beta1.HTTPHeaderMatch{ - { - Type: &k8sHeaderExactType, - Name: gwv1beta1.HTTPHeaderName(hdr1), - Value: hdr1Value, - }, - { - Type: &k8sHeaderExactType, - Name: gwv1beta1.HTTPHeaderName(hdr2), - Value: hdr2Value, - }, - }, - }, - }, - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: backendRef1, - }, + Name: &hdr2, + Match: &vpclattice.HeaderMatchType{ + Exact: &hdr2Value, }, }, }, - }, - }), - expectedRuleSpec: model.RuleSpec{ - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(*backendRef1.Weight), + }, + }, }, - - {}, - {}, - {}, }, }, }, { - name: " negative 6 header match ", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - samerule: true, - + name: " negative 6 header match (max headers is 5)", + wantErrIsNil: false, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -1094,35 +1041,10 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - PathMatchExact: true, - PathMatchValue: path1, - NumOfHeaderMatches: 2, - MatchedHeaders: [5]vpclattice.HeaderMatch{ - - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr1Value}, - Name: &hdr1, - }, - { - Match: &vpclattice.HeaderMatchType{ - Exact: &hdr2Value}, - Name: &hdr2, - }, - - {}, - {}, - {}, - }, - }, }, { - name: "Negative, multiple methods", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: true, - samerule: true, - + name: "Negative, multiple methods", + wantErrIsNil: false, route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -1164,17 +1086,10 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - PathMatchExact: true, - PathMatchValue: path1, - }, }, { - name: "GRPC match on service and method", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - + name: "GRPC match on service and method", + wantErrIsNil: true, route: core.NewGRPCRoute(gwv1alpha2.GRPCRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -1200,7 +1115,6 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }, - BackendRefs: []gwv1alpha2.GRPCBackendRef{ { BackendRef: backendRef1, @@ -1210,19 +1124,26 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - Method: "POST", - NumOfHeaderMatches: 0, - PathMatchExact: true, - PathMatchValue: "/service/method", + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchExact: true, + PathMatchValue: "/service/method", + Method: "POST", + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(*backendRef1.Weight), + }, + }, + }, + }, }, }, { - name: "GRPC match on service", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - + name: "GRPC match on service", + wantErrIsNil: true, route: core.NewGRPCRoute(gwv1alpha2.GRPCRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -1247,7 +1168,6 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }, - BackendRefs: []gwv1alpha2.GRPCBackendRef{ { BackendRef: backendRef1, @@ -1257,19 +1177,26 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - Method: "POST", - NumOfHeaderMatches: 0, - PathMatchPrefix: true, - PathMatchValue: "/service/", + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchPrefix: true, + PathMatchValue: "/service/", + Method: "POST", + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(*backendRef1.Weight), + }, + }, + }, + }, }, }, { - name: "GRPC match on all", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - + name: "GRPC match on all", + wantErrIsNil: true, route: core.NewGRPCRoute(gwv1alpha2.GRPCRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -1303,19 +1230,26 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - Method: "POST", - NumOfHeaderMatches: 0, - PathMatchPrefix: true, - PathMatchValue: "/", + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchPrefix: true, + PathMatchValue: "/", + Method: "POST", + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(*backendRef1.Weight), + }, + }, + }, + }, }, }, { - name: "GRPC match with 5 headers", - gwListenerPort: *PortNumberPtr(80), - wantErrIsNil: false, - samerule: true, - + name: "GRPC match with 5 headers", + wantErrIsNil: true, route: core.NewGRPCRoute(gwv1alpha2.GRPCRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service1", @@ -1376,180 +1310,126 @@ func Test_HeadersRuleBuild(t *testing.T) { }, }, }), - expectedRuleSpec: model.RuleSpec{ - Method: "POST", - NumOfHeaderMatches: 5, - PathMatchPrefix: true, - PathMatchValue: "/service/", - MatchedHeaders: [model.MAX_NUM_OF_MATCHED_HEADERS]vpclattice.HeaderMatch{ - { - Name: pointer.String("foo1"), - Match: &vpclattice.HeaderMatchType{ - Exact: pointer.String("bar1"), + expectedSpec: []model.RuleSpec{ + { + StackListenerId: "listener-id", + PathMatchPrefix: true, + PathMatchValue: "/service/", + Method: "POST", + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + StackTargetGroupId: "tg-0", + Weight: int64(*backendRef1.Weight), + }, }, }, - { - Name: pointer.String("foo2"), - Match: &vpclattice.HeaderMatchType{ - Exact: pointer.String("bar2"), + MatchedHeaders: []vpclattice.HeaderMatch{ + { + Name: aws.String("foo1"), + Match: &vpclattice.HeaderMatchType{ + Exact: aws.String("bar1"), + }, }, - }, - { - Name: pointer.String("foo3"), - Match: &vpclattice.HeaderMatchType{ - Exact: pointer.String("bar3"), + { + Name: aws.String("foo2"), + Match: &vpclattice.HeaderMatchType{ + Exact: aws.String("bar2"), + }, }, - }, - { - Name: pointer.String("foo4"), - Match: &vpclattice.HeaderMatchType{ - Exact: pointer.String("bar4"), + { + Name: aws.String("foo3"), + Match: &vpclattice.HeaderMatchType{ + Exact: aws.String("bar3"), + }, }, - }, - { - Name: pointer.String("foo5"), - Match: &vpclattice.HeaderMatchType{ - Exact: pointer.String("bar5"), + { + Name: aws.String("foo4"), + Match: &vpclattice.HeaderMatchType{ + Exact: aws.String("bar4"), + }, + }, + { + Name: aws.String("foo5"), + Match: &vpclattice.HeaderMatchType{ + Exact: aws.String("bar5"), + }, }, }, }, }, }, } - for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := gomock.NewController(t) defer c.Finish() ctx := context.TODO() - mockK8sClient := mock_client.NewMockClient(c) - - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, gwName types.NamespacedName, gw *gwv1beta1.Gateway, arg3 ...interface{}) error { - - gw.Spec.Listeners = append(gw.Spec.Listeners, gwv1beta1.Listener{ - Port: tt.gwListenerPort, - Name: *tt.route.Spec().ParentRefs()[0].SectionName, - }) - return nil + k8sSchema := runtime.NewScheme() + k8sSchema.AddKnownTypes(mcsv1alpha1.SchemeGroupVersion, &mcsv1alpha1.ServiceImport{}) + clientgoscheme.AddToScheme(k8sSchema) + k8sClient := testclient.NewFakeClientWithScheme(k8sSchema) + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: string(backendRef1.Name), + Namespace: "default", }, - ) - - ds := latticestore.NewLatticeDataStore() - + Status: corev1.ServiceStatus{}, + } + assert.NoError(t, k8sClient.Create(ctx, svc.DeepCopy())) stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.route.K8sObject()))) task := &latticeServiceModelBuildTask{ - log: gwlog.FallbackLogger, - route: tt.route, - stack: stack, - client: mockK8sClient, - listenerByResID: make(map[string]*model.Listener), - datastore: ds, + log: gwlog.FallbackLogger, + route: tt.route, + stack: stack, + client: k8sClient, + brTgBuilder: &dummyTgBuilder{}, } - err := task.buildRules(ctx) - + err := task.buildRules(ctx, "listener-id") if tt.wantErrIsNil { - assert.Error(t, err) + assert.NoError(t, err) + } else { + assert.NotNil(t, err) return } - assert.NoError(t, err) var resRules []*model.Rule stack.ListResources(&resRules) - if len(resRules) > 0 { - // for debug fmt.Printf("resRules :%v \n", *resRules[0]) - } - - // we are unit- testing various combination of one rule for now - var i = 1 - for _, resRule := range resRules { - - // for debugging, fmt.Printf("i = %d resRule :%v \n, expected rule: %v\n", i, resRule, tt.expectedRuleSpec) - - sameRule := isRuleSpecSame(&tt.expectedRuleSpec, &resRule.Spec) - - if tt.samerule { - assert.True(t, sameRule) - } else { - assert.False(t, sameRule) - } - i++ - - } + validateEqual(t, tt.expectedSpec, resRules) }) } } -func isRuleSpecSame(rule1 *model.RuleSpec, rule2 *model.RuleSpec) bool { - - // debug fmt.Printf("rule1 :%v \n", rule1) - // debug fmt.Printf("rule2: %v \n", rule2) - // Path Exact Match - if rule1.PathMatchExact || rule2.PathMatchExact { - if rule1.PathMatchExact != rule2.PathMatchExact { - return false - } - - if rule1.PathMatchValue != rule2.PathMatchValue { - return false - } - } - - // Path Prefix Match - if rule1.PathMatchPrefix || rule2.PathMatchPrefix { - if rule1.PathMatchPrefix != rule2.PathMatchPrefix { - return false - } - if rule1.PathMatchValue != rule2.PathMatchValue { - return false - } - } - - // Method match - if rule1.Method != rule2.Method { - return false - } +func validateEqual(t *testing.T, expectedRules []model.RuleSpec, actualRules []*model.Rule) { + assert.Equal(t, len(expectedRules), len(actualRules)) + assert.Equal(t, len(expectedRules), len(actualRules)) - // Header Match - if rule1.NumOfHeaderMatches != rule2.NumOfHeaderMatches { - return false - } + for i, expectedSpec := range expectedRules { + actualRule := actualRules[i] - // Verify each header - for i := 0; i < rule1.NumOfHeaderMatches; i++ { - // verify rule1 header is in rule2 - rule1Hdr := rule1.MatchedHeaders[i] + assert.Equal(t, expectedSpec.StackListenerId, actualRule.Spec.StackListenerId) + assert.Equal(t, expectedSpec.PathMatchValue, actualRule.Spec.PathMatchValue) + assert.Equal(t, expectedSpec.PathMatchPrefix, actualRule.Spec.PathMatchPrefix) + assert.Equal(t, expectedSpec.PathMatchExact, actualRule.Spec.PathMatchExact) + assert.Equal(t, expectedSpec.Method, actualRule.Spec.Method) - found := false - for j := 0; j < rule2.NumOfHeaderMatches; j++ { + // priority is not determined by model building, but in synthesis, so we don't + // validate priority here - // fmt.Printf("rule1 match :%v\n", rule1Hdr) - rule2Hdr := rule2.MatchedHeaders[j] - // fmt.Printf("rule2 match: %v\n", rule2Hdr) + assert.True(t, reflect.DeepEqual(expectedSpec.MatchedHeaders, actualRule.Spec.MatchedHeaders)) - if *rule1Hdr.Name == *rule2Hdr.Name { - if rule1Hdr.Match.Exact != nil && rule2Hdr.Match.Exact != nil { - if *rule1Hdr.Match.Exact == *rule2Hdr.Match.Exact { - found = true - break - } - } else if rule1Hdr.Match.Prefix != nil && rule2Hdr.Match.Prefix != nil { - if *rule1Hdr.Match.Prefix == *rule2Hdr.Match.Prefix { - found = true - break - } - } - } + assert.Equal(t, len(expectedSpec.Action.TargetGroups), len(actualRule.Spec.Action.TargetGroups)) + for j, etg := range expectedSpec.Action.TargetGroups { + atg := actualRule.Spec.Action.TargetGroups[j] - } - if !found { - return false + assert.Equal(t, etg.Weight, atg.Weight) + assert.Equal(t, etg.StackTargetGroupId, atg.StackTargetGroupId) + assert.Equal(t, etg.SvcImportTG, etg.SvcImportTG) } } - return true } diff --git a/pkg/gateway/model_build_targetgroup.go b/pkg/gateway/model_build_targetgroup.go index 154f3636..bd0f6b2f 100644 --- a/pkg/gateway/model_build_targetgroup.go +++ b/pkg/gateway/model_build_targetgroup.go @@ -4,8 +4,8 @@ import ( "context" "errors" "fmt" - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + apierrors "k8s.io/apimachinery/pkg/api/errors" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -16,38 +16,32 @@ import ( "github.com/aws/aws-sdk-go/service/vpclattice" anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" - pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-application-networking-k8s/pkg/config" "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) type SvcExportTargetGroupModelBuilder interface { - Build(ctx context.Context, srvExport *mcsv1alpha1.ServiceExport) (core.Stack, *model.TargetGroup, error) + // used during standard model build + Build(ctx context.Context, svcExport *mcsv1alpha1.ServiceExport) (core.Stack, error) + + // used for reconciliation of existing target groups against a service export object + BuildTargetGroup(ctx context.Context, svcExport *mcsv1alpha1.ServiceExport) (*model.TargetGroup, error) } type SvcExportTargetGroupBuilder struct { - log gwlog.Logger - client client.Client - serviceExport *mcsv1alpha1.ServiceExport - datastore *latticestore.LatticeDataStore - cloud pkg_aws.Cloud - defaultTags map[string]string + log gwlog.Logger + client client.Client } func NewSvcExportTargetGroupBuilder( log gwlog.Logger, client client.Client, - datastore *latticestore.LatticeDataStore, - cloud pkg_aws.Cloud, ) *SvcExportTargetGroupBuilder { return &SvcExportTargetGroupBuilder{ - log: log, - client: client, - datastore: datastore, - cloud: cloud, + log: log, + client: client, } } @@ -55,99 +49,91 @@ type svcExportTargetGroupModelBuildTask struct { log gwlog.Logger client client.Client serviceExport *mcsv1alpha1.ServiceExport - targetGroup *model.TargetGroup - tgByResID map[string]*model.TargetGroup stack core.Stack - datastore *latticestore.LatticeDataStore - cloud pkg_aws.Cloud } func (b *SvcExportTargetGroupBuilder) Build( ctx context.Context, - srvExport *mcsv1alpha1.ServiceExport, -) (core.Stack, *model.TargetGroup, error) { - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(srvExport))) + svcExport *mcsv1alpha1.ServiceExport, +) (core.Stack, error) { + stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(svcExport))) task := &svcExportTargetGroupModelBuildTask{ log: b.log, - serviceExport: srvExport, + serviceExport: svcExport, stack: stack, - tgByResID: make(map[string]*model.TargetGroup), - datastore: b.datastore, - cloud: b.cloud, client: b.client, } if err := task.run(ctx); err != nil { - return task.stack, task.targetGroup, err + return nil, err } - return task.stack, task.targetGroup, nil + return task.stack, nil } -func (t *svcExportTargetGroupModelBuildTask) run(ctx context.Context) error { - err := t.BuildTargetGroupForServiceExport(ctx) - if err != nil { - return fmt.Errorf("failed to build target group for service export %s-%s due to %w", - t.serviceExport.Name, t.serviceExport.Namespace, err) - } +func (b *SvcExportTargetGroupBuilder) BuildTargetGroup(ctx context.Context, svcExport *mcsv1alpha1.ServiceExport) (*model.TargetGroup, error) { + stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(svcExport))) - err = t.BuildTargets(ctx) - if err != nil { - t.log.Debugf("Failed to build targets for service export %s-%s due to %s", - t.serviceExport.Name, t.serviceExport.Namespace, err) + task := &svcExportTargetGroupModelBuildTask{ + log: b.log, + serviceExport: svcExport, + stack: stack, + client: b.client, } - return nil + return task.buildTargetGroup(ctx) } -func (t *svcExportTargetGroupModelBuildTask) BuildTargets(ctx context.Context) error { - targetTask := &latticeTargetsModelBuildTask{ - log: t.log, - client: t.client, - tgName: t.serviceExport.Name, - tgNamespace: t.serviceExport.Namespace, - stack: t.stack, - datastore: t.datastore, +func (t *svcExportTargetGroupModelBuildTask) run(ctx context.Context) error { + tg, err := t.buildTargetGroup(ctx) + if err != nil { + return fmt.Errorf("failed to build target group for service export %s-%s due to %w", + t.serviceExport.Name, t.serviceExport.Namespace, err) } - err := targetTask.buildLatticeTargets(ctx) - if err != nil { - return err + if !tg.IsDeleted { + err = t.buildTargets(ctx, tg.ID()) + if err != nil { + t.log.Debugf("Failed to build targets for service export %s-%s due to %s", + t.serviceExport.Name, t.serviceExport.Namespace, err) + return err + } } return nil } -func (t *svcExportTargetGroupModelBuildTask) BuildTargetGroupForServiceExport(ctx context.Context) error { - tgName := latticestore.TargetGroupName(t.serviceExport.Name, t.serviceExport.Namespace) - var tg *model.TargetGroup - var err error - if t.serviceExport.DeletionTimestamp.IsZero() { - tg, err = t.buildTargetGroupForServiceExportCreation(ctx, tgName) - } else { - tg, err = t.buildTargetGroupForServiceExportDeletion(ctx, tgName) - } - +func (t *svcExportTargetGroupModelBuildTask) buildTargets(ctx context.Context, stackTgId string) error { + targetsBuilder := NewTargetsBuilder(t.log, t.client, t.stack) + _, err := targetsBuilder.BuildForServiceExport(ctx, t.serviceExport, stackTgId) if err != nil { return err } - - t.tgByResID[tgName] = tg - t.targetGroup = tg return nil } -func (t *svcExportTargetGroupModelBuildTask) buildTargetGroupForServiceExportCreation(ctx context.Context, targetGroupName string) (*model.TargetGroup, error) { +func (t *svcExportTargetGroupModelBuildTask) buildTargetGroup(ctx context.Context) (*model.TargetGroup, error) { svc := &corev1.Service{} + noSvcFoundAndDeleting := false if err := t.client.Get(ctx, k8s.NamespacedName(t.serviceExport), svc); err != nil { - t.datastore.SetTargetGroupByServiceExport(targetGroupName, false, false) - return nil, fmt.Errorf("Failed to find corresponding k8sService %s, error :%w ", k8s.NamespacedName(t.serviceExport), err) + if apierrors.IsNotFound(err) && !t.serviceExport.DeletionTimestamp.IsZero() { + // if we're deleting, it's OK if the service isn't there + noSvcFoundAndDeleting = true + } else { // either it's some other error or we aren't deleting + return nil, fmt.Errorf("Failed to find corresponding k8sService %s, error :%w ", k8s.NamespacedName(t.serviceExport), err) + } } - ipAddressType, err := buildTargetGroupIpAdressType(svc) - if err != nil { - return nil, err + var ipAddressType string + var err error + if noSvcFoundAndDeleting { + ipAddressType = "IPV4" // just pick a default + } else { + ipAddressType, err = buildTargetGroupIpAddressType(svc) + if err != nil { + return nil, err + } } tgp, err := GetAttachedPolicy(ctx, t.client, k8s.NamespacedName(t.serviceExport), &anv1alpha1.TargetGroupPolicy{}) @@ -168,233 +154,169 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargetGroupForServiceExportCre healthCheckConfig = parseHealthCheckConfig(tgp) } - stackTG := model.NewTargetGroup(t.stack, targetGroupName, model.TargetGroupSpec{ - Name: targetGroupName, - Type: model.TargetGroupTypeIP, - Config: model.TargetGroupConfig{ - VpcID: config.VpcID, - // Fill in default HTTP port as we are using target port anyway. - Port: 80, - IsServiceImport: false, - IsServiceExport: true, - K8SServiceName: t.serviceExport.Name, - K8SServiceNamespace: t.serviceExport.Namespace, - Protocol: protocol, - ProtocolVersion: protocolVersion, - HealthCheckConfig: healthCheckConfig, - IpAddressType: ipAddressType, - }, - }) - - t.log.Debugw("stackTG:", - "targetGroupName", stackTG.Spec.Name, - "K8SServiceName", stackTG.Spec.Config.K8SServiceName, - "K8SServiceNamespace", stackTG.Spec.Config.K8SServiceNamespace, - "Protocol", stackTG.Spec.Config.Protocol, - "ProtocolVersion", stackTG.Spec.Config.ProtocolVersion, - "IpAddressType", stackTG.Spec.Config.IpAddressType, - "HealthCheckConfig", stackTG.Spec.Config.HealthCheckConfig, - ) - - t.datastore.AddTargetGroup(targetGroupName, "", "", "", false, "") - t.datastore.SetTargetGroupByServiceExport(targetGroupName, false, true) - return stackTG, nil -} - -func (t *svcExportTargetGroupModelBuildTask) buildTargetGroupForServiceExportDeletion(ctx context.Context, targetGroupName string) (*model.TargetGroup, error) { - stackTG := model.NewTargetGroup(t.stack, targetGroupName, model.TargetGroupSpec{ - Name: targetGroupName, - LatticeID: "", - IsDeleted: true, - }) - t.datastore.SetTargetGroupByServiceExport(targetGroupName, false, false) - dsTG, err := t.datastore.GetTargetGroup(targetGroupName, "", false) - if err != nil { - return nil, fmt.Errorf("%w: targetGroupName: %s", err, targetGroupName) + spec := model.TargetGroupSpec{ + Type: model.TargetGroupTypeIP, + Port: 80, + Protocol: protocol, + ProtocolVersion: protocolVersion, + IpAddressType: ipAddressType, + HealthCheckConfig: healthCheckConfig, } + spec.VpcId = config.VpcID + spec.K8SParentRefType = model.ParentRefTypeSvcExport + spec.EKSClusterName = config.ClusterName + spec.K8SServiceName = t.serviceExport.Name + spec.K8SServiceNamespace = t.serviceExport.Namespace - if !dsTG.ByBackendRef { - // When handling the serviceExport deletion request while having dsTG.ByBackendRef==false, - // That means this target group is not in use anymore, i.e., it is not referenced by latticeService rules(aka http/grpc route rules), - // so, it can be deleted. Assign the stackTG.Spec.LatticeID to make target group manager can delete it - t.log.Debugf("Target group %s is not in use anymore and can be deleted", stackTG.Spec.Name) - stackTG.Spec.LatticeID = dsTG.ID + stackTG, err := model.NewTargetGroup(t.stack, spec) + if err != nil { + return nil, err } + stackTG.IsDeleted = !t.serviceExport.DeletionTimestamp.IsZero() return stackTG, nil } -// Build target group for backend service ref used in Route -func (t *latticeServiceModelBuildTask) buildTargetGroupsForRoute( - ctx context.Context, - client client.Client, -) error { - for _, rule := range t.route.Spec().Rules() { - for _, backendRef := range rule.BackendRefs() { - tgName := t.buildTargetGroupName(ctx, backendRef) - tgSpec, err := t.buildTargetGroupSpec(ctx, client, backendRef) - if err != nil { - return fmt.Errorf("buildTargetGroupSpec err %w", err) - } - - // add targetgroup to localcache for service reconcile to reference - if *backendRef.Kind() == "Service" { - t.datastore.AddTargetGroup(tgName, "", "", "", tgSpec.Config.IsServiceImport, t.route.Name()) - } else { - // for serviceimport, the httproutename is "" - t.datastore.AddTargetGroup(tgName, "", "", "", tgSpec.Config.IsServiceImport, "") - } +type BackendRefTargetGroupModelBuilder interface { + Build(ctx context.Context, route core.Route, backendRef core.BackendRef, stack core.Stack) (core.Stack, *model.TargetGroup, error) +} - if t.route.DeletionTimestamp().IsZero() { - t.datastore.SetTargetGroupByBackendRef(tgName, t.route.Name(), tgSpec.Config.IsServiceImport, true) - } else { - t.datastore.SetTargetGroupByBackendRef(tgName, t.route.Name(), tgSpec.Config.IsServiceImport, false) - dsTG, _ := t.datastore.GetTargetGroup(tgName, t.route.Name(), tgSpec.Config.IsServiceImport) - tgSpec.IsDeleted = true - tgSpec.LatticeID = dsTG.ID - } +type BackendRefTargetGroupBuilder struct { + log gwlog.Logger + client client.Client +} - tg := model.NewTargetGroup(t.stack, tgName, tgSpec) - t.tgByResID[tgName] = tg - } +func NewBackendRefTargetGroupBuilder(log gwlog.Logger, client client.Client) BackendRefTargetGroupModelBuilder { + return &BackendRefTargetGroupBuilder{ + log: log, + client: client, } - return nil } -// Triggered from route/service/targetgroup -func (t *latticeServiceModelBuildTask) buildTargetsForRoute(ctx context.Context) error { - for _, rule := range t.route.Spec().Rules() { - for _, backendRef := range rule.BackendRefs() { - if string(*backendRef.Kind()) == "ServiceImport" { - continue - } - - backendNamespace := t.route.Namespace() - if backendRef.Namespace() != nil { - backendNamespace = string(*backendRef.Namespace()) - } +type backendRefTargetGroupModelBuildTask struct { + log gwlog.Logger + client client.Client + stack core.Stack + route core.Route + backendRef core.BackendRef +} - var port int32 - if backendRef.Port() != nil { - port = int32(*backendRef.Port()) - } +func (b *BackendRefTargetGroupBuilder) Build( + ctx context.Context, + route core.Route, + backendRef core.BackendRef, + stack core.Stack, +) (core.Stack, *model.TargetGroup, error) { + if stack == nil { + stack = core.NewDefaultStack(core.StackID(k8s.NamespacedName(route.K8sObject()))) + b.log.Debugf("Creating new stack for build task") + } - targetTask := &latticeTargetsModelBuildTask{ - log: t.log, - client: t.client, - tgName: string(backendRef.Name()), - tgNamespace: backendNamespace, - routeName: t.route.Name(), - backendRefPort: port, - stack: t.stack, - datastore: t.datastore, - route: t.route, - } + task := backendRefTargetGroupModelBuildTask{ + log: b.log, + client: b.client, + stack: stack, + route: route, + backendRef: backendRef, + } - err := targetTask.buildLatticeTargets(ctx) - if err != nil { - return err - } - } + stackTg, err := task.buildTargetGroup(ctx) + if err != nil { + return nil, nil, err } - return nil + return task.stack, stackTg, nil } -// Now, Only k8sService and serviceImport creation deletion use this function to build TargetGroupSpec, serviceExport does not use this function to create TargetGroupSpec -func (t *latticeServiceModelBuildTask) buildTargetGroupSpec( - ctx context.Context, - client client.Client, - backendRef core.BackendRef, -) (model.TargetGroupSpec, error) { - var namespace string - - if backendRef.Namespace() != nil { - namespace = string(*backendRef.Namespace()) - } else { - namespace = t.route.Namespace() +func (t *backendRefTargetGroupModelBuildTask) buildTargetGroup(ctx context.Context) (*model.TargetGroup, error) { + if string(*t.backendRef.Kind()) == "ServiceImport" { + return nil, errors.New("not supported for ServiceImport BackendRef") } - backendKind := string(*backendRef.Kind()) - t.log.Debugf("buildTargetGroupSpec, kind %s", backendKind) + tgSpec, err := t.buildTargetGroupSpec(ctx) + if err != nil { + return nil, fmt.Errorf("buildTargetGroupSpec err %w", err) + } - var vpc = config.VpcID - var eksCluster = "" - var isServiceImport bool - var isDeleted bool + stackTG, err := model.NewTargetGroup(t.stack, tgSpec) + if err != nil { + return nil, err + } + t.log.Debugf("Added target group for backendRef %s to the stack %s", t.backendRef.Name(), stackTG.ID()) - if t.route.DeletionTimestamp().IsZero() { - isDeleted = false - } else { - isDeleted = true + stackTG.IsDeleted = !t.route.DeletionTimestamp().IsZero() + if !stackTG.IsDeleted { + t.buildTargets(ctx, stackTG.ID()) } - ipAddressType := vpclattice.IpAddressTypeIpv4 + return stackTG, nil +} - if backendKind == "ServiceImport" { - isServiceImport = true - namespaceName := types.NamespacedName{ - Namespace: namespace, - Name: string(backendRef.Name()), - } - serviceImport := &mcsv1alpha1.ServiceImport{} - - if err := client.Get(context.TODO(), namespaceName, serviceImport); err == nil { - t.log.Debugf("Building target group spec using service import %s", namespaceName) - vpc = serviceImport.Annotations["multicluster.x-k8s.io/aws-vpc"] - eksCluster = serviceImport.Annotations["multicluster.x-k8s.io/aws-eks-cluster-name"] - } else { - t.log.Errorf("Error building target group spec using service import %s due to %s", namespaceName, err) - if !isDeleted { - //Return error for creation request only. - //For ServiceImport deletion request, we should go ahead to build TargetGroupSpec model, - //although the targetGroupSynthesizer could skip TargetGroup deletion triggered by ServiceImport deletion - return model.TargetGroupSpec{}, err - } - } +func (t *backendRefTargetGroupModelBuildTask) buildTargets(ctx context.Context, stackTgId string) error { + if string(*t.backendRef.Kind()) == "ServiceImport" { + t.log.Debugf("Service import does not manage targets, returning") + return nil + } - } else { - var namespace = t.route.Namespace() - if backendRef.Namespace() != nil { - namespace = string(*backendRef.Namespace()) + backendRefNsName := getBackendRefNsName(t.route, t.backendRef) + svc := &corev1.Service{} + if err := t.client.Get(ctx, backendRefNsName, svc); err != nil { + t.log.Infof("Error finding backend service %s due to %s", backendRefNsName, err) + if t.route.DeletionTimestamp().IsZero() { + // Ignore error for deletion request + return err } + } - // find out service target port - serviceNamespaceName := types.NamespacedName{ - Namespace: namespace, - Name: string(backendRef.Name()), - } + targetsBuilder := NewTargetsBuilder(t.log, t.client, t.stack) + _, err := targetsBuilder.Build(ctx, svc, t.backendRef, stackTgId) + if err != nil { + return err + } - svc := &corev1.Service{} - if err := t.client.Get(ctx, serviceNamespaceName, svc); err != nil { - t.log.Infof("Error finding backend service %s due to %s", serviceNamespaceName, err) - if !isDeleted { - //Return error for creation request only, - //For k8sService deletion request, we should go ahead to build TargetGroupSpec model - return model.TargetGroupSpec{}, err - } - } + return nil +} + +// Now, Only k8sService and serviceImport creation deletion use this function to build TargetGroupSpec, serviceExport does not use this function to create TargetGroupSpec +func (t *backendRefTargetGroupModelBuildTask) buildTargetGroupSpec(ctx context.Context) (model.TargetGroupSpec, error) { + backendKind := string(*t.backendRef.Kind()) + t.log.Debugf("buildTargetGroupSpec, kind %s", backendKind) - var err error + vpc := config.VpcID + eksCluster := config.ClusterName + routeIsDeleted := !t.route.DeletionTimestamp().IsZero() - ipAddressType, err = buildTargetGroupIpAdressType(svc) + backendRefNsName := getBackendRefNsName(t.route, t.backendRef) - // Ignore error for creation request - if !isDeleted && err != nil { + svc := &corev1.Service{} + if err := t.client.Get(ctx, backendRefNsName, svc); err != nil { + t.log.Infof("Error finding backend service %s due to %s", backendRefNsName, err) + if !routeIsDeleted { + // Ignore error for deletion request return model.TargetGroupSpec{}, err } } - tgName := latticestore.TargetGroupName(string(backendRef.Name()), namespace) - - refObjNamespacedName := types.NamespacedName{ - Namespace: namespace, - Name: string(backendRef.Name()), + ipAddressType := vpclattice.IpAddressTypeIpv4 + var err error + if svc != nil { + ipAddressType, err = buildTargetGroupIpAddressType(svc) + if err != nil { + if routeIsDeleted { + // Ignore error for deletion request + t.log.Debugf("Unable to determine IP address type for deleted route, using default") + ipAddressType = vpclattice.IpAddressTypeIpv4 + } else { + // we care that there's an error if we are not deleting + return model.TargetGroupSpec{}, err + } + } } - tgp, err := GetAttachedPolicy(ctx, t.client, refObjNamespacedName, &anv1alpha1.TargetGroupPolicy{}) + tgp, err := GetAttachedPolicy(ctx, t.client, backendRefNsName, &anv1alpha1.TargetGroupPolicy{}) if err != nil { return model.TargetGroupSpec{}, err } + protocol := "HTTP" protocolVersion := vpclattice.TargetGroupProtocolVersionHttp1 var healthCheckConfig *vpclattice.HealthCheckConfig @@ -410,39 +332,42 @@ func (t *latticeServiceModelBuildTask) buildTargetGroupSpec( } // GRPC takes precedence over other protocolVersions. + parentRefType := model.ParentRefTypeHTTPRoute if _, ok := t.route.(*core.GRPCRoute); ok { protocolVersion = vpclattice.TargetGroupProtocolVersionGrpc - } - - return model.TargetGroupSpec{ - Name: tgName, - Type: model.TargetGroupTypeIP, - Config: model.TargetGroupConfig{ - VpcID: vpc, - EKSClusterName: eksCluster, - IsServiceImport: isServiceImport, - IsServiceExport: false, - K8SServiceName: string(backendRef.Name()), - K8SServiceNamespace: namespace, - K8SHTTPRouteName: t.route.Name(), - K8SHTTPRouteNamespace: t.route.Namespace(), - Protocol: protocol, - ProtocolVersion: protocolVersion, - HealthCheckConfig: healthCheckConfig, - // Fill in default HTTP port as we are using target port anyway. - Port: 80, - IpAddressType: ipAddressType, - }, - IsDeleted: isDeleted, - }, nil + parentRefType = model.ParentRefTypeGRPCRoute + } + + spec := model.TargetGroupSpec{ + Type: model.TargetGroupTypeIP, + Port: 80, + Protocol: protocol, + ProtocolVersion: protocolVersion, + IpAddressType: ipAddressType, + HealthCheckConfig: healthCheckConfig, + } + spec.VpcId = vpc + spec.K8SParentRefType = parentRefType + spec.EKSClusterName = eksCluster + spec.K8SServiceName = backendRefNsName.Name + spec.K8SServiceNamespace = backendRefNsName.Namespace + spec.K8SRouteName = t.route.Name() + spec.K8SRouteNamespace = t.route.Namespace() + + return spec, nil } -func (t *latticeServiceModelBuildTask) buildTargetGroupName(_ context.Context, backendRef core.BackendRef) string { +func getBackendRefNsName(route core.Route, backendRef core.BackendRef) types.NamespacedName { + var namespace = route.Namespace() if backendRef.Namespace() != nil { - return latticestore.TargetGroupName(string(backendRef.Name()), string(*backendRef.Namespace())) - } else { - return latticestore.TargetGroupName(string(backendRef.Name()), t.route.Namespace()) + namespace = string(*backendRef.Namespace()) + } + + backendRefNsName := types.NamespacedName{ + Namespace: namespace, + Name: string(backendRef.Name()), } + return backendRefNsName } func parseHealthCheckConfig(tgp *anv1alpha1.TargetGroupPolicy) *vpclattice.HealthCheckConfig { @@ -468,7 +393,7 @@ func parseHealthCheckConfig(tgp *anv1alpha1.TargetGroupPolicy) *vpclattice.Healt } } -func buildTargetGroupIpAdressType(svc *corev1.Service) (string, error) { +func buildTargetGroupIpAddressType(svc *corev1.Service) (string, error) { ipFamilies := svc.Spec.IPFamilies if len(ipFamilies) != 1 { diff --git a/pkg/gateway/model_build_targetgroup_mock.go b/pkg/gateway/model_build_targetgroup_mock.go new file mode 100644 index 00000000..6d77c90d --- /dev/null +++ b/pkg/gateway/model_build_targetgroup_mock.go @@ -0,0 +1,107 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./pkg/gateway/model_build_targetgroup.go + +// Package gateway is a generated GoMock package. +package gateway + +import ( + context "context" + reflect "reflect" + + core "github.com/aws/aws-application-networking-k8s/pkg/model/core" + lattice "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + gomock "github.com/golang/mock/gomock" + v1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" +) + +// MockSvcExportTargetGroupModelBuilder is a mock of SvcExportTargetGroupModelBuilder interface. +type MockSvcExportTargetGroupModelBuilder struct { + ctrl *gomock.Controller + recorder *MockSvcExportTargetGroupModelBuilderMockRecorder +} + +// MockSvcExportTargetGroupModelBuilderMockRecorder is the mock recorder for MockSvcExportTargetGroupModelBuilder. +type MockSvcExportTargetGroupModelBuilderMockRecorder struct { + mock *MockSvcExportTargetGroupModelBuilder +} + +// NewMockSvcExportTargetGroupModelBuilder creates a new mock instance. +func NewMockSvcExportTargetGroupModelBuilder(ctrl *gomock.Controller) *MockSvcExportTargetGroupModelBuilder { + mock := &MockSvcExportTargetGroupModelBuilder{ctrl: ctrl} + mock.recorder = &MockSvcExportTargetGroupModelBuilderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockSvcExportTargetGroupModelBuilder) EXPECT() *MockSvcExportTargetGroupModelBuilderMockRecorder { + return m.recorder +} + +// Build mocks base method. +func (m *MockSvcExportTargetGroupModelBuilder) Build(ctx context.Context, svcExport *v1alpha1.ServiceExport) (core.Stack, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Build", ctx, svcExport) + ret0, _ := ret[0].(core.Stack) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Build indicates an expected call of Build. +func (mr *MockSvcExportTargetGroupModelBuilderMockRecorder) Build(ctx, svcExport interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*MockSvcExportTargetGroupModelBuilder)(nil).Build), ctx, svcExport) +} + +// BuildTargetGroup mocks base method. +func (m *MockSvcExportTargetGroupModelBuilder) BuildTargetGroup(ctx context.Context, svcExport *v1alpha1.ServiceExport) (*lattice.TargetGroup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "BuildTargetGroup", ctx, svcExport) + ret0, _ := ret[0].(*lattice.TargetGroup) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// BuildTargetGroup indicates an expected call of BuildTargetGroup. +func (mr *MockSvcExportTargetGroupModelBuilderMockRecorder) BuildTargetGroup(ctx, svcExport interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "BuildTargetGroup", reflect.TypeOf((*MockSvcExportTargetGroupModelBuilder)(nil).BuildTargetGroup), ctx, svcExport) +} + +// MockBackendRefTargetGroupModelBuilder is a mock of BackendRefTargetGroupModelBuilder interface. +type MockBackendRefTargetGroupModelBuilder struct { + ctrl *gomock.Controller + recorder *MockBackendRefTargetGroupModelBuilderMockRecorder +} + +// MockBackendRefTargetGroupModelBuilderMockRecorder is the mock recorder for MockBackendRefTargetGroupModelBuilder. +type MockBackendRefTargetGroupModelBuilderMockRecorder struct { + mock *MockBackendRefTargetGroupModelBuilder +} + +// NewMockBackendRefTargetGroupModelBuilder creates a new mock instance. +func NewMockBackendRefTargetGroupModelBuilder(ctrl *gomock.Controller) *MockBackendRefTargetGroupModelBuilder { + mock := &MockBackendRefTargetGroupModelBuilder{ctrl: ctrl} + mock.recorder = &MockBackendRefTargetGroupModelBuilderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBackendRefTargetGroupModelBuilder) EXPECT() *MockBackendRefTargetGroupModelBuilderMockRecorder { + return m.recorder +} + +// Build mocks base method. +func (m *MockBackendRefTargetGroupModelBuilder) Build(ctx context.Context, route core.Route, backendRef core.BackendRef, stack core.Stack) (core.Stack, *lattice.TargetGroup, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Build", ctx, route, backendRef, stack) + ret0, _ := ret[0].(core.Stack) + ret1, _ := ret[1].(*lattice.TargetGroup) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// Build indicates an expected call of Build. +func (mr *MockBackendRefTargetGroupModelBuilderMockRecorder) Build(ctx, route, backendRef, stack interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Build", reflect.TypeOf((*MockBackendRefTargetGroupModelBuilder)(nil).Build), ctx, route, backendRef, stack) +} diff --git a/pkg/gateway/model_build_targetgroup_test.go b/pkg/gateway/model_build_targetgroup_test.go index 6914af47..fd489ddb 100644 --- a/pkg/gateway/model_build_targetgroup_test.go +++ b/pkg/gateway/model_build_targetgroup_test.go @@ -2,8 +2,13 @@ package gateway import ( "context" - "errors" "fmt" + mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" + "github.com/aws/aws-application-networking-k8s/pkg/config" + "github.com/aws/aws-application-networking-k8s/pkg/k8s" + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "strings" "testing" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -14,21 +19,18 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" clientgoscheme "k8s.io/client-go/kubernetes/scheme" testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" - mock_client "github.com/aws/aws-application-networking-k8s/mocks/controller-runtime/client" anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" - "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) -func Test_TGModelByServicexportBuild(t *testing.T) { +func Test_TGModelByServicExportBuild(t *testing.T) { + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" + now := metav1.Now() tests := []struct { name string @@ -40,7 +42,7 @@ func Test_TGModelByServicexportBuild(t *testing.T) { wantIPv6TargetGroup bool }{ { - name: "Adding ServieExport where service object exist", + name: "Adding ServiceExport where service object exist", svcExport: &mcsv1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ Name: "export1", @@ -74,7 +76,7 @@ func Test_TGModelByServicexportBuild(t *testing.T) { wantIPv6TargetGroup: false, }, { - name: "Adding ServieExport where service object does NOT exist", + name: "Adding ServiceExport where service object does NOT exist", svcExport: &mcsv1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ Name: "export2", @@ -96,13 +98,11 @@ func Test_TGModelByServicexportBuild(t *testing.T) { DeletionTimestamp: &now, }, }, - - wantErrIsNil: true, - wantIsDeleted: true, - wantIPv6TargetGroup: false, + wantIsDeleted: true, + wantErrIsNil: true, }, { - name: "Deleting ServieExport where service object exist", + name: "Deleting ServiceExport where service object exist", svcExport: &mcsv1alpha1.ServiceExport{ ObjectMeta: metav1.ObjectMeta{ Name: "export4", @@ -217,57 +217,61 @@ func Test_TGModelByServicexportBuild(t *testing.T) { if tt.svc != nil { assert.NoError(t, k8sClient.Create(ctx, tt.svc.DeepCopy())) - } if tt.endPoints != nil { assert.NoError(t, k8sClient.Create(ctx, tt.endPoints[0].DeepCopy())) } - ds := latticestore.NewLatticeDataStore() - if !tt.svcExport.DeletionTimestamp.IsZero() { - // When test serviceExport deletion, we expect latticeDataStore already had this tg entry - tgName := latticestore.TargetGroupName(tt.svcExport.Name, tt.svcExport.Namespace) - ds.AddTargetGroup(tgName, "vpc-123456789", "123456789", "tg-123", false, "") - } - builder := NewSvcExportTargetGroupBuilder(gwlog.FallbackLogger, k8sClient, ds, nil) - - stack, tg, err := builder.Build(ctx, tt.svcExport) - - fmt.Printf("stack %v tg %v err %v\n", stack, tg, err) + builder := NewSvcExportTargetGroupBuilder(gwlog.FallbackLogger, k8sClient) + stack, err := builder.Build(ctx, tt.svcExport) + fmt.Printf("stack %v err %v\n", stack, err) if !tt.wantErrIsNil { assert.NotNil(t, err) return } + assert.Nil(t, err) + var resTargetGroups []*model.TargetGroup + err = stack.ListResources(&resTargetGroups) assert.Nil(t, err) - tgName := latticestore.TargetGroupName(tt.svcExport.Name, tt.svcExport.Namespace) - assert.Equal(t, tgName, tg.Spec.Name) + assert.Equal(t, 1, len(resTargetGroups)) - // for serviceexport, the routeName is "" - dsTG, err := ds.GetTargetGroup(tgName, "", false) + stackTg := resTargetGroups[0] + spec := model.TargetGroupSpec{} + spec.K8SServiceName = tt.svcExport.Name + spec.K8SServiceNamespace = tt.svcExport.Namespace + tgNamePrefix := model.TgNamePrefix(spec) + generatedName := model.GenerateTgName(stackTg.Spec) + assert.True(t, strings.HasPrefix(generatedName, tgNamePrefix)) - assert.Nil(t, err) if tt.wantIsDeleted { - assert.Equal(t, false, dsTG.ByServiceExport) - assert.Equal(t, true, tg.Spec.IsDeleted) - assert.Equal(t, "tg-123", tg.Spec.LatticeID) + assert.True(t, stackTg.IsDeleted) + return + } + assert.False(t, stackTg.IsDeleted) + + if tt.wantIPv6TargetGroup { + assert.Equal(t, vpclattice.IpAddressTypeIpv6, stackTg.Spec.IpAddressType) } else { - assert.Equal(t, true, dsTG.ByServiceExport) - assert.Equal(t, "", tg.Spec.LatticeID) - if tt.wantIPv6TargetGroup { - assert.Equal(t, vpclattice.IpAddressTypeIpv6, tg.Spec.Config.IpAddressType) - } else { - assert.Equal(t, vpclattice.IpAddressTypeIpv4, tg.Spec.Config.IpAddressType) - } + assert.Equal(t, vpclattice.IpAddressTypeIpv4, stackTg.Spec.IpAddressType) } + assert.Equal(t, config.ClusterName, stackTg.Spec.EKSClusterName) + assert.Equal(t, config.VpcID, stackTg.Spec.VpcId) + assert.Equal(t, model.ParentRefTypeSvcExport, stackTg.Spec.K8SParentRefType) + assert.Equal(t, tt.svc.Name, stackTg.Spec.K8SServiceName) + assert.Equal(t, tt.svc.Namespace, stackTg.Spec.K8SServiceNamespace) + assert.Equal(t, "", stackTg.Spec.K8SRouteName) + assert.Equal(t, "", stackTg.Spec.K8SRouteNamespace) }) } } func Test_TGModelByHTTPRouteBuild(t *testing.T) { + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" now := metav1.Now() namespacePtr := func(ns string) *gwv1beta1.Namespace { @@ -294,13 +298,15 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { name: "Add LatticeService", route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ - Name: "service1", + Name: "service1", + Namespace: "ns1", }, Spec: gwv1beta1.HTTPRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ ParentRefs: []gwv1beta1.ParentReference{ { - Name: "gateway1", + Name: "gateway1", + Namespace: namespacePtr("ns1"), }, }, }, @@ -333,6 +339,7 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service2", + Namespace: "ns1", Finalizers: []string{"gateway.k8s.aws/resources"}, DeletionTimestamp: &now, }, @@ -340,7 +347,8 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { CommonRouteSpec: gwv1beta1.CommonRouteSpec{ ParentRefs: []gwv1beta1.ParentReference{ { - Name: "gateway1", + Name: "gateway1", + Namespace: namespacePtr("ns1"), }, }, }, @@ -373,13 +381,15 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service3", + Namespace: "ns1", Finalizers: []string{"gateway.k8s.aws/resources"}, }, Spec: gwv1beta1.HTTPRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ ParentRefs: []gwv1beta1.ParentReference{ { - Name: "gateway1", + Name: "gateway1", + Namespace: namespacePtr("ns1"), }, }, }, @@ -412,13 +422,15 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service4", + Namespace: "ns1", Finalizers: []string{"gateway.k8s.aws/resources"}, }, Spec: gwv1beta1.HTTPRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ ParentRefs: []gwv1beta1.ParentReference{ { - Name: "gateway1", + Name: "gateway1", + Namespace: namespacePtr("ns1"), }, }, }, @@ -451,13 +463,15 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "service5", + Namespace: "ns1", Finalizers: []string{"gateway.k8s.aws/resources"}, }, Spec: gwv1beta1.HTTPRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ ParentRefs: []gwv1beta1.ParentReference{ { - Name: "gateway1", + Name: "gateway1", + Namespace: namespacePtr("ns1"), }, }, }, @@ -495,103 +509,90 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { k8sSchema := runtime.NewScheme() clientgoscheme.AddToScheme(k8sSchema) anv1alpha1.AddToScheme(k8sSchema) + gwv1beta1.AddToScheme(k8sSchema) k8sClient := testclient.NewFakeClientWithScheme(k8sSchema) - ds := latticestore.NewLatticeDataStore() - - //builder := NewLatticeServiceBuilder(k8sClient, ds, nil) stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.route.K8sObject()))) - task := &latticeServiceModelBuildTask{ - log: gwlog.FallbackLogger, - route: tt.route, - stack: stack, - client: k8sClient, - tgByResID: make(map[string]*model.TargetGroup), - datastore: ds, - } - if tt.svcExist { // populate K8S service for _, httpRules := range tt.route.Spec().Rules() { for _, httpBackendRef := range httpRules.BackendRefs() { - if tt.svcExist { - svc := corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: string(httpBackendRef.Name()), - Namespace: string(*httpBackendRef.Namespace()), - }, - } - - if tt.wantIPv6TargetGroup { - svc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv6Protocol} - } else { - svc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol} - } - - fmt.Printf("create K8S service %v\n", svc) + svc := corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: string(httpBackendRef.Name()), + Namespace: string(*httpBackendRef.Namespace()), + }, + } - assert.NoError(t, k8sClient.Create(ctx, svc.DeepCopy())) + if tt.wantIPv6TargetGroup { + svc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv6Protocol} + } else { + svc.Spec.IPFamilies = []corev1.IPFamily{corev1.IPv4Protocol} } + assert.NoError(t, k8sClient.Create(ctx, svc.DeepCopy())) } } } - err := task.buildTargetGroupsForRoute(ctx, k8sClient) + // this test assumes a single rule and single backendRef, which means we + // should always just get one target group + assert.Equal(t, 1, len(tt.route.Spec().Rules())) + rule := tt.route.Spec().Rules()[0] + assert.Equal(t, 1, len(rule.BackendRefs())) + httpBackendRef := rule.BackendRefs()[0] + + // we just want to test the target group logic, not service, listener, etc + // this is done on a per backend-ref basis + builder := NewBackendRefTargetGroupBuilder(gwlog.FallbackLogger, k8sClient) + + _, stackTg, err := builder.Build(ctx, tt.route, httpBackendRef, stack) if !tt.wantErrIsNil { assert.NotNil(t, err) - } else { - assert.Nil(t, err) + return } + assert.Nil(t, err) - if tt.wantErrIsNil { - // verify data store - for _, httpRules := range tt.route.Spec().Rules() { - for _, httpBackendRef := range httpRules.BackendRefs() { - tgName := latticestore.TargetGroupName(string(httpBackendRef.Name()), string(*httpBackendRef.Namespace())) - - fmt.Printf("httpBackendRef %s\n", *httpBackendRef.Kind()) - if "Service" == *httpBackendRef.Kind() { - if tt.wantIsDeleted { - tg := task.tgByResID[tgName] - fmt.Printf("--task.tgByResID[tgName] %v \n", tg) - assert.Equal(t, true, tg.Spec.IsDeleted) - } else { - dsTG, err := ds.GetTargetGroup(tgName, tt.route.Name(), false) - assert.Equal(t, true, dsTG.ByBackendRef) - fmt.Printf("--dsTG %v\n", dsTG) - assert.Nil(t, err) - } - - // Verify IpAddressType of Target Group - tg := task.tgByResID[tgName] - - ipAddressType := tg.Spec.Config.IpAddressType - - if tt.wantIPv6TargetGroup { - assert.Equal(t, vpclattice.IpAddressTypeIpv6, ipAddressType) - } else { - assert.Equal(t, vpclattice.IpAddressTypeIpv4, ipAddressType) - } - } else { - // the routeName for serviceimport is "" - dsTG, err := ds.GetTargetGroup(tgName, "", true) - fmt.Printf("dsTG %v\n", dsTG) - assert.Nil(t, err) - } - assert.Nil(t, err) + var stackTgs []*model.TargetGroup + _ = stack.ListResources(&stackTgs) + assert.Equal(t, 1, len(stackTgs)) - } - } + // make sure the name is in the format we expect - based off the backend ref/svc name/ns + spec := model.TargetGroupSpec{} + spec.K8SServiceName = string(httpBackendRef.Name()) + spec.K8SServiceNamespace = string(*httpBackendRef.Namespace()) + tgNamePrefix := model.TgNamePrefix(spec) + generatedName := model.GenerateTgName(stackTg.Spec) + assert.True(t, strings.HasPrefix(generatedName, tgNamePrefix)) + + assert.Equal(t, tt.wantIsDeleted, stackTg.IsDeleted) + if tt.wantIsDeleted { + return } + + if tt.wantIPv6TargetGroup { + assert.Equal(t, vpclattice.IpAddressTypeIpv6, stackTg.Spec.IpAddressType) + } else { + assert.Equal(t, vpclattice.IpAddressTypeIpv4, stackTg.Spec.IpAddressType) + } + + assert.Equal(t, config.ClusterName, stackTg.Spec.EKSClusterName) + assert.Equal(t, config.VpcID, stackTg.Spec.VpcId) + assert.Equal(t, model.ParentRefTypeHTTPRoute, stackTg.Spec.K8SParentRefType) + assert.Equal(t, spec.K8SServiceName, stackTg.Spec.K8SServiceName) + assert.Equal(t, spec.K8SServiceNamespace, stackTg.Spec.K8SServiceNamespace) + assert.Equal(t, tt.route.Name(), stackTg.Spec.K8SRouteName) + assert.Equal(t, tt.route.Namespace(), stackTg.Spec.K8SRouteNamespace) }) } } -func Test_TGModelByHTTPRouteImportBuild(t *testing.T) { - now := metav1.Now() +func Test_ServiceImportToTGBuildReturnsError(t *testing.T) { + config.VpcID = "vpc-id" + config.ClusterName = "cluster-name" + namespacePtr := func(ns string) *gwv1beta1.Namespace { p := gwv1beta1.Namespace(ns) return &p @@ -603,16 +604,11 @@ func Test_TGModelByHTTPRouteImportBuild(t *testing.T) { } tests := []struct { - name string - route core.Route - svcImportExist bool - wantError error - wantErrIsNil bool - wantName string - wantIsDeleted bool + name string + route core.Route }{ { - name: "Add LatticeService", + name: "Service import does not create target group - returns error", route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ ObjectMeta: metav1.ObjectMeta{ Name: "serviceimport1", @@ -642,88 +638,6 @@ func Test_TGModelByHTTPRouteImportBuild(t *testing.T) { }, }, }), - svcImportExist: true, - wantError: nil, - wantName: "service1", - wantIsDeleted: false, - wantErrIsNil: true, - }, - { - name: "Add LatticeService, implicit namespace", - route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "serviceimport1", - Namespace: "tg1-ns2", - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gateway1", - }, - }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ - { - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: gwv1beta1.BackendRef{ - BackendObjectReference: gwv1beta1.BackendObjectReference{ - Name: "service1-tg2", - Kind: kindPtr("ServiceImport"), - }, - }, - }, - }, - }, - }, - }, - }), - svcImportExist: true, - wantError: nil, - wantName: "service1", - wantIsDeleted: false, - wantErrIsNil: true, - }, - { - name: "Delete LatticeService", - route: core.NewHTTPRoute(gwv1beta1.HTTPRoute{ - ObjectMeta: metav1.ObjectMeta{ - Name: "serviceimport2", - Finalizers: []string{"gateway.k8s.aws/resources"}, - DeletionTimestamp: &now, - }, - Spec: gwv1beta1.HTTPRouteSpec{ - CommonRouteSpec: gwv1beta1.CommonRouteSpec{ - ParentRefs: []gwv1beta1.ParentReference{ - { - Name: "gateway1", - }, - }, - }, - Rules: []gwv1beta1.HTTPRouteRule{ - - { - BackendRefs: []gwv1beta1.HTTPBackendRef{ - { - BackendRef: gwv1beta1.BackendRef{ - BackendObjectReference: gwv1beta1.BackendObjectReference{ - Name: "service1-tg2", - Namespace: namespacePtr("tg1-ns1"), - Kind: kindPtr("ServiceImport"), - }, - }, - }, - }, - }, - }, - }, - }), - svcImportExist: true, - wantError: nil, - wantName: "service1", - wantIsDeleted: true, - wantErrIsNil: true, }, } @@ -735,91 +649,20 @@ func Test_TGModelByHTTPRouteImportBuild(t *testing.T) { mockK8sClient := mock_client.NewMockClient(c) - ds := latticestore.NewLatticeDataStore() - - //builder := NewLatticeServiceBuilder(mockK8sClient, ds, nil) - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(tt.route.K8sObject()))) - task := &latticeServiceModelBuildTask{ - log: gwlog.FallbackLogger, - route: tt.route, - stack: stack, - client: mockK8sClient, - tgByResID: make(map[string]*model.TargetGroup), - datastore: ds, - } - - for _, httpRules := range tt.route.Spec().Rules() { - for _, httpBackendRef := range httpRules.BackendRefs() { - if tt.svcImportExist { - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any(), gomock.Any()).DoAndReturn( - func(ctx context.Context, name types.NamespacedName, svcImport *mcsv1alpha1.ServiceImport, arg3 ...interface{}) error { - //TODO add more - svcImport.ObjectMeta.Name = string(httpBackendRef.Name()) - return nil - }, - ) - } else { - mockK8sClient.EXPECT().Get(ctx, gomock.Any(), gomock.Any()).Return(errors.New("serviceimport not exist")) - } - } - } - mockK8sClient.EXPECT().List(ctx, gomock.Any(), gomock.Any()).Return(nil) - - err := task.buildTargetGroupsForRoute(ctx, mockK8sClient) - - fmt.Printf("err %v\n", err) - - if !tt.wantErrIsNil { - assert.NotNil(t, err) - } else { - assert.Nil(t, err) - } + // these tests only support a single rule and backend ref + rule := tt.route.Spec().Rules()[0] + httpBackendRef := rule.BackendRefs()[0] - if tt.wantErrIsNil { - // verify data store - for _, httpRules := range tt.route.Spec().Rules() { - for _, httpBackendRef := range httpRules.BackendRefs() { - ns := tt.route.Namespace() - if httpBackendRef.Namespace() != nil { - ns = string(*httpBackendRef.Namespace()) - } - tgName := latticestore.TargetGroupName(string(httpBackendRef.Name()), ns) - - fmt.Printf("httpBackendRef %s\n", *httpBackendRef.Kind()) - if "Service" == *httpBackendRef.Kind() { - if tt.wantIsDeleted { - tg := task.tgByResID[tgName] - fmt.Printf("--task.tgByResID[tgName] %v \n", tg) - assert.Equal(t, true, tg.Spec.IsDeleted) - } else { - dsTG, err := ds.GetTargetGroup(tgName, tt.route.Name(), false) - assert.Equal(t, true, dsTG.ByBackendRef) - fmt.Printf("--dsTG %v\n", dsTG) - assert.Nil(t, err) - } - } else { - dsTG, err := ds.GetTargetGroup(tgName, "", true) - fmt.Printf("dsTG %v\n", dsTG) - if tt.wantIsDeleted { - tg := task.tgByResID[tgName] - assert.Equal(t, true, tg.Spec.IsDeleted) - } else { - assert.Equal(t, false, dsTG.ByBackendRef) - assert.Equal(t, false, dsTG.ByServiceExport) - assert.Nil(t, err) - } - } - assert.Nil(t, err) - } - } - } + builder := NewBackendRefTargetGroupBuilder(gwlog.FallbackLogger, mockK8sClient) + _, _, err := builder.Build(ctx, tt.route, httpBackendRef, stack) + assert.NotNil(t, err) }) } } -func Test_buildTargetGroupIpAdressType(t *testing.T) { +func Test_buildTargetGroupIpAddressType(t *testing.T) { type args struct { svc *corev1.Service } @@ -880,13 +723,13 @@ func Test_buildTargetGroupIpAdressType(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := buildTargetGroupIpAdressType(tt.args.svc) + got, err := buildTargetGroupIpAddressType(tt.args.svc) if (err != nil) != tt.wantErr { - t.Errorf("buildTargetGroupIpAdressType() error = %v, wantErr %v", err, tt.wantErr) + t.Errorf("buildTargetGroupIpAddressType() error = %v, wantErr %v", err, tt.wantErr) return } if got != tt.want { - t.Errorf("buildTargetGroupIpAdressType() = %v, want %v", got, tt.want) + t.Errorf("buildTargetGroupIpAddressType() = %v, want %v", got, tt.want) } }) } diff --git a/pkg/gateway/model_build_targets.go b/pkg/gateway/model_build_targets.go index 4cd5cd3e..0c831eb3 100644 --- a/pkg/gateway/model_build_targets.go +++ b/pkg/gateway/model_build_targets.go @@ -3,7 +3,6 @@ package gateway import ( "context" "errors" - "fmt" "strconv" "strings" @@ -12,9 +11,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" - pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-application-networking-k8s/pkg/k8s" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/pkg/model/core" model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -26,103 +23,102 @@ const ( ) type LatticeTargetsBuilder interface { - Build(ctx context.Context, service *corev1.Service, routeName string) (core.Stack, *model.Targets, error) + Build(ctx context.Context, service *corev1.Service, backendRef core.BackendRef, stackTgId string) (core.Stack, error) + BuildForServiceExport(ctx context.Context, serviceExport *mcsv1alpha1.ServiceExport, stackTgId string) (core.Stack, error) } type LatticeTargetsModelBuilder struct { - log gwlog.Logger - client client.Client - defaultTags map[string]string - datastore *latticestore.LatticeDataStore - cloud pkg_aws.Cloud + log gwlog.Logger + client client.Client + stack core.Stack } func NewTargetsBuilder( log gwlog.Logger, client client.Client, - cloud pkg_aws.Cloud, - datastore *latticestore.LatticeDataStore, + stack core.Stack, ) *LatticeTargetsModelBuilder { return &LatticeTargetsModelBuilder{ - log: log, - client: client, - cloud: cloud, - datastore: datastore, + log: log, + client: client, + stack: stack, } } -func (b *LatticeTargetsModelBuilder) Build(ctx context.Context, service *corev1.Service, routeName string) (core.Stack, *model.Targets, error) { - stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(service))) +func (b *LatticeTargetsModelBuilder) Build(ctx context.Context, service *corev1.Service, + backendRef core.BackendRef, stackTgId string) (core.Stack, error) { + return b.build(ctx, nil, service, backendRef, b.stack, stackTgId) +} - task := &latticeTargetsModelBuildTask{ - log: b.log, - client: b.client, - tgName: service.Name, - tgNamespace: service.Namespace, - routeName: routeName, - stack: stack, - datastore: b.datastore, - } +func (b *LatticeTargetsModelBuilder) BuildForServiceExport(ctx context.Context, + serviceExport *mcsv1alpha1.ServiceExport, stackTgId string) (core.Stack, error) { - if err := task.run(ctx); err != nil { - return nil, nil, corev1.ErrIntOverflowGenerated + return b.build(ctx, serviceExport, nil, nil, b.stack, stackTgId) +} + +func (b *LatticeTargetsModelBuilder) build(ctx context.Context, + serviceExport *mcsv1alpha1.ServiceExport, + service *corev1.Service, backendRef core.BackendRef, + stack core.Stack, stackTgId string, +) (core.Stack, error) { + isServiceExport := serviceExport != nil + isBackendRef := service != nil && backendRef != nil + if !(isServiceExport || isBackendRef) { + return nil, errors.New("either service export or route/service/backendRef must be specified") + } + if isServiceExport && isBackendRef { + return nil, errors.New("either service export or route/service/backendRef must be specified, but not both") } - return task.stack, task.latticeTargets, nil -} + if isServiceExport { + b.log.Debugf("Processing targets for service export %s-%s", serviceExport.Name, serviceExport.Namespace) -func (t *latticeTargetsModelBuildTask) run(ctx context.Context) error { - return t.buildLatticeTargets(ctx) -} + serviceName := types.NamespacedName{ + Namespace: serviceExport.Namespace, + Name: serviceExport.Name, + } -func (t *latticeTargetsModelBuildTask) buildLatticeTargets(ctx context.Context) error { - ds := t.datastore - tgName := latticestore.TargetGroupName(t.tgName, t.tgNamespace) - tg, err := ds.GetTargetGroup(tgName, t.routeName, false) // isServiceImport= false + tmpSvc := &corev1.Service{} + if err := b.client.Get(ctx, serviceName, tmpSvc); err != nil { + return nil, err + } + service = tmpSvc + } else { + b.log.Debugf("Processing targets for service %s-%s", service.Name, service.Namespace) + } - if err != nil { - errmsg := fmt.Sprintf("Build Targets failed because target group (name=%s, namespace=%s found not in datastore)", t.tgName, t.tgNamespace) - return errors.New(errmsg) + if stack == nil { + stack = core.NewDefaultStack(core.StackID(k8s.NamespacedName(service))) } - if !tg.ByBackendRef && !tg.ByServiceExport { - errmsg := fmt.Sprintf("Build Targets failed because its target Group name=%s, namespace=%s is no longer referenced", t.tgName, t.tgNamespace) - return errors.New(errmsg) + if !service.DeletionTimestamp.IsZero() { + b.log.Debugf("service %s/%s is deleted, skipping target build", service.Name, service.Namespace) + return stack, nil } - svc := &corev1.Service{} - namespacedName := types.NamespacedName{ - Namespace: t.tgNamespace, - Name: t.tgName, + task := &latticeTargetsModelBuildTask{ + log: b.log, + client: b.client, + serviceExport: serviceExport, + service: service, + backendRef: backendRef, + stack: stack, + stackTgId: stackTgId, } - if err := t.client.Get(ctx, namespacedName, svc); err != nil { - return fmt.Errorf("Build Targets failed because K8S service %s does not exist", namespacedName) + if err := task.run(ctx); err != nil { + return nil, err } - definedPorts := make(map[int32]struct{}) + return task.stack, nil +} - if tg.ByServiceExport { - serviceExport := &mcsv1alpha1.ServiceExport{} - err = t.client.Get(ctx, namespacedName, serviceExport) - if err != nil { - t.log.Errorf("Failed to find service export %s-%s in datastore due to %s", t.tgName, t.tgNamespace, err) - } else { - portsAnnotations := strings.Split(serviceExport.ObjectMeta.Annotations[portAnnotationsKey], ",") +func (t *latticeTargetsModelBuildTask) run(ctx context.Context) error { + return t.buildLatticeTargets(ctx) +} - for _, portAnnotation := range portsAnnotations { - definedPort, err := strconv.ParseInt(portAnnotation, 10, 32) - if err != nil { - t.log.Errorf("Failed to read Annotations/Port: %s due to %s", - serviceExport.ObjectMeta.Annotations[portAnnotationsKey], err) - } else { - definedPorts[int32(definedPort)] = struct{}{} - } - } - } - } else if tg.ByBackendRef && t.backendRefPort != undefinedPort { - definedPorts[t.backendRefPort] = struct{}{} - } +func (t *latticeTargetsModelBuildTask) buildLatticeTargets(ctx context.Context) error { + definedPorts := t.getDefinedPorts() // A service port MUST have a name if there are multiple ports exposed from a service. // Therefore, if a port is named, endpoint port is only relevant if it has the same name. @@ -134,7 +130,7 @@ func (t *latticeTargetsModelBuildTask) buildLatticeTargets(ctx context.Context) servicePortNames := make(map[string]struct{}) skipMatch := false - for _, port := range svc.Spec.Ports { + for _, port := range t.service.Spec.Ports { if _, ok := definedPorts[port.Port]; ok { if port.Name != "" { servicePortNames[port.Name] = struct{}{} @@ -151,51 +147,90 @@ func (t *latticeTargetsModelBuildTask) buildLatticeTargets(ctx context.Context) } var targetList []model.Target - endpoints := &corev1.Endpoints{} - - if svc.DeletionTimestamp.IsZero() { - if err := t.client.Get(ctx, namespacedName, endpoints); err != nil { - return fmt.Errorf("build targets failed because K8S service %s does not exist", namespacedName) + if t.service.DeletionTimestamp.IsZero() { + var err error + targetList, err = t.getTargetListFromEndpoints(ctx, servicePortNames, skipMatch) + if err != nil { + return err } + } + + spec := model.TargetsSpec{ + StackTargetGroupId: t.stackTgId, + TargetList: targetList, + } + + _, err := model.NewTargets(t.stack, spec) + if err != nil { + return err + } + + return nil +} + +func (t *latticeTargetsModelBuildTask) getTargetListFromEndpoints(ctx context.Context, servicePortNames map[string]struct{}, skipMatch bool) ([]model.Target, error) { + nsName := types.NamespacedName{ + Namespace: t.service.Namespace, + Name: t.service.Name, + } + + endpoints := &corev1.Endpoints{} + if err := t.client.Get(ctx, nsName, endpoints); err != nil { + return nil, err + } - for _, endPoint := range endpoints.Subsets { - for _, address := range endPoint.Addresses { - for _, port := range endPoint.Ports { + var targetList []model.Target + for _, subset := range endpoints.Subsets { + for _, address := range subset.Addresses { + for _, port := range subset.Ports { + // Note that the Endpoint's port name is from ServicePort, but the actual registered port + // is from Pods(targets). + if _, ok := servicePortNames[port.Name]; ok || skipMatch { target := model.Target{ TargetIP: address.IP, Port: int64(port.Port), } - // Note that the Endpoint's port name is from ServicePort, but the actual registered port - // is from Pods(targets). - if _, ok := servicePortNames[port.Name]; ok || skipMatch { - targetList = append(targetList, target) - } + targetList = append(targetList, target) } } } } + return targetList, nil +} - spec := model.TargetsSpec{ - Name: t.tgName, - Namespace: t.tgNamespace, - RouteName: t.routeName, - TargetIPList: targetList, - } +func (t *latticeTargetsModelBuildTask) getDefinedPorts() map[int32]struct{} { + definedPorts := make(map[int32]struct{}) - t.latticeTargets = model.NewTargets(t.stack, tgName, spec) + isServiceExport := t.serviceExport != nil + if isServiceExport { + portsAnnotations := strings.Split(t.serviceExport.ObjectMeta.Annotations[portAnnotationsKey], ",") - return nil + for _, portAnnotation := range portsAnnotations { + if portAnnotation != "" { + definedPort, err := strconv.ParseInt(portAnnotation, 10, 32) + if err != nil { + t.log.Infof("failed to read Annotations/Port: %s due to %s", + t.serviceExport.ObjectMeta.Annotations[portAnnotationsKey], err) + } else { + definedPorts[int32(definedPort)] = struct{}{} + } + } + } + } else if t.backendRef.Port() != nil { + backendRefPort := int32(*t.backendRef.Port()) + if backendRefPort != undefinedPort { + definedPorts[backendRefPort] = struct{}{} + } + } + return definedPorts } type latticeTargetsModelBuildTask struct { - log gwlog.Logger - client client.Client - tgName string - tgNamespace string - routeName string - backendRefPort int32 - latticeTargets *model.Targets - stack core.Stack - datastore *latticestore.LatticeDataStore - route core.Route + log gwlog.Logger + client client.Client + serviceExport *mcsv1alpha1.ServiceExport + service *corev1.Service + backendRef core.BackendRef + stack core.Stack + stackTgId string } diff --git a/pkg/gateway/model_build_targets_test.go b/pkg/gateway/model_build_targets_test.go index e066d2d5..ad6cf455 100644 --- a/pkg/gateway/model_build_targets_test.go +++ b/pkg/gateway/model_build_targets_test.go @@ -2,51 +2,48 @@ package gateway import ( "context" - "fmt" - "reflect" - "testing" - "time" - + "github.com/aws/aws-application-networking-k8s/pkg/model/core" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" clientgoscheme "k8s.io/client-go/kubernetes/scheme" + "reflect" testclient "sigs.k8s.io/controller-runtime/pkg/client/fake" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" - - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" - "github.com/aws/aws-application-networking-k8s/pkg/model/core" - model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - - "k8s.io/apimachinery/pkg/util/intstr" + "testing" ) func Test_Targets(t *testing.T) { + namespacePtr := func(ns string) *gwv1beta1.Namespace { + p := gwv1beta1.Namespace(ns) + return &p + } + kindPtr := func(k string) *gwv1beta1.Kind { + p := gwv1beta1.Kind(k) + return &p + } + tests := []struct { name string - srvExportName string - srvExportNamespace string port int32 endPoints []corev1.Endpoints svc corev1.Service serviceExport mcsv1alpha1.ServiceExport - inDataStore bool refByServiceExport bool refByService bool wantErrIsNil bool expectedTargetList []model.Target - route core.Route }{ { - name: "Add all endpoints to build spec", - srvExportName: "export1", - srvExportNamespace: "ns1", - port: 0, + name: "Add all endpoints to build spec", + port: 0, endPoints: []corev1.Endpoints{ { ObjectMeta: metav1.ObjectMeta{ @@ -68,7 +65,6 @@ func Test_Targets(t *testing.T) { DeletionTimestamp: nil, }, }, - inDataStore: true, refByService: true, wantErrIsNil: true, expectedTargetList: []model.Target{ @@ -83,10 +79,8 @@ func Test_Targets(t *testing.T) { }, }, { - name: "Add endpoints with matching service port to build spec", - srvExportName: "export1", - srvExportNamespace: "ns1", - port: 80, + name: "Add endpoints with matching service port to build spec", + port: 80, endPoints: []corev1.Endpoints{ { ObjectMeta: metav1.ObjectMeta{ @@ -122,7 +116,6 @@ func Test_Targets(t *testing.T) { }, }, }, - inDataStore: true, refByService: true, wantErrIsNil: true, expectedTargetList: []model.Target{ @@ -137,10 +130,8 @@ func Test_Targets(t *testing.T) { }, }, { - name: "Add all endpoints to build spec with port annotation", - srvExportName: "export1", - srvExportNamespace: "ns1", - port: 3090, + name: "Add all endpoints to build spec with port annotation", + port: 3090, endPoints: []corev1.Endpoints{ { ObjectMeta: metav1.ObjectMeta{ @@ -184,7 +175,6 @@ func Test_Targets(t *testing.T) { Annotations: map[string]string{"multicluster.x-k8s.io/port": "81"}, }, }, - inDataStore: true, refByServiceExport: true, wantErrIsNil: true, expectedTargetList: []model.Target{ @@ -198,126 +188,9 @@ func Test_Targets(t *testing.T) { }, }, }, - { - name: "Delete svc and all endpoints to build spec", - srvExportName: "export1", - srvExportNamespace: "ns1", - port: 0, - endPoints: []corev1.Endpoints{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "ns1", - Name: "export1", - }, - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{{IP: "10.10.1.1"}, {IP: "10.10.2.2"}}, - Ports: []corev1.EndpointPort{{Name: "a", Port: 8675}, {Name: "b", Port: 309}}, - }, - }, - }, - }, - svc: corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "ns1", - Name: "export1", - DeletionTimestamp: &metav1.Time{ - Time: time.Now(), - }, - }, - }, - inDataStore: true, - refByServiceExport: true, - wantErrIsNil: true, - expectedTargetList: nil, - }, - { - name: "Delete svc and no endpoints to build spec", - srvExportName: "export1", - srvExportNamespace: "ns1", - port: 0, - endPoints: []corev1.Endpoints{}, - svc: corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "ns1", - Name: "export1", - DeletionTimestamp: &metav1.Time{ - Time: time.Now(), - }, - }, - }, - inDataStore: true, - refByServiceExport: true, - wantErrIsNil: true, - expectedTargetList: nil, - }, - { - name: "Endpoints without TargetGroup", - srvExportName: "export2", - srvExportNamespace: "ns1", - port: 0, - endPoints: []corev1.Endpoints{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "ns1", - Name: "export1", - }, - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{{IP: "10.10.1.1"}, {IP: "10.10.2.2"}}, - Ports: []corev1.EndpointPort{{Name: "a", Port: 8675}, {Name: "b", Port: 309}}, - }, - }, - }, - }, - svc: corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "ns1", - Name: "export1", - DeletionTimestamp: nil, - }, - }, - inDataStore: false, - refByServiceExport: true, - wantErrIsNil: false, - }, - { - name: "Endpoints's TargetGroup is NOT referenced by serviceexport", - srvExportName: "export3", - srvExportNamespace: "ns1", - port: 0, - endPoints: []corev1.Endpoints{ - { - ObjectMeta: metav1.ObjectMeta{ - Namespace: "ns1", - Name: "export1", - }, - Subsets: []corev1.EndpointSubset{ - { - Addresses: []corev1.EndpointAddress{{IP: "10.10.1.1"}, {IP: "10.10.2.2"}}, - Ports: []corev1.EndpointPort{{Name: "a", Port: 8675}, {Name: "b", Port: 309}}, - }, - }, - }, - }, - svc: corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Namespace: "ns1", - Name: "export1", - DeletionTimestamp: nil, - }, - }, - inDataStore: true, - refByServiceExport: false, - refByService: false, - wantErrIsNil: false, - }, { name: "Endpoints does NOT exists", - srvExportName: "export4", - srvExportNamespace: "ns1", port: 0, - inDataStore: false, refByServiceExport: true, wantErrIsNil: false, svc: corev1.Service{ @@ -329,10 +202,8 @@ func Test_Targets(t *testing.T) { }, }, { - name: "Add all endpoints to build spec", - srvExportName: "export5", - srvExportNamespace: "ns1", - port: 0, + name: "Add all endpoints to build spec", + port: 0, endPoints: []corev1.Endpoints{ { ObjectMeta: metav1.ObjectMeta{ @@ -354,7 +225,6 @@ func Test_Targets(t *testing.T) { DeletionTimestamp: nil, }, }, - inDataStore: true, refByService: true, wantErrIsNil: true, expectedTargetList: []model.Target{ @@ -377,10 +247,8 @@ func Test_Targets(t *testing.T) { }, }, { - name: "Only add endpoints for port 8675 to build spec", - srvExportName: "export6", - srvExportNamespace: "ns1", - port: 8675, + name: "Only add endpoints for port 8675 to build spec", + port: 8675, endPoints: []corev1.Endpoints{ { ObjectMeta: metav1.ObjectMeta{ @@ -402,7 +270,6 @@ func Test_Targets(t *testing.T) { DeletionTimestamp: nil, }, }, - inDataStore: true, refByServiceExport: true, wantErrIsNil: true, expectedTargetList: []model.Target{ @@ -415,12 +282,17 @@ func Test_Targets(t *testing.T) { Port: 8675, }, }, + serviceExport: mcsv1alpha1.ServiceExport{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "ns1", + Name: "export6", + DeletionTimestamp: nil, + }, + }, }, { - name: "BackendRef port does not match service port", - srvExportName: "export7", - srvExportNamespace: "ns1", - port: 8750, + name: "BackendRef port does not match service port", + port: 8750, endPoints: []corev1.Endpoints{ { ObjectMeta: metav1.ObjectMeta{ @@ -442,7 +314,6 @@ func Test_Targets(t *testing.T) { DeletionTimestamp: nil, }, }, - inDataStore: false, refByService: true, refByServiceExport: true, wantErrIsNil: false, @@ -470,49 +341,40 @@ func Test_Targets(t *testing.T) { assert.NoError(t, k8sClient.Create(ctx, tt.svc.DeepCopy())) - ds := latticestore.NewLatticeDataStore() - - if tt.inDataStore { - tgName := latticestore.TargetGroupName(tt.srvExportName, tt.srvExportNamespace) - err := ds.AddTargetGroup(tgName, "", "", "", false, "") - assert.Nil(t, err) - if tt.refByServiceExport { - ds.SetTargetGroupByServiceExport(tgName, false, true) - } - if tt.refByService { - ds.SetTargetGroupByBackendRef(tgName, "", false, true) - } + br := gwv1beta1.HTTPBackendRef{} + br.Name = "name" + br.Namespace = namespacePtr("ns") + br.Kind = kindPtr("Service") + br.Port = PortNumberPtr(int(tt.port)) + corebr := core.NewHTTPBackendRef(br) + nsn := types.NamespacedName{ + Name: "stack", + Namespace: "ns", } + stack := core.NewDefaultStack(core.StackID(nsn)) + builder := NewTargetsBuilder(gwlog.FallbackLogger, k8sClient, stack) - srvName := types.NamespacedName{ - Name: tt.srvExportName, - Namespace: tt.srvExportNamespace, - } - targetTask := &latticeTargetsModelBuildTask{ - log: gwlog.FallbackLogger, - client: k8sClient, - tgName: tt.srvExportName, - tgNamespace: tt.srvExportNamespace, - datastore: ds, - backendRefPort: tt.port, - stack: core.NewDefaultStack(core.StackID(srvName)), - route: tt.route, + var err error + if tt.refByServiceExport { + _, err = builder.BuildForServiceExport(ctx, &tt.serviceExport, "tg-id") + } else { + _, err = builder.Build(ctx, &tt.svc, &corebr, "tg-id") } - err := targetTask.buildLatticeTargets(ctx) - if tt.wantErrIsNil { - assert.Nil(t, err) - - fmt.Printf("t.latticeTargets %v \n", targetTask.latticeTargets) - assert.Equal(t, tt.srvExportName, targetTask.latticeTargets.Spec.Name) - assert.Equal(t, tt.srvExportNamespace, targetTask.latticeTargets.Spec.Namespace) - - // verify targets, ports are built correctly - assert.Equal(t, tt.expectedTargetList, targetTask.latticeTargets.Spec.TargetIPList) - } else { + if !tt.wantErrIsNil { assert.NotNil(t, err) + return } + assert.Nil(t, err) + + var stackTargets []*model.Targets + _ = stack.ListResources(&stackTargets) + assert.Equal(t, 1, len(stackTargets)) + st := stackTargets[0] + + assert.Equal(t, "tg-id", st.Spec.StackTargetGroupId) + assert.Equal(t, tt.expectedTargetList, st.Spec.TargetList) }) } } diff --git a/pkg/k8s/finalizer_mock.go b/pkg/k8s/finalizer_mock.go new file mode 100644 index 00000000..0525e6c3 --- /dev/null +++ b/pkg/k8s/finalizer_mock.go @@ -0,0 +1,74 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: ./pkg/k8s/finalizer.go + +// Package k8s is a generated GoMock package. +package k8s + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" + client "sigs.k8s.io/controller-runtime/pkg/client" +) + +// MockFinalizerManager is a mock of FinalizerManager interface. +type MockFinalizerManager struct { + ctrl *gomock.Controller + recorder *MockFinalizerManagerMockRecorder +} + +// MockFinalizerManagerMockRecorder is the mock recorder for MockFinalizerManager. +type MockFinalizerManagerMockRecorder struct { + mock *MockFinalizerManager +} + +// NewMockFinalizerManager creates a new mock instance. +func NewMockFinalizerManager(ctrl *gomock.Controller) *MockFinalizerManager { + mock := &MockFinalizerManager{ctrl: ctrl} + mock.recorder = &MockFinalizerManagerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockFinalizerManager) EXPECT() *MockFinalizerManagerMockRecorder { + return m.recorder +} + +// AddFinalizers mocks base method. +func (m *MockFinalizerManager) AddFinalizers(ctx context.Context, object client.Object, finalizers ...string) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, object} + for _, a := range finalizers { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "AddFinalizers", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddFinalizers indicates an expected call of AddFinalizers. +func (mr *MockFinalizerManagerMockRecorder) AddFinalizers(ctx, object interface{}, finalizers ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, object}, finalizers...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddFinalizers", reflect.TypeOf((*MockFinalizerManager)(nil).AddFinalizers), varargs...) +} + +// RemoveFinalizers mocks base method. +func (m *MockFinalizerManager) RemoveFinalizers(ctx context.Context, object client.Object, finalizers ...string) error { + m.ctrl.T.Helper() + varargs := []interface{}{ctx, object} + for _, a := range finalizers { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "RemoveFinalizers", varargs...) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveFinalizers indicates an expected call of RemoveFinalizers. +func (mr *MockFinalizerManagerMockRecorder) RemoveFinalizers(ctx, object interface{}, finalizers ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{ctx, object}, finalizers...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveFinalizers", reflect.TypeOf((*MockFinalizerManager)(nil).RemoveFinalizers), varargs...) +} diff --git a/pkg/latticestore/introspect.go b/pkg/latticestore/introspect.go deleted file mode 100644 index 1888b3ad..00000000 --- a/pkg/latticestore/introspect.go +++ /dev/null @@ -1,115 +0,0 @@ -package latticestore - -import ( - "encoding/json" - "net" - "net/http" - "strings" - "time" - - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" - "github.com/aws/aws-application-networking-k8s/pkg/utils/retry" -) - -const ( - // defaultIntrospectionAddress is listening on localhost 61679 for ipamd introspection - defaultIntrospectionBindAddress = "0.0.0.0:61680" -) - -type rootResponse struct { - AvailableCommands []string -} - -// LoggingHandler is a object for handling http request -type LoggingHandler struct { - h http.Handler -} - -func (lh LoggingHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { - gwlog.FallbackLogger.Debugf("Handling http request: %s, from: %s, URI: %s\n", r.Method, r.RemoteAddr, r.RequestURI) - lh.h.ServeHTTP(w, r) -} - -func (c *LatticeDataStore) ServeIntrospection() { - gwlog.FallbackLogger.Debugf("Starting LatticeDataStore serve Introspection\n") - - server := c.setupIntrospectionServer() - for { - _ = retry.WithBackoff(retry.NewSimpleBackoff(time.Second, time.Minute, 0.2, 2), func() error { - var ln net.Listener - var err error - - if strings.HasPrefix(server.Addr, "unix:") { - socket := strings.TrimPrefix(server.Addr, "unix:") - ln, err = net.Listen("unix", socket) - } else { - ln, err = net.Listen("tcp", server.Addr) - } - - if err == nil { - err = server.Serve(ln) - } - - return err - }) - } -} - -func (c *LatticeDataStore) setupIntrospectionServer() *http.Server { - serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){ - "/v1/latticecache": latticecacheHandler(c), - } - paths := make([]string, 0, len(serverFunctions)) - for path := range serverFunctions { - paths = append(paths, path) - } - availableCommands := &rootResponse{paths} - // Autogenerated list of the above serverFunctions paths - availableCommandResponse, err := json.Marshal(&availableCommands) - - if err != nil { - gwlog.FallbackLogger.Debugf("Failed to marshal: %s", err) - } - - defaultHandler := func(w http.ResponseWriter, r *http.Request) { - gwlog.FallbackLogger.Debug(w.Write(availableCommandResponse)) - } - serveMux := http.NewServeMux() - serveMux.HandleFunc("/", defaultHandler) - for key, fn := range serverFunctions { - serveMux.HandleFunc(key, fn) - } - - // Log all requests and then pass through to serveMux - loggingServeMux := http.NewServeMux() - loggingServeMux.Handle("/", LoggingHandler{serveMux}) - - addr := defaultIntrospectionBindAddress - - gwlog.FallbackLogger.Infof("Serving introspection endpoints on %s", addr) - - server := &http.Server{ - Addr: addr, - Handler: loggingServeMux, - ReadTimeout: 5 * time.Second, - WriteTimeout: 5 * time.Second, - } - return server -} - -func latticecacheHandler(c *LatticeDataStore) func(http.ResponseWriter, *http.Request) { - return func(w http.ResponseWriter, r *http.Request) { - store := dumpCurrentLatticeDataStore(c) - //TODO - responseJSON, err := json.Marshal(store) - - if err != nil { - gwlog.FallbackLogger.Errorf("Failed to marshal latticecache %s", err) - http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError) - return - } - - gwlog.FallbackLogger.Debugf("store :%v", store) - gwlog.FallbackLogger.Debug(w.Write(responseJSON)) - } -} diff --git a/pkg/latticestore/latticestore.go b/pkg/latticestore/latticestore.go deleted file mode 100644 index 000b9e2b..00000000 --- a/pkg/latticestore/latticestore.go +++ /dev/null @@ -1,392 +0,0 @@ -package latticestore - -import ( - "errors" - "fmt" - "sync" - - "github.com/aws/aws-application-networking-k8s/pkg/utils" - "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" -) - -// ERROR CODE -const ( - DATASTORE_TG_NOT_EXIST = "target Group does not exist in Data Store" - DATASTORE_LISTENER_NOT_EXIST = "listener does not exist in Data Store" -) - -// this package is used to cache lattice info that relates to K8S object. -// e.g. the AWSARN for the matching K8S object - -type ListenerKey struct { - Name string - Namespace string - Port int64 - Protocol string - //TODO for TLS we need to add Protocol -} - -type Listener struct { - Key ListenerKey - ARN string - ID string -} - -type ListenerPool map[ListenerKey]*Listener - -type TargetGroupKey struct { - Name string - RouteName string - IsServiceImport bool -} - -type TargetGroup struct { - TargetGroupKey TargetGroupKey - ARN string - ID string - EndPoints []Target - VpcID string - ByServiceExport bool // triggered by K8S serviceexport object - ByBackendRef bool // triggered by backend ref which points to service -} - -type Target struct { - TargetIP string - TargetPort int64 -} - -type TargetGroupPool map[TargetGroupKey]*TargetGroup - -type LatticeDataStore struct { - log gwlog.Logger - lock sync.Mutex - targetGroups TargetGroupPool - listeners ListenerPool -} - -type LatticeDataStoreInfo struct { - TargetGroups map[string]TargetGroup - Listeners map[string]Listener -} - -var defaultLatticeDataStore *LatticeDataStore - -func NewLatticeDataStoreWithLog(log gwlog.Logger) *LatticeDataStore { - defaultLatticeDataStore = &LatticeDataStore{ - log: log, - targetGroups: make(TargetGroupPool), - listeners: make(ListenerPool), - } - return defaultLatticeDataStore -} - -func NewLatticeDataStore() *LatticeDataStore { - return NewLatticeDataStoreWithLog(gwlog.FallbackLogger) -} - -func dumpCurrentLatticeDataStore(ds *LatticeDataStore) *LatticeDataStoreInfo { - ds.lock.Lock() - defer ds.lock.Unlock() - - var store = LatticeDataStoreInfo{ - TargetGroups: make(map[string]TargetGroup), - Listeners: make(map[string]Listener), - } - - for tgkey, targetgroup := range ds.targetGroups { - - key := fmt.Sprintf("%s-%s", tgkey.Name, targetgroup.VpcID) - store.TargetGroups[key] = *targetgroup - } - - for listenerKey, listener := range ds.listeners { - key := fmt.Sprintf("%s-%s-%d", listenerKey.Name, listener.Key.Namespace, listenerKey.Port) - store.Listeners[key] = *listener - } - - return &store - -} -func GetDefaultLatticeDataStore() *LatticeDataStore { - return defaultLatticeDataStore -} - -// the max tg name length is 127 -// worst case - k8s-(50)-(50)-https-http2 (117 chars) -func TargetGroupName(name, namespace string) string { - return fmt.Sprintf("k8s-%s-%s", - utils.Truncate(name, 50), - utils.Truncate(namespace, 50), - ) -} - -// worst case - (70)-(20)-(21)-https-http2 (125 chars) -func TargetGroupLongName(defaultName, routeName, vpcId string) string { - return fmt.Sprintf("%s-%s-%s", - utils.Truncate(defaultName, 70), - utils.Truncate(routeName, 20), - utils.Truncate(vpcId, 21), - ) -} - -func (ds *LatticeDataStore) AddTargetGroup(name string, vpc string, arn string, tgID string, - isServiceImport bool, routeName string) error { - ds.lock.Lock() - defer ds.lock.Unlock() - - ds.log.Debugf("AddTargetGroup, name: %s, isServiceImport: %t, vpc: %s, arn: %s", name, isServiceImport, vpc, arn) - - targetGroupKey := TargetGroupKey{ - Name: name, - RouteName: routeName, - IsServiceImport: isServiceImport, - } - - tg, ok := ds.targetGroups[targetGroupKey] - - if ok { - ds.log.Debugf("UpdateTargetGroup, name: %s, vpc: %s, arn: %s", name, vpc, arn) - if arn != "" { - tg.ARN = arn - } - tg.VpcID = vpc - - if tgID != "" { - tg.ID = tgID - } - - } else { - - ds.targetGroups[targetGroupKey] = &TargetGroup{ - TargetGroupKey: targetGroupKey, - ARN: arn, - VpcID: vpc, - ID: tgID, - ByServiceExport: false, - ByBackendRef: false, - } - tg, _ = ds.targetGroups[targetGroupKey] - } - - return nil -} - -func (ds *LatticeDataStore) SetTargetGroupByServiceExport(name string, isServiceImport bool, byServiceExport bool) error { - ds.lock.Lock() - defer ds.lock.Unlock() - - targetGroupKey := TargetGroupKey{ - Name: name, - IsServiceImport: isServiceImport, - } - - tg, ok := ds.targetGroups[targetGroupKey] - - if ok { - tg.ByServiceExport = byServiceExport - return nil - } else { - return errors.New(DATASTORE_TG_NOT_EXIST) - } - -} - -func (ds *LatticeDataStore) SetTargetGroupByBackendRef(name string, routeName string, isServiceImport bool, byBackendRef bool) error { - ds.lock.Lock() - defer ds.lock.Unlock() - - targetGroupKey := TargetGroupKey{ - Name: name, - RouteName: routeName, - IsServiceImport: isServiceImport, - } - - tg, ok := ds.targetGroups[targetGroupKey] - - if ok { - tg.ByBackendRef = byBackendRef - return nil - } else { - return errors.New(DATASTORE_TG_NOT_EXIST) - } - -} - -func (ds *LatticeDataStore) DelTargetGroup(name string, routeName string, isServiceImport bool) error { - ds.lock.Lock() - defer ds.lock.Unlock() - - ds.log.Debugf("DelTargetGroup, name: %s, isServiceImport: %t", name, isServiceImport) - - targetGroupKey := TargetGroupKey{ - Name: name, - RouteName: routeName, - IsServiceImport: isServiceImport, - } - - _, ok := ds.targetGroups[targetGroupKey] - - if !ok { - ds.log.Debugf("Deleting unknown TargetGroup, name: %s, isServiceImport: %t", name, isServiceImport) - return errors.New(DATASTORE_TG_NOT_EXIST) - } - - delete(ds.targetGroups, targetGroupKey) - return nil - -} - -func (ds *LatticeDataStore) GetTargetGroup(name string, routeName string, isServiceImport bool) (TargetGroup, error) { - ds.lock.Lock() - defer ds.lock.Unlock() - - targetGroupKey := TargetGroupKey{ - Name: name, - RouteName: routeName, - IsServiceImport: isServiceImport, - } - - tg, ok := ds.targetGroups[targetGroupKey] - - if !ok { - return TargetGroup{}, errors.New(DATASTORE_TG_NOT_EXIST) - } - - return *tg, nil - -} - -func (ds *LatticeDataStore) GetTargetGroupsByName(name string) []TargetGroup { - tgs := make([]TargetGroup, 0) - - for _, tg := range ds.targetGroups { - if tg.TargetGroupKey.Name == name && !tg.TargetGroupKey.IsServiceImport { - tgs = append(tgs, *tg) - - } - } - - return tgs -} - -func (ds *LatticeDataStore) UpdateTargetsForTargetGroup(name string, routeName string, targetList []Target) error { - ds.lock.Lock() - defer ds.lock.Unlock() - - targetGroupKey := TargetGroupKey{ - Name: name, - RouteName: routeName, - IsServiceImport: false, // only update target list in the local cluster - } - - tg, ok := ds.targetGroups[targetGroupKey] - - if !ok { - ds.log.Debugf("UpdateTargetGroup name does NOT exist: %s", name) - return errors.New(DATASTORE_TG_NOT_EXIST) - } - - tg.EndPoints = make([]Target, len(targetList)) - copy(tg.EndPoints, targetList) - - ds.log.Debugf("Success UpdateTarget Group name: %s, targetIPList: %+v", name, tg.EndPoints) - - return nil -} - -func (ds *LatticeDataStore) AddListener(name string, namespace string, port int64, protocol string, arn string, id string) error { - ds.lock.Lock() - defer ds.lock.Unlock() - - listenerKey := ListenerKey{ - Name: name, - Namespace: namespace, - Port: port, - Protocol: protocol, - } - - ds.listeners[listenerKey] = &Listener{ - Key: listenerKey, - ARN: arn, - ID: id, - } - - ds.log.Debugf("AddListener name: %s, arn: %s, id %s", name, arn, id) - - return nil -} - -func (ds *LatticeDataStore) DelListener(name string, namespace string, port int64, protocol string) error { - ds.lock.Lock() - defer ds.lock.Unlock() - - listenerKey := ListenerKey{ - Name: name, - Namespace: namespace, - Port: port, - Protocol: protocol, - } - - ds.log.Debugf("DataStore: deleting listener name: %+v", listenerKey) - _, ok := ds.listeners[listenerKey] - - if !ok { - ds.log.Debugf("Deleting unknown listener %+v", listenerKey) - return errors.New(DATASTORE_LISTENER_NOT_EXIST) - } - - delete(ds.listeners, listenerKey) - - return nil - -} - -func (ds *LatticeDataStore) GetlListener(name string, namespace string, port int64, protocol string) (Listener, error) { - ds.lock.Lock() - defer ds.lock.Unlock() - - listenerKey := ListenerKey{ - Name: name, - Namespace: namespace, - Port: port, - Protocol: protocol, - } - - listener, ok := ds.listeners[listenerKey] - - if !ok { - ds.log.Debugf("Deleting unknown listener %+v", listenerKey) - return Listener{}, errors.New(DATASTORE_LISTENER_NOT_EXIST) - } - - return *listener, nil -} - -func (ds *LatticeDataStore) GetAllListeners(name string, namespace string) ([]*Listener, error) { - var listenerList []*Listener - - ds.lock.Lock() - defer ds.lock.Unlock() - - for _, lis := range ds.listeners { - - if lis.Key.Name == name && - lis.Key.Namespace == namespace { - listener := Listener{ - Key: ListenerKey{ - Name: name, - Namespace: namespace, - Port: lis.Key.Port, - Protocol: lis.Key.Protocol, - }, - ID: lis.ID, - } - listenerList = append(listenerList, &listener) - } - } - - return listenerList, nil - -} - -//TODO delete diff --git a/pkg/latticestore/latticestore_test.go b/pkg/latticestore/latticestore_test.go deleted file mode 100644 index 60130dd0..00000000 --- a/pkg/latticestore/latticestore_test.go +++ /dev/null @@ -1,191 +0,0 @@ -package latticestore - -import ( - "errors" - "fmt" - "testing" - - "github.com/stretchr/testify/assert" -) - -// TODO: with more data -func Test_dumpCurrentLatticeDataStore(t *testing.T) { - inputDataStore := NewLatticeDataStore() - - store := dumpCurrentLatticeDataStore(inputDataStore) - - fmt.Printf("store:%v \n", store) - - assert.NotEqual(t, store, nil, "Expected store not nil") -} - -func Test_GetDefaultLatticeDataStore(t *testing.T) { - inputDataStore := NewLatticeDataStore() - defaultDataStore := GetDefaultLatticeDataStore() - - assert.Equal(t, inputDataStore, defaultDataStore, "") -} - -func Test_TargetGroup(t *testing.T) { - inputDataStore := NewLatticeDataStore() - - name := "tg1" - name1 := "tg2" - unknowntg := "unknowntg" - namespace := "default" - namespace1 := "ns" - vpc := "vpc-123" - tgID := "1234" - arn := "arn" - serviceImport := true - //byBackendRef := true - //byServiceExport := false - K8SService := false - routeName := "httproute" - - // GetTargetGroup on an unknown TG - tgName := TargetGroupName(name, namespace) - _, err := inputDataStore.GetTargetGroup(tgName, routeName, serviceImport) - assert.Equal(t, errors.New(DATASTORE_TG_NOT_EXIST), err) - - // Happy Path for a serviceImport - err = inputDataStore.AddTargetGroup(tgName, vpc, arn, tgID, serviceImport, "") - assert.Nil(t, err) - - store := dumpCurrentLatticeDataStore(inputDataStore) - fmt.Printf("store:%v \n", store) - - assert.Equal(t, 1, len(store.TargetGroups), "") - - // Verify GetTargetGroup return TG just added - tg, err := inputDataStore.GetTargetGroup(tgName, "", serviceImport) - assert.Nil(t, err) - assert.Equal(t, vpc, tg.VpcID) - assert.Equal(t, arn, tg.ARN) - assert.Equal(t, tgID, tg.ID) - // by default - assert.Equal(t, false, tg.ByBackendRef) - assert.Equal(t, false, tg.ByServiceExport) - - inputDataStore.SetTargetGroupByBackendRef(tgName, "", serviceImport, true) - tg, err = inputDataStore.GetTargetGroup(tgName, "", serviceImport) - assert.Nil(t, err) - assert.Equal(t, true, tg.ByBackendRef) - - inputDataStore.SetTargetGroupByServiceExport(tgName, serviceImport, true) - tg, err = inputDataStore.GetTargetGroup(tgName, "", serviceImport) - assert.Nil(t, err) - assert.Equal(t, true, tg.ByServiceExport) - - // Verify GetTargetGroup will fail if it is K8SService - _, err = inputDataStore.GetTargetGroup(tgName, "", K8SService) - assert.Equal(t, errors.New(DATASTORE_TG_NOT_EXIST), err) - - // Add same TG again, no impact - err = inputDataStore.AddTargetGroup(tgName, vpc, arn, tgID, serviceImport, "") - assert.Nil(t, err) - - store = dumpCurrentLatticeDataStore(inputDataStore) - fmt.Printf("store:%v \n", store) - assert.Equal(t, 1, len(store.TargetGroups), "") - - // add 2nd TG - tgName1 := TargetGroupName(name1, namespace1) - err = inputDataStore.AddTargetGroup(tgName1, vpc, arn, tgID, K8SService, routeName) - assert.Nil(t, err) - - store = dumpCurrentLatticeDataStore(inputDataStore) - fmt.Printf("store:%v \n", store) - assert.Equal(t, 2, len(store.TargetGroups), "") - - // add targets - var targets []Target - targets = append(targets, Target{TargetIP: "1.1.1.1", TargetPort: 10}) - targets = append(targets, Target{TargetIP: "2.2.2.2", TargetPort: 20}) - unknownTGName := TargetGroupName(unknowntg, namespace) - // Update an unknown TG - err = inputDataStore.UpdateTargetsForTargetGroup(unknownTGName, routeName, targets) - assert.Equal(t, errors.New(DATASTORE_TG_NOT_EXIST), err) - - // update with the correct name - err = inputDataStore.UpdateTargetsForTargetGroup(tgName1, routeName, targets) - assert.Nil(t, err) - - store = dumpCurrentLatticeDataStore(inputDataStore) - fmt.Printf("store:%v \n", store) - - // Update targets - targets = append(targets, Target{TargetIP: "3.3.3.3", TargetPort: 30}) - err = inputDataStore.UpdateTargetsForTargetGroup(tgName1, routeName, targets) - assert.Nil(t, err) - - store = dumpCurrentLatticeDataStore(inputDataStore) - fmt.Printf("store:%v \n", store) - - // delete 2nd TG - err = inputDataStore.DelTargetGroup(tgName1, routeName, K8SService) - assert.Nil(t, err) - - _, err = inputDataStore.GetTargetGroup(tgName1, routeName, K8SService) - assert.Equal(t, errors.New(DATASTORE_TG_NOT_EXIST), err) - - // delete twice - err = inputDataStore.DelTargetGroup(tgName1, routeName, K8SService) - assert.Equal(t, errors.New(DATASTORE_TG_NOT_EXIST), err) - -} - -func Test_Listener(t *testing.T) { - - ds := NewLatticeDataStore() - - listenerName1 := "listener1" - listenerNamespace1 := "default" - arn1 := "arn1" - id1 := "id1" - port1 := 80 - protocol1 := "http" - - listenerName2 := "listener2" - listenerNamespace2 := "space2" - port2 := 443 - protocol2 := "https" - arn2 := "arn2" - id2 := "id2" - - err := ds.AddListener(listenerName1, listenerNamespace1, int64(port1), protocol1, arn1, id1) - assert.NoError(t, err) - - err = ds.AddListener(listenerName1, listenerNamespace1, int64(port1), protocol1, arn1, id1) - assert.NoError(t, err) - - err = ds.AddListener(listenerName1, listenerNamespace1, int64(port2), protocol2, arn2, id2) - assert.NoError(t, err) - - err = ds.AddListener(listenerName2, listenerNamespace2, int64(port1), protocol2, arn1, id1) - assert.NoError(t, err) - - err = ds.AddListener(listenerName2, listenerNamespace2, int64(port1), protocol1, arn1, id1) - assert.NoError(t, err) - - err = ds.AddListener(listenerName2, listenerNamespace2, int64(port2), protocol2, arn2, id2) - assert.NoError(t, err) - - listenerList, err := ds.GetAllListeners(listenerName1, listenerNamespace1) - assert.NoError(t, err) - assert.Equal(t, len(listenerList), 2) - - listener, err := ds.GetlListener(listenerName1, listenerNamespace1, int64(port1), protocol1) - assert.NoError(t, err) - assert.Equal(t, listener.Key.Name, listenerName1) - assert.Equal(t, listener.Key.Namespace, listenerNamespace1) - assert.Equal(t, listener.Key.Port, int64(port1)) - assert.Equal(t, listener.ARN, arn1) - assert.Equal(t, listener.ID, id1) - - err = ds.DelListener(listenerName1, listenerNamespace1, int64(port1), protocol1) - assert.NoError(t, err) - - _, err = ds.GetlListener(listenerName1, listenerNamespace1, int64(port1), protocol1) - assert.Error(t, err) -} diff --git a/pkg/model/core/httproute.go b/pkg/model/core/httproute.go index 13c71a39..723be086 100644 --- a/pkg/model/core/httproute.go +++ b/pkg/model/core/httproute.go @@ -204,6 +204,10 @@ type HTTPBackendRef struct { r gwv1beta1.HTTPBackendRef } +func NewHTTPBackendRef(r gwv1beta1.HTTPBackendRef) HTTPBackendRef { + return HTTPBackendRef{r: r} +} + func (r *HTTPBackendRef) Weight() *int32 { return r.r.Weight } diff --git a/pkg/model/core/stack.go b/pkg/model/core/stack.go index c92bdd36..4fe2648a 100644 --- a/pkg/model/core/stack.go +++ b/pkg/model/core/stack.go @@ -1,6 +1,10 @@ package core import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" "github.com/aws/aws-application-networking-k8s/pkg/model/core/graph" "github.com/pkg/errors" "reflect" @@ -14,6 +18,9 @@ type Stack interface { // Add a resource into stack. AddResource(res Resource) error + // Get a resource by its id and type + GetResource(id string, resType Resource) (Resource, error) + // Add a dependency relationship between resources. AddDependency(dependee Resource, depender Resource) error @@ -35,8 +42,6 @@ func NewDefaultStack(stackID StackID) *defaultStack { } } -var _ Stack = &defaultStack{} - // default implementation for stack. type defaultStack struct { stackID StackID @@ -60,6 +65,20 @@ func (s *defaultStack) AddResource(res Resource) error { return nil } +// Get a resource from the pointer, then return the result +func (s *defaultStack) GetResource(id string, resType Resource) (Resource, error) { + resUID := graph.ResourceUID{ + ResType: reflect.TypeOf(resType), + ResID: id, + } + + if r, ok := s.resources[resUID]; ok { + return r, nil + } + + return nil, fmt.Errorf("resource %s not found", id) +} + // Add a dependency relationship between resources. func (s *defaultStack) AddDependency(dependee Resource, depender Resource) error { dependeeResUID := s.computeResourceUID(dependee) @@ -76,6 +95,8 @@ func (s *defaultStack) AddDependency(dependee Resource, depender Resource) error // ListResources list all resources for specific type. // pResourceSlice must be a pointer to a slice of resources, which will be filled. +// note this list is ORDERED according to the order in which resources were added +// this is to increase predictability, issue reproducibility, and for ease of testing func (s *defaultStack) ListResources(pResourceSlice interface{}) error { v := reflect.ValueOf(pResourceSlice) if v.Kind() != reflect.Ptr { @@ -87,8 +108,9 @@ func (s *defaultStack) ListResources(pResourceSlice interface{}) error { } resType := v.Type().Elem() var resForType []Resource - for resID, res := range s.resources { - if resID.ResType == resType { + for _, node := range s.resourceGraph.Nodes() { + if node.ResType == resType { + res, _ := s.resources[node] resForType = append(resForType, res) } } @@ -96,6 +118,7 @@ func (s *defaultStack) ListResources(pResourceSlice interface{}) error { for i := range resForType { v.Index(i).Set(reflect.ValueOf(resForType[i])) } + return nil } @@ -112,3 +135,14 @@ func (s *defaultStack) computeResourceUID(res Resource) graph.ResourceUID { ResID: res.ID(), } } + +func IdFromHash(res any) (string, error) { + bytes, err := json.Marshal(res) + if err != nil { + return "", err + } + + hash := sha256.Sum256(bytes) + id := fmt.Sprintf("id-%s", hex.EncodeToString(hash[:])) + return id, nil +} diff --git a/pkg/model/core/stack_id.go b/pkg/model/core/stack_id.go index 3eeb5e07..8c3ad574 100644 --- a/pkg/model/core/stack_id.go +++ b/pkg/model/core/stack_id.go @@ -5,7 +5,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -// stackID is the identifier of a stack, it must be compatible with Kubernetes namespaced name. +// stackId is the identifier of a stack, it must be compatible with Kubernetes namespaced name. type StackID types.NamespacedName // String returns the string representation of a StackID. diff --git a/pkg/model/core/stack_mock.go b/pkg/model/core/stack_mock.go index 55cae806..349d20e9 100644 --- a/pkg/model/core/stack_mock.go +++ b/pkg/model/core/stack_mock.go @@ -61,6 +61,21 @@ func (mr *MockStackMockRecorder) AddResource(res interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddResource", reflect.TypeOf((*MockStack)(nil).AddResource), res) } +// GetResource mocks base method. +func (m *MockStack) GetResource(id string, resType Resource) (Resource, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetResource", id, resType) + ret0, _ := ret[0].(Resource) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetResource indicates an expected call of GetResource. +func (mr *MockStackMockRecorder) GetResource(id, resType interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockStack)(nil).GetResource), id, resType) +} + // ListResources mocks base method. func (m *MockStack) ListResources(pResourceSlice interface{}) error { m.ctrl.T.Helper() diff --git a/pkg/model/core/stack_test.go b/pkg/model/core/stack_test.go index b9822fa7..d8dfbea1 100644 --- a/pkg/model/core/stack_test.go +++ b/pkg/model/core/stack_test.go @@ -106,3 +106,19 @@ func Test_defaultStack_ListResources(t *testing.T) { }) } } + +func Test_Get(t *testing.T) { + stack := NewDefaultStack(StackID{Namespace: "namespace", Name: "name"}) + fr := FakeResource{ + ResourceMeta: ResourceMeta{ + resType: "fake", + id: "id-B", + }, + Spec: FakeResourceSpec{}, + Status: nil, + } + stack.AddResource(&fr) + gr, err := stack.GetResource(fr.ID(), &FakeResource{}) + assert.NoError(t, err) + assert.Equal(t, &fr, gr) +} diff --git a/pkg/model/lattice/listener.go b/pkg/model/lattice/listener.go index f2ea3ffb..6b8eaa8e 100644 --- a/pkg/model/lattice/listener.go +++ b/pkg/model/lattice/listener.go @@ -8,45 +8,37 @@ type Listener struct { core.ResourceMeta `json:"-"` Spec ListenerSpec `json:"spec"` Status *ListenerStatus `json:"status,omitempty"` + IsDeleted bool `json:"isdeleted"` } type ListenerSpec struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - Port int64 `json:"port"` - Protocol string `json:"protocol"` - DefaultAction DefaultAction `json:"defaultaction"` + StackServiceId string `json:"stackserviceid"` + K8SRouteName string `json:"k8sroutename"` + K8SRouteNamespace string `json:"k8sroutenamespace"` + Port int64 `json:"port"` + Protocol string `json:"protocol"` } -type DefaultAction struct { - BackendServiceName string `json:"backendservicename"` - BackendServiceNamespace string `json:"backendservicenamespace"` -} type ListenerStatus struct { Name string `json:"name"` - Namespace string `json:"namespace"` - ListenerARN string `json:"listenerARN"` - ListenerID string `json:"listenerID"` - ServiceID string `json:"serviceID"` - Port int64 `json:"port"` - Protocol string `json:"protocol"` + ListenerArn string `json:"listenerarn"` + Id string `json:"listenerid"` + ServiceId string `json:"serviceid"` } -func NewListener(stack core.Stack, id string, port int64, protocol string, name string, namespace string, action DefaultAction) *Listener { +func NewListener(stack core.Stack, spec ListenerSpec) (*Listener, error) { + id, err := core.IdFromHash(spec) + if err != nil { + return nil, err + } listener := &Listener{ ResourceMeta: core.NewResourceMeta(stack, "AWS::VPCServiceNetwork::Listener", id), - Spec: ListenerSpec{ - Name: name, - Namespace: namespace, - Port: port, - Protocol: protocol, - DefaultAction: action, - }, - Status: nil, + Spec: spec, + Status: nil, } stack.AddResource(listener) - return listener + return listener, nil } diff --git a/pkg/model/lattice/rule.go b/pkg/model/lattice/rule.go index b7bd75ef..a4267f75 100644 --- a/pkg/model/lattice/rule.go +++ b/pkg/model/lattice/rule.go @@ -1,9 +1,8 @@ package lattice import ( - "time" - "github.com/aws/aws-sdk-go/service/vpclattice" + "time" "github.com/aws/aws-application-networking-k8s/pkg/model/core" ) @@ -15,23 +14,19 @@ type Rule struct { } const ( - MAX_NUM_OF_MATCHED_HEADERS = 5 + MaxRulePriority = 100 ) type RuleSpec struct { - ServiceName string `json:"name"` - ServiceNamespace string `json:"namespace"` - ListenerPort int64 `json:"port"` - ListenerProtocol string `json:"protocol"` - PathMatchValue string `json:"pathmatchvalue"` - PathMatchExact bool `json:"pathmatchexact"` - PathMatchPrefix bool `json:"pathmatchprefix"` - NumOfHeaderMatches int `json:"numofheadermatches"` - MatchedHeaders [MAX_NUM_OF_MATCHED_HEADERS]vpclattice.HeaderMatch - Method string `json:"method"` - RuleID string `json:"id"` - Action RuleAction `json:"action"` - CreateTime time.Time `json:"time"` + StackListenerId string `json:"stacklistenerid"` + PathMatchValue string `json:"pathmatchvalue"` + PathMatchExact bool `json:"pathmatchexact"` + PathMatchPrefix bool `json:"pathmatchprefix"` + MatchedHeaders []vpclattice.HeaderMatch `json:"matchedheaders"` + Method string `json:"method"` + Priority int64 `json:"priority"` + Action RuleAction `json:"action"` + CreateTime time.Time `json:"createtime"` } type RuleAction struct { @@ -39,39 +34,48 @@ type RuleAction struct { } type RuleTargetGroup struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - RouteName string `json:"routename"` - IsServiceImport bool `json:"isServiceImport"` - Weight int64 `json:"weight"` + StackTargetGroupId string `json:"stacktargetgroupid"` + SvcImportTG *SvcImportTargetGroup `json:"svcimporttg"` + LatticeTgId string `json:"latticetgid"` + Weight int64 `json:"weight"` +} + +type SvcImportTargetGroup struct { + EKSClusterName string `json:"eksclustername"` + K8SServiceName string `json:"k8sservicename"` + K8SServiceNamespace string `json:"k8sservicenamespace"` + VpcId string `json:"vpcid"` } type RuleStatus struct { - RuleARN string `json:"ARN"` - RuleID string `json:"ID"` - Priority int64 `json:"priority"` - ListenerID string `json:"Listner"` - ServiceID string `json:"Service"` - UpdatePriorityNeeded bool `json:"updatepriorityneeded"` - UpdateTGsNeeded bool `json:"updateTGneeded"` + Name string `json:"name"` + Arn string `json:"arn"` + Id string `json:"id"` + ServiceId string `json:"serviceid"` + ListenerId string `json:"listenerid"` + // we submit priority updates as a batch after all rules have been created/modified + // this ensures we do not set the same priority on two rules at the same time + // we have the Priority field here for convenience in these scenarios, + // so we can check for differences and update as a batch when needed + Priority int64 `json:"priority"` } -func NewRule(stack core.Stack, id string, name string, namespace string, port int64, - protocol string, action RuleAction, ruleSpec RuleSpec) *Rule { +func NewRule(stack core.Stack, spec RuleSpec) (*Rule, error) { + id, err := core.IdFromHash(spec) + if err != nil { + return nil, err + } + + if spec.CreateTime.IsZero() { + spec.CreateTime = time.Now() + } - ruleSpec.ServiceName = name - ruleSpec.ServiceNamespace = namespace - ruleSpec.ListenerPort = port - ruleSpec.ListenerProtocol = protocol - ruleSpec.RuleID = id - ruleSpec.Action = action - ruleSpec.CreateTime = time.Now() rule := &Rule{ ResourceMeta: core.NewResourceMeta(stack, "AWS::VPCServiceNetwork::Rule", id), - Spec: ruleSpec, + Spec: spec, Status: nil, } stack.AddResource(rule) - return rule + return rule, nil } diff --git a/pkg/model/lattice/service.go b/pkg/model/lattice/service.go index 4c299c18..3a95c7a9 100644 --- a/pkg/model/lattice/service.go +++ b/pkg/model/lattice/service.go @@ -9,37 +9,45 @@ type Service struct { core.ResourceMeta `json:"-"` Spec ServiceSpec `json:"spec"` Status *ServiceStatus `json:"status,omitempty"` + IsDeleted bool `json:"isdeleted"` } type ServiceSpec struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - RouteType core.RouteType - Protocols []*string `json:"protocols"` - ServiceNetworkNames []string `json:"servicenetworkhname"` - CustomerDomainName string `json:"customerdomainname"` - CustomerCertARN string `json:"customercertarn"` - IsDeleted bool + Name string `json:"name"` + Namespace string `json:"namespace"` + RouteType core.RouteType `json:"routetype"` + ServiceNetworkNames []string `json:"servicenetworkhnames"` + CustomerDomainName string `json:"customerdomainname"` + CustomerCertARN string `json:"customercertarn"` } type ServiceStatus struct { - Arn string `json:"latticeServiceARN"` - Id string `json:"latticeServiceID"` - Dns string `json:"latticeServiceDNS"` + Arn string `json:"arn"` + Id string `json:"id"` + Dns string `json:"dns"` } -func NewLatticeService(stack core.Stack, id string, spec ServiceSpec) *Service { +func NewLatticeService(stack core.Stack, spec ServiceSpec) (*Service, error) { + id := spec.LatticeServiceName() + service := &Service{ ResourceMeta: core.NewResourceMeta(stack, "AWS::VPCServiceNetwork::Service", id), Spec: spec, Status: nil, } - stack.AddResource(service) + err := stack.AddResource(service) + if err != nil { + return nil, err + } - return service + return service, nil } func (s *Service) LatticeServiceName() string { - return utils.LatticeServiceName(s.Spec.Name, s.Spec.Namespace) + return s.Spec.LatticeServiceName() +} + +func (s *ServiceSpec) LatticeServiceName() string { + return utils.LatticeServiceName(s.Name, s.Namespace) } diff --git a/pkg/model/lattice/targetgroup.go b/pkg/model/lattice/targetgroup.go index edc0cbe4..ebe9bbd4 100644 --- a/pkg/model/lattice/targetgroup.go +++ b/pkg/model/lattice/targetgroup.go @@ -1,64 +1,132 @@ package lattice import ( + "errors" + "fmt" "github.com/aws/aws-application-networking-k8s/pkg/model/core" + "github.com/aws/aws-application-networking-k8s/pkg/utils" "github.com/aws/aws-sdk-go/service/vpclattice" + "math/rand" + "reflect" ) const ( - K8SServiceNameKey = "K8SServiceName" - K8SServiceNamespaceKey = "K8SServiceNamespace" - K8SParentRefTypeKey = "K8SParentRefTypeKey" - K8SHTTPRouteNameKey = "K8SHTTPRouteName" - K8SHTTPRouteNamespaceKey = "K8SHTTPRouteNamespace" - K8SServiceExportType = "K8SServiceExportType" - K8SHTTPRouteType = "K8SHTTPRouteType" + EKSClusterNameKey = "EKSClusterName" + K8SServiceNameKey = "K8SServiceName" + K8SServiceNamespaceKey = "K8SServiceNamespace" + K8SRouteNameKey = "K8SRouteName" + K8SRouteNamespaceKey = "K8SRouteNamespace" + + K8SParentRefTypeKey = "K8SParentRefTypeKey" + + MaxNamespaceLength = 55 + MaxNameLength = 55 + RandomSuffixLength = 10 ) type TargetGroup struct { core.ResourceMeta `json:"-"` Spec TargetGroupSpec `json:"spec"` Status *TargetGroupStatus `json:"status,omitempty"` + IsDeleted bool `json:"isdeleted"` } type TargetGroupSpec struct { - Name string - Config TargetGroupConfig `json:"config"` - Type TargetGroupType - IsDeleted bool - LatticeID string -} - -type TargetGroupConfig struct { + VpcId string `json:"vpcid"` + Type TargetGroupType `json:"type"` Port int32 `json:"port"` Protocol string `json:"protocol"` ProtocolVersion string `json:"protocolversion"` - VpcID string `json:"vpcid"` IpAddressType string `json:"ipaddresstype"` - EKSClusterName string `json:"eksclustername"` - IsServiceImport bool `json:"serviceimport"` - HealthCheckConfig *vpclattice.HealthCheckConfig `json:"healthCheckConfig"` - - // the following fields are used for AWS resource tagging - IsServiceExport bool `json:"serviceexport"` - K8SServiceName string `json:"k8sservice"` - K8SServiceNamespace string `json:"k8sservicenamespace"` - K8SHTTPRouteName string `json:"k8shttproutename"` - K8SHTTPRouteNamespace string `json:"k8shttproutenamespace"` + HealthCheckConfig *vpclattice.HealthCheckConfig `json:"healthcheckconfig"` + TargetGroupTagFields +} +type TargetGroupTagFields struct { + EKSClusterName string `json:"eksclustername"` + K8SParentRefType ParentRefType `json:"k8sparentreftype"` + K8SServiceName string `json:"k8sservicename"` + K8SServiceNamespace string `json:"k8sservicenamespace"` + K8SRouteName string `json:"k8sroutename"` + K8SRouteNamespace string `json:"k8sroutenamespace"` } type TargetGroupStatus struct { - TargetGroupARN string `json:"latticeServiceARN"` - TargetGroupID string `json:"latticeServiceID"` + Name string `json:"name"` + Arn string `json:"arn"` + Id string `json:"id"` } type TargetGroupType string +type ParentRefType string +type RouteType string const ( TargetGroupTypeIP TargetGroupType = "IP" + + ParentRefTypeSvcExport ParentRefType = "ServiceExport" + ParentRefTypeHTTPRoute ParentRefType = "HTTPRoute" + ParentRefTypeGRPCRoute ParentRefType = "GRPCRoute" + ParentRefTypeInvalid ParentRefType = "INVALID" ) -func NewTargetGroup(stack core.Stack, id string, spec TargetGroupSpec) *TargetGroup { +func TGTagFieldsFromTags(tags map[string]*string) TargetGroupTagFields { + return TargetGroupTagFields{ + EKSClusterName: getMapValue(tags, EKSClusterNameKey), + K8SParentRefType: GetParentRefType(getMapValue(tags, K8SParentRefTypeKey)), + K8SServiceName: getMapValue(tags, K8SServiceNameKey), + K8SServiceNamespace: getMapValue(tags, K8SServiceNamespaceKey), + K8SRouteName: getMapValue(tags, K8SRouteNameKey), + K8SRouteNamespace: getMapValue(tags, K8SRouteNamespaceKey), + } +} + +func getMapValue(m map[string]*string, key string) string { + v, ok := m[key] + if !ok || v == nil { + return "" + } + return *v +} + +func GetParentRefType(s string) ParentRefType { + if s == "" { + return "" // empty is OK + } + + switch s { + case string(ParentRefTypeHTTPRoute): + return ParentRefTypeHTTPRoute + case string(ParentRefTypeGRPCRoute): + return ParentRefTypeGRPCRoute + case string(ParentRefTypeSvcExport): + return ParentRefTypeSvcExport + default: + return ParentRefTypeInvalid + } +} + +func TagFieldsMatch(spec TargetGroupSpec, tags TargetGroupTagFields) bool { + specTags := TargetGroupTagFields{ + EKSClusterName: spec.EKSClusterName, + K8SParentRefType: spec.K8SParentRefType, + K8SServiceName: spec.K8SServiceName, + K8SServiceNamespace: spec.K8SServiceNamespace, + K8SRouteName: spec.K8SRouteName, + K8SRouteNamespace: spec.K8SRouteNamespace, + } + return reflect.DeepEqual(specTags, tags) +} + +func NewTargetGroup(stack core.Stack, spec TargetGroupSpec) (*TargetGroup, error) { + if err := spec.Validate(); err != nil { + return nil, err + } + + id, err := core.IdFromHash(spec) + if err != nil { + return nil, err + } + tg := &TargetGroup{ ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", id), Spec: spec, @@ -67,5 +135,50 @@ func NewTargetGroup(stack core.Stack, id string, spec TargetGroupSpec) *TargetGr stack.AddResource(tg) - return tg + return tg, nil +} + +func (t *TargetGroupTagFields) IsServiceExport() bool { + return t.K8SParentRefType == ParentRefTypeSvcExport +} + +func (t *TargetGroupTagFields) IsRoute() bool { + return t.K8SParentRefType == ParentRefTypeHTTPRoute || + t.K8SParentRefType == ParentRefTypeGRPCRoute +} + +func (t *TargetGroupSpec) Validate() error { + requiredFields := []string{t.K8SServiceName, t.K8SServiceNamespace, + t.Protocol, t.ProtocolVersion, t.VpcId, t.EKSClusterName, t.IpAddressType, + string(t.K8SParentRefType)} + + for _, s := range requiredFields { + if s == "" { + return errors.New("one or more required fields are missing") + } + } + + if t.IsRoute() { + if t.K8SRouteName == "" || t.K8SRouteNamespace == "" { + return errors.New("route name or namespace missing for route-based target group") + } + } + + return nil +} + +func TgNamePrefix(spec TargetGroupSpec) string { + truncSvcNamespace := utils.Truncate(spec.K8SServiceNamespace, MaxNamespaceLength) + truncSvcName := utils.Truncate(spec.K8SServiceName, MaxNameLength) + return fmt.Sprintf("k8s-%s-%s", truncSvcNamespace, truncSvcName) +} + +func GenerateTgName(spec TargetGroupSpec) string { + // tg max name length 128 + prefix := TgNamePrefix(spec) + randomSuffix := make([]rune, RandomSuffixLength) + for i := range randomSuffix { + randomSuffix[i] = rune(rand.Intn(26) + 'a') + } + return fmt.Sprintf("%s-%s", prefix, string(randomSuffix)) } diff --git a/pkg/model/lattice/targets.go b/pkg/model/lattice/targets.go index 1b8e1062..6f7adcd6 100644 --- a/pkg/model/lattice/targets.go +++ b/pkg/model/lattice/targets.go @@ -9,21 +9,24 @@ type Targets struct { Spec TargetsSpec `json:"spec"` } +// unlike target groups, which can reference a service export, targets +// are always sourced from the local cluster. When we update targets, +// we find all the target groups linked to the specific service type TargetsSpec struct { - Name string `json:"name"` - Namespace string `json:"namespace"` - RouteName string `json:"routename"` - TargetGroupID string `json:"targetgroupID"` - TargetIPList []Target `json:"targetIPlist"` + StackTargetGroupId string `json:"stacktargetgroupid"` + TargetList []Target `json:"targetlist"` } type Target struct { - TargetIP string `json:"targetID"` + TargetIP string `json:"targetip"` Port int64 `json:"port"` } -func NewTargets(stack core.Stack, id string, spec TargetsSpec) *Targets { - +func NewTargets(stack core.Stack, spec TargetsSpec) (*Targets, error) { + id, err := core.IdFromHash(spec) + if err != nil { + return nil, err + } targets := &Targets{ ResourceMeta: core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::Targets", id), Spec: spec, @@ -31,5 +34,5 @@ func NewTargets(stack core.Stack, id string, spec TargetsSpec) *Targets { stack.AddResource(targets) - return targets + return targets, nil } diff --git a/pkg/utils/common.go b/pkg/utils/common.go index d46b16b9..39c2a847 100644 --- a/pkg/utils/common.go +++ b/pkg/utils/common.go @@ -35,8 +35,8 @@ func SliceFilter[T any](in []T, f FilterFunc[T]) []T { return out } -func LatticeServiceName(name string, namespace string) string { - return fmt.Sprintf("%s-%s", Truncate(name, 20), Truncate(namespace, 18)) +func LatticeServiceName(k8sSourceRouteName string, k8sSourceRouteNamespace string) string { + return fmt.Sprintf("%s-%s", Truncate(k8sSourceRouteName, 20), Truncate(k8sSourceRouteNamespace, 18)) } func TargetRefToLatticeResourceName( diff --git a/scripts/gen_mocks.sh b/scripts/gen_mocks.sh index e4a0f10d..c2c5fa7f 100755 --- a/scripts/gen_mocks.sh +++ b/scripts/gen_mocks.sh @@ -2,6 +2,7 @@ ## mocks for interfaces from 3rd-party project should be put inside ./mocks folder. ## mockgen version v1.5.0 mockgen -package=mock_client -destination=./mocks/controller-runtime/client/client_mocks.go sigs.k8s.io/controller-runtime/pkg/client Client +mockgen -package=mock_client -destination=./mocks/controller-runtime/client/record_mock.go k8s.io/client-go/tools/record EventRecorder mockgen -package=services -destination=./pkg/aws/services/vpclattice_mocks.go -source=./pkg/aws/services/vpclattice.go mockgen -package=aws -destination=./pkg/aws/cloud_mocks.go -source=./pkg/aws/cloud.go mockgen -package=lattice -destination=./pkg/deploy/lattice/service_network_manager_mock.go -source=./pkg/deploy/lattice/service_network_manager.go @@ -10,6 +11,9 @@ mockgen -package=lattice -destination=./pkg/deploy/lattice/targets_manager_mock. mockgen -package=lattice -destination=./pkg/deploy/lattice/service_manager_mock.go -source=./pkg/deploy/lattice/service_manager.go mockgen -package=lattice -destination=./pkg/deploy/lattice/listener_manager_mock.go -source=./pkg/deploy/lattice/listener_manager.go mockgen -package=lattice -destination=./pkg/deploy/lattice/rule_manager_mock.go -source=./pkg/deploy/lattice/rule_manager.go +mockgen -package=k8s -destination=./pkg/k8s/finalizer_mock.go -source=./pkg/k8s/finalizer.go FinalizerManager +mockgen -package=gateway -destination=./pkg/gateway/model_build_lattice_service_mock.go -source=./pkg/gateway/model_build_lattice_service.go +mockgen -package=gateway -destination=./pkg/gateway/model_build_targetgroup_mock.go -source=./pkg/gateway/model_build_targetgroup.go mockgen -package=externaldns -destination=./pkg/deploy/externaldns/dnsendpoint_manager_mock.go -source=./pkg/deploy/externaldns/dnsendpoint_manager.go # need some manual update to remote core for stack_mock.go mockgen -package=core -destination=./pkg/model/core/stack_mock.go -source=./pkg/model/core/stack.go diff --git a/test/pkg/test/framework.go b/test/pkg/test/framework.go index 619c8793..379c911d 100644 --- a/test/pkg/test/framework.go +++ b/test/pkg/test/framework.go @@ -3,6 +3,7 @@ package test import ( "context" "fmt" + an_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" "os" "reflect" "strings" @@ -28,7 +29,7 @@ import ( "github.com/samber/lo" "github.com/samber/lo/parallel" appsv1 "k8s.io/api/apps/v1" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" @@ -36,19 +37,17 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - mcs_api "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" + mcsv1alpha1 "sigs.k8s.io/mcs-api/pkg/apis/v1alpha1" "github.com/aws/aws-application-networking-k8s/controllers" - "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" - an_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" + anv1alpha1 "github.com/aws/aws-application-networking-k8s/pkg/apis/applicationnetworking/v1alpha1" "github.com/aws/aws-application-networking-k8s/pkg/config" "github.com/aws/aws-application-networking-k8s/pkg/model/core" - "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/pkg/utils" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" "github.com/aws/aws-application-networking-k8s/pkg/aws/services" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" ) type TestObject struct { @@ -61,11 +60,11 @@ var ( CurrentClusterVpcId = os.Getenv("CLUSTER_VPC_ID") TestObjects = []TestObject{ {&gwv1beta1.HTTPRoute{}, &gwv1beta1.HTTPRouteList{}}, - {&mcs_api.ServiceExport{}, &mcs_api.ServiceExportList{}}, - {&mcs_api.ServiceImport{}, &mcs_api.ServiceImportList{}}, + {&mcsv1alpha1.ServiceExport{}, &mcsv1alpha1.ServiceExportList{}}, + {&mcsv1alpha1.ServiceImport{}, &mcsv1alpha1.ServiceImportList{}}, {&gwv1beta1.Gateway{}, &gwv1beta1.GatewayList{}}, {&appsv1.Deployment{}, &appsv1.DeploymentList{}}, - {&v1.Service{}, &v1.ServiceList{}}, + {&corev1.Service{}, &corev1.ServiceList{}}, } ) @@ -74,7 +73,7 @@ func init() { utilruntime.Must(clientgoscheme.AddToScheme(testScheme)) utilruntime.Must(gwv1alpha2.AddToScheme(testScheme)) utilruntime.Must(gwv1beta1.AddToScheme(testScheme)) - utilruntime.Must(mcs_api.AddToScheme(testScheme)) + utilruntime.Must(mcsv1alpha1.AddToScheme(testScheme)) addOptionalCRDs(testScheme) } @@ -87,16 +86,16 @@ func addOptionalCRDs(scheme *runtime.Scheme) { metav1.AddToGroupVersion(scheme, dnsEndpoint) awsGatewayControllerCRDGroupVersion := schema.GroupVersion{ - Group: v1alpha1.GroupName, + Group: anv1alpha1.GroupName, Version: "v1alpha1", } - scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.TargetGroupPolicy{}, &v1alpha1.TargetGroupPolicyList{}) + scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &anv1alpha1.TargetGroupPolicy{}, &anv1alpha1.TargetGroupPolicyList{}) metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion) - scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.VpcAssociationPolicy{}, &v1alpha1.VpcAssociationPolicyList{}) + scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &anv1alpha1.VpcAssociationPolicy{}, &anv1alpha1.VpcAssociationPolicyList{}) metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion) - scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &v1alpha1.AccessLogPolicy{}, &v1alpha1.AccessLogPolicyList{}) + scheme.AddKnownTypes(awsGatewayControllerCRDGroupVersion, &anv1alpha1.AccessLogPolicy{}, &anv1alpha1.AccessLogPolicyList{}) metav1.AddToGroupVersion(scheme, awsGatewayControllerCRDGroupVersion) } @@ -109,7 +108,8 @@ type Framework struct { Log gwlog.Logger LatticeClient services.Lattice Ec2Client *ec2.EC2 - GrpcurlRunner *v1.Pod + GrpcurlRunner *corev1.Pod + DefaultTags services.Tags Cloud an_aws.Cloud } @@ -127,12 +127,13 @@ func NewFramework(ctx context.Context, log gwlog.Logger, testNamespace string) * Client: lo.Must(client.New(controllerRuntimeConfig, client.Options{Scheme: testScheme})), LatticeClient: services.NewDefaultLattice(session.Must(session.NewSession()), config.Region), // region is currently hardcoded Ec2Client: ec2.New(session.Must(session.NewSession(&aws.Config{Region: aws.String(config.Region)}))), - GrpcurlRunner: &v1.Pod{}, + GrpcurlRunner: &corev1.Pod{}, ctx: ctx, Log: log, k8sScheme: testScheme, namespace: testNamespace, controllerRuntimeConfig: controllerRuntimeConfig, + DefaultTags: an_aws.NewDefaultCloud(nil, cloudConfig).DefaultTags(), Cloud: an_aws.NewDefaultCloud(nil, cloudConfig), } SetDefaultEventuallyTimeout(3 * time.Minute) @@ -163,7 +164,7 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { if err == nil { // for err != nil, it is possible that this service network own by other account, and it is shared to current account by RAM env.Log.Infof("Found Tags for serviceNetwork %v tags: %v", *sn.Name, retrievedTags) - value, ok := retrievedTags.Tags[lattice.K8SServiceNetworkOwnedByVPC] + value, ok := retrievedTags.Tags[model.K8SServiceNetworkOwnedByVPC] if ok { g.Expect(*value).To(Not(Equal(CurrentClusterVpcId))) } @@ -178,7 +179,7 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { }) if err == nil { // for err != nil, it is possible that this service own by other account, and it is shared to current account by RAM env.Log.Infof("Found Tags for service %v tags: %v", *service.Name, retrievedTags) - value, ok := retrievedTags.Tags[lattice.K8SServiceOwnedByVPC] + value, ok := retrievedTags.Tags[model.K8SServiceOwnedByVPC] if ok { g.Expect(*value).To(Not(Equal(CurrentClusterVpcId))) } @@ -198,8 +199,8 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { }) if err == nil { env.Log.Infof("Found Tags for tg %v tags: %v", *tg.Name, retrievedTags) - tagValue, ok := retrievedTags.Tags[lattice.K8SParentRefTypeKey] - if ok && *tagValue == lattice.K8SServiceExportType { + tagValue, ok := retrievedTags.Tags[model.K8SParentRefTypeKey] + if ok && *tagValue == string(model.ParentRefTypeSvcExport) { env.Log.Infof("TargetGroup: %s was created by k8s controller, by a ServiceExport", *tg.Id) //This tg is created by k8s controller, by a ServiceExport, //ServiceExport still have a known targetGroup leaking issue, @@ -231,6 +232,65 @@ func (env *Framework) ExpectDeletedThenNotFound(ctx context.Context, objects ... } func (env *Framework) ExpectDeleted(ctx context.Context, objects ...client.Object) { + httpRouteType := reflect.TypeOf(&gwv1beta1.HTTPRoute{}) + grpcRouteType := reflect.TypeOf(&gwv1alpha2.GRPCRoute{}) + + routeObjects := []client.Object{} + + // first, find routes + for _, object := range objects { + t := reflect.TypeOf(object) + if httpRouteType == t || grpcRouteType == t { + routeObjects = append(routeObjects, object) + } + } + + if len(routeObjects) > 0 { + env.Log.Infof("Found %d route objects", len(routeObjects)) + + for _, route := range routeObjects { + // for routes, we can speed up deletion by first removing their rules + // get the latest version first tho + t := reflect.TypeOf(route) + nsName := types.NamespacedName{ + Name: route.GetName(), + Namespace: route.GetNamespace(), + } + + if httpRouteType == t { + http := &gwv1beta1.HTTPRoute{} + err := env.Get(ctx, nsName, http) + if err != nil { + env.Log.Infof("Error getting http route %s", err) + continue + } + + env.Log.Infof("Clearing http route rules for %s", http.Name) + http.Spec.Rules = make([]gwv1beta1.HTTPRouteRule, 0) + err = env.Update(ctx, http) + if err != nil { + env.Log.Infof("Error clearing http route rules %s", err) + } + } else if grpcRouteType == t { + grpc := &gwv1alpha2.GRPCRoute{} + err := env.Get(ctx, nsName, grpc) + if err != nil { + env.Log.Infof("Error getting grpc route %s", err) + continue + } + env.Log.Infof("Clearing grpc route rules for %s", grpc.Name) + grpc.Spec.Rules = make([]gwv1alpha2.GRPCRouteRule, 0) + err = env.Update(ctx, grpc) + if err != nil { + env.Log.Infof("Error clearing grpc route rules %s", err) + } + } + } + + // sleep once for all routes + env.SleepForRouteUpdate() + } + for _, object := range objects { env.Log.Infof("Deleting %s %s/%s", reflect.TypeOf(object), object.GetNamespace(), object.GetName()) err := env.Delete(ctx, object) @@ -285,43 +345,19 @@ func (env *Framework) GetServiceNetwork(ctx context.Context, gateway *gwv1beta1. func (env *Framework) GetVpcLatticeService(ctx context.Context, route core.Route) *vpclattice.ServiceSummary { var found *vpclattice.ServiceSummary - rnProvider := controllers.RouteLSNProvider{Route: route} - + latticeServiceName := utils.LatticeServiceName(route.Name(), route.Namespace()) Eventually(func(g Gomega) { - svc, err := env.LatticeClient.FindService(ctx, &rnProvider) + svc, err := env.LatticeClient.FindService(ctx, latticeServiceName) g.Expect(err).ToNot(HaveOccurred()) found = svc g.Expect(found).ToNot(BeNil()) - g.Expect(found.Status).To(Equal(lo.ToPtr(vpclattice.ServiceStatusActive))) - g.Expect(found.DnsEntry).To(ContainSubstring(rnProvider.LatticeServiceName())) + g.Expect(found.Status).To(Equal(aws.String(vpclattice.ServiceStatusActive))) + g.Expect(found.DnsEntry).To(ContainSubstring(latticeServiceName)) }).WithOffset(1).Should(Succeed()) return found } -func (env *Framework) GetTargetGroup(ctx context.Context, service *v1.Service) *vpclattice.TargetGroupSummary { - return env.GetTargetGroupWithProtocol(ctx, service, "http", "http1") -} - -func (env *Framework) GetTargetGroupWithProtocol(ctx context.Context, service *v1.Service, protocol, protocolVersion string) *vpclattice.TargetGroupSummary { - latticeTGName := fmt.Sprintf("%s-%s-%s", - latticestore.TargetGroupName(service.Name, service.Namespace), protocol, protocolVersion) - var found *vpclattice.TargetGroupSummary - Eventually(func(g Gomega) { - targetGroups, err := env.LatticeClient.ListTargetGroupsAsList(ctx, &vpclattice.ListTargetGroupsInput{}) - g.Expect(err).To(BeNil()) - for _, targetGroup := range targetGroups { - if lo.FromPtr(targetGroup.Name) == latticeTGName { - found = targetGroup - break - } - } - g.Expect(found).ToNot(BeNil()) - g.Expect(found.Status).To(Equal(lo.ToPtr(vpclattice.TargetGroupStatusActive))) - }).WithOffset(1).Should(Succeed()) - return found -} - func (env *Framework) GetFullTargetGroupFromSummary( ctx context.Context, tgSummary *vpclattice.TargetGroupSummary) *vpclattice.GetTargetGroupOutput { @@ -337,6 +373,81 @@ func (env *Framework) GetFullTargetGroupFromSummary( return tg } +func (env *Framework) GetTargetGroup(ctx context.Context, service *corev1.Service) *vpclattice.TargetGroupSummary { + return env.GetTargetGroupWithProtocol(ctx, service, vpclattice.TargetGroupProtocolHttp, vpclattice.TargetGroupProtocolVersionHttp1) +} + +func (env *Framework) GetTargetGroupWithProtocol(ctx context.Context, service *corev1.Service, protocol, protocolVersion string) *vpclattice.TargetGroupSummary { + tgSpec := model.TargetGroupSpec{ + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service.Name, + K8SServiceNamespace: service.Namespace, + }, + Protocol: strings.ToUpper(protocol), + ProtocolVersion: strings.ToUpper(protocolVersion), + } + + var found *vpclattice.TargetGroupSummary + Eventually(func(g Gomega) { + tg, err := env.FindTargetGroupFromSpec(ctx, tgSpec) + if err != nil { + gwlog.FallbackLogger.Infof("Error getting target group %s, %s due to %s", + tgSpec.K8SServiceName, tgSpec.K8SServiceNamespace, err) + } + g.Expect(err).To(BeNil()) + g.Expect(tg).ToNot(BeNil()) + g.Expect(tg.Status).To(Equal(aws.String(vpclattice.TargetGroupStatusActive))) + + found = tg + }).WithOffset(1).Should(Succeed()) + + gwlog.FallbackLogger.Infof("Found target group %s, %s", *found.Name, *found.Id) + return found +} + +func (env *Framework) FindTargetGroupFromSpec(ctx context.Context, tgSpec model.TargetGroupSpec) (*vpclattice.TargetGroupSummary, error) { + targetGroups, err := env.LatticeClient.ListTargetGroupsAsList(ctx, &vpclattice.ListTargetGroupsInput{}) + if err != nil { + return nil, err + } + + for _, targetGroup := range targetGroups { + if aws.StringValue(targetGroup.Protocol) != tgSpec.Protocol { + continue + } + tg, err := env.LatticeClient.GetTargetGroupWithContext(ctx, &vpclattice.GetTargetGroupInput{TargetGroupIdentifier: targetGroup.Id}) + if err != nil { + return nil, err + } + + if aws.StringValue(tg.Config.ProtocolVersion) != tgSpec.ProtocolVersion { + continue + } + + res, err := env.LatticeClient.ListTagsForResourceWithContext(ctx, + &vpclattice.ListTagsForResourceInput{ResourceArn: targetGroup.Arn}) + if err != nil { + return nil, err + } + + modelTags := model.TGTagFieldsFromTags(res.Tags) + if modelTags.K8SServiceName != tgSpec.K8SServiceName || modelTags.K8SServiceNamespace != tgSpec.K8SServiceNamespace { + continue + } + + // we don't always specify these on the tgSpec, but use them if present + // if they aren't present we will ignore. This isn't perfect logic but should be good enough for these tests + if (tgSpec.K8SRouteName != "" && tgSpec.K8SRouteName != modelTags.K8SRouteName) || + (tgSpec.K8SRouteNamespace != "" && tgSpec.K8SRouteNamespace != modelTags.K8SRouteNamespace) { + continue + } + + // close enough :D + return targetGroup, nil + } + return nil, nil +} + // TODO: Create a new function that only verifying deployment len(podList.Items)==*deployment.Spec.Replicas, and don't do lattice.ListTargets() api call func (env *Framework) GetTargets(ctx context.Context, targetGroup *vpclattice.TargetGroupSummary, deployment *appsv1.Deployment) []*vpclattice.TargetSummary { var found []*vpclattice.TargetSummary @@ -362,7 +473,7 @@ func (env *Framework) GetAllTargets(ctx context.Context, targetGroup *vpclattice func GetTargets(targetGroup *vpclattice.TargetGroupSummary, deployment *appsv1.Deployment, env *Framework, ctx context.Context) ([]string, []*vpclattice.TargetSummary) { env.Log.Infoln("Trying to retrieve registered targets for targetGroup", targetGroup.Name) env.Log.Infoln("deployment.Spec.Selector.MatchLabels:", deployment.Spec.Selector.MatchLabels) - podList := &v1.PodList{} + podList := &corev1.PodList{} expectedMatchingLabels := make(map[string]string, len(deployment.Spec.Selector.MatchLabels)) for k, v := range deployment.Spec.Selector.MatchLabels { expectedMatchingLabels[k] = v @@ -374,7 +485,7 @@ func GetTargets(targetGroup *vpclattice.TargetGroupSummary, deployment *appsv1.D retrievedTargets, err := env.LatticeClient.ListTargetsAsList(ctx, &vpclattice.ListTargetsInput{TargetGroupIdentifier: targetGroup.Id}) Expect(err).To(BeNil()) - podIps := utils.SliceMap(podList.Items, func(pod v1.Pod) string { return pod.Status.PodIP }) + podIps := utils.SliceMap(podList.Items, func(pod corev1.Pod) string { return pod.Status.PodIP }) return podIps, retrievedTargets } @@ -531,6 +642,6 @@ func (env *Framework) RunGrpcurlCmd(opts RunGrpcurlCmdOptions) (string, string, return env.PodExec(*env.GrpcurlRunner, cmd) } -func (env *Framework) SleepForRouteDeletion() { - time.Sleep(30 * time.Second) +func (env *Framework) SleepForRouteUpdate() { + time.Sleep(10 * time.Second) } diff --git a/test/pkg/test/pod_manager.go b/test/pkg/test/pod_manager.go index 475f3a2c..09d81d95 100644 --- a/test/pkg/test/pod_manager.go +++ b/test/pkg/test/pod_manager.go @@ -53,6 +53,11 @@ func (env *Framework) PodExec(pod corev1.Pod, cmd string) (string, string, error Stderr: &stderr, }) + // uncomment to see full output during test runs + //gwlog.FallbackLogger.Debugf("PodExec stdout: %s", stdout.String()) + //gwlog.FallbackLogger.Debugf("PodExec stderr: %s", stderr.String()) + //gwlog.FallbackLogger.Debugf("PodExec err: %s", err) + return stdout.String(), stderr.String(), err } diff --git a/test/suites/integration/access_log_policy_test.go b/test/suites/integration/access_log_policy_test.go index ab569211..c3e07567 100644 --- a/test/suites/integration/access_log_policy_test.go +++ b/test/suites/integration/access_log_policy_test.go @@ -1125,8 +1125,6 @@ var _ = Describe("Access Log Policy", Ordered, func() { // Delete HTTPRoute, Service, and Deployment defer func() { - testFramework.ExpectDeleted(ctx, route) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, route, k8sService, deployment) }() @@ -1164,8 +1162,6 @@ var _ = Describe("Access Log Policy", Ordered, func() { }).Should(Succeed()) // Delete HTTPRoute - testFramework.ExpectDeleted(ctx, route) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, route) Eventually(func(g Gomega) { @@ -1200,8 +1196,6 @@ var _ = Describe("Access Log Policy", Ordered, func() { }).Should(Succeed()) // Delete HTTPRoute - testFramework.ExpectDeleted(ctx, route) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, route) // Change ALP destination type @@ -1259,8 +1253,6 @@ var _ = Describe("Access Log Policy", Ordered, func() { AfterAll(func() { // Delete Kubernetes Routes, Services, and Deployments - testFramework.ExpectDeleted(ctx, httpRoute, grpcRoute) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, httpRoute, grpcRoute, diff --git a/test/suites/integration/byoc_test.go b/test/suites/integration/byoc_test.go index 1db03616..53a9417d 100644 --- a/test/suites/integration/byoc_test.go +++ b/test/suites/integration/byoc_test.go @@ -115,8 +115,6 @@ var _ = Describe("Bring your own certificate (BYOC)", Ordered, func() { Expect(err).To(BeNil()) log.Infof("deleted route53 hosted zone, id: %s", hostedZoneId) - testFramework.ExpectDeleted(context.TODO(), httpRoute, service, deployment) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(context.TODO(), httpRoute, service, deployment) removeGatewayBYOCListener() diff --git a/test/suites/integration/defined_target_ports_test.go b/test/suites/integration/defined_target_ports_test.go index 63a0f83e..873462e9 100644 --- a/test/suites/integration/defined_target_ports_test.go +++ b/test/suites/integration/defined_target_ports_test.go @@ -1,10 +1,9 @@ package integration import ( + "github.com/aws/aws-application-networking-k8s/pkg/utils" "os" - "github.com/aws/aws-application-networking-k8s/controllers" - "github.com/aws/aws-sdk-go/service/vpclattice" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -39,13 +38,11 @@ var _ = Describe("Defined Target Ports", func() { }) AfterEach(func() { - testFramework.ExpectDeleted(ctx, httpRoute) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, + httpRoute, serviceExport, deployment, service, - httpRoute, ) }) @@ -71,8 +68,8 @@ var _ = Describe("Defined Target Ports", func() { // Verify VPC Lattice Service exists route, _ := core.NewRoute(httpRoute) vpcLatticeService = testFramework.GetVpcLatticeService(ctx, route) - rnp := controllers.RouteLSNProvider{Route: route} - Expect(*vpcLatticeService.DnsEntry).To(ContainSubstring(rnp.LatticeServiceName())) + lsn := utils.LatticeServiceName(route.Name(), route.Namespace()) + Expect(*vpcLatticeService.DnsEntry).To(ContainSubstring(lsn)) performVerification(service, deployment, definedPorts) }) diff --git a/test/suites/integration/grpcroute_test.go b/test/suites/integration/grpcroute_test.go index 2cf25354..9eb27352 100644 --- a/test/suites/integration/grpcroute_test.go +++ b/test/suites/integration/grpcroute_test.go @@ -381,14 +381,12 @@ var _ = Describe("GRPCRoute test", Ordered, func() { }) AfterAll(func() { - testFramework.ExpectDeleted(ctx, grpcRoute) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, + grpcRoute, grpcBinService, grpcBinDeployment, grpcHelloWorldService, grpcHelloWorldDeployment, - grpcRoute, ) }) }) diff --git a/test/suites/integration/httproute_creation_test.go b/test/suites/integration/httproute_creation_test.go index 12166a1f..9d9759c0 100644 --- a/test/suites/integration/httproute_creation_test.go +++ b/test/suites/integration/httproute_creation_test.go @@ -104,14 +104,12 @@ var _ = Describe("HTTPRoute Creation", func() { }) AfterEach(func() { - testFramework.ExpectDeleted(ctx, httpRoute) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, + httpRoute, deployment, service, serviceImport, serviceExport, - httpRoute, ) }) }) diff --git a/test/suites/integration/httproute_header_match_test.go b/test/suites/integration/httproute_header_match_test.go index 3a880ac1..1ccbfaa3 100644 --- a/test/suites/integration/httproute_header_match_test.go +++ b/test/suites/integration/httproute_header_match_test.go @@ -89,7 +89,7 @@ var _ = Describe("HTTPRoute header matches", func() { stdout, _, err := testFramework.PodExec(pod, cmd) g.Expect(err).To(BeNil()) g.Expect(stdout).To(ContainSubstring("test-v3 handler pod")) - }).WithTimeout(30 * time.Second).WithOffset(1).Should(Succeed()) + }).WithTimeout(time.Minute).WithOffset(1).Should(Succeed()) // check incorrect headers Eventually(func(g Gomega) { @@ -101,8 +101,6 @@ var _ = Describe("HTTPRoute header matches", func() { }) AfterEach(func() { - testFramework.ExpectDeleted(ctx, headerMatchHttpRoute) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, headerMatchHttpRoute, service, diff --git a/test/suites/integration/httproute_method_match_test.go b/test/suites/integration/httproute_method_match_test.go index 0544c8d5..bf2a7448 100644 --- a/test/suites/integration/httproute_method_match_test.go +++ b/test/suites/integration/httproute_method_match_test.go @@ -128,8 +128,6 @@ var _ = Describe("HTTPRoute method matches", func() { }) AfterEach(func() { - testFramework.ExpectDeleted(ctx, methodMatchHttpRoute) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, methodMatchHttpRoute, deployment1, diff --git a/test/suites/integration/httproute_mutation_do_not_leak_target_group_test.go b/test/suites/integration/httproute_mutation_do_not_leak_target_group_test.go index d141c652..5d9d301e 100644 --- a/test/suites/integration/httproute_mutation_do_not_leak_target_group_test.go +++ b/test/suites/integration/httproute_mutation_do_not_leak_target_group_test.go @@ -2,7 +2,7 @@ package integration import ( "fmt" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" "github.com/aws/aws-application-networking-k8s/test/pkg/test" "github.com/aws/aws-sdk-go/service/vpclattice" . "github.com/onsi/ginkgo/v2" @@ -57,13 +57,32 @@ var _ = Describe("HTTPRoute Mutation", func() { for _, targetGroup := range targetGroups { fmt.Println("targetGroup.Name: ", *targetGroup.Name) - if strings.HasPrefix(lo.FromPtr(targetGroup.Name), latticestore.TargetGroupName(service1.Name, service1.Namespace)) { + spec1 := model.TargetGroupSpec{ + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service1.Name, + K8SServiceNamespace: service1.Namespace, + }, + } + spec2 := model.TargetGroupSpec{ + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service2.Name, + K8SServiceNamespace: service2.Namespace, + }, + } + spec3 := model.TargetGroupSpec{ + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service3.Name, + K8SServiceNamespace: service3.Namespace, + }, + } + + if strings.HasPrefix(lo.FromPtr(targetGroup.Name), model.TgNamePrefix(spec1)) { service1TgFound = true } - if strings.HasPrefix(lo.FromPtr(targetGroup.Name), latticestore.TargetGroupName(service2.Name, service2.Namespace)) { + if strings.HasPrefix(lo.FromPtr(targetGroup.Name), model.TgNamePrefix(spec2)) { service2TgFound = true } - if strings.HasPrefix(lo.FromPtr(targetGroup.Name), latticestore.TargetGroupName(service3.Name, service3.Namespace)) { + if strings.HasPrefix(lo.FromPtr(targetGroup.Name), model.TgNamePrefix(spec3)) { service3TgFound = true } } @@ -84,19 +103,39 @@ var _ = Describe("HTTPRoute Mutation", func() { service1TgFound := false service2TgFound := false service3TgFound := false + + spec1 := model.TargetGroupSpec{ + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service1.Name, + K8SServiceNamespace: service1.Namespace, + }, + } + spec2 := model.TargetGroupSpec{ + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service2.Name, + K8SServiceNamespace: service2.Namespace, + }, + } + spec3 := model.TargetGroupSpec{ + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service3.Name, + K8SServiceNamespace: service3.Namespace, + }, + } + targetGroups, err := testFramework.LatticeClient.ListTargetGroupsAsList(ctx, &vpclattice.ListTargetGroupsInput{}) fmt.Println("Retrieved targetGroups len: ", len(targetGroups)) g.Expect(err).To(BeNil()) for _, targetGroup := range targetGroups { fmt.Println("targetGroup.Name: ", *targetGroup.Name) - if strings.HasPrefix(lo.FromPtr(targetGroup.Name), latticestore.TargetGroupName(service1.Name, service1.Namespace)) { + if strings.HasPrefix(lo.FromPtr(targetGroup.Name), model.TgNamePrefix(spec1)) { service1TgFound = true } - if strings.HasPrefix(lo.FromPtr(targetGroup.Name), latticestore.TargetGroupName(service2.Name, service2.Namespace)) { + if strings.HasPrefix(lo.FromPtr(targetGroup.Name), model.TgNamePrefix(spec2)) { service2TgFound = true } - if strings.HasPrefix(lo.FromPtr(targetGroup.Name), latticestore.TargetGroupName(service3.Name, service3.Namespace)) { + if strings.HasPrefix(lo.FromPtr(targetGroup.Name), model.TgNamePrefix(spec3)) { service3TgFound = true } } @@ -107,8 +146,6 @@ var _ = Describe("HTTPRoute Mutation", func() { }) AfterEach(func() { - testFramework.ExpectDeleted(ctx, pathMatchHttpRoute) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, pathMatchHttpRoute, deployment1, diff --git a/test/suites/integration/httproute_path_match_test.go b/test/suites/integration/httproute_path_match_test.go index d24ba3dd..ebff7362 100644 --- a/test/suites/integration/httproute_path_match_test.go +++ b/test/suites/integration/httproute_path_match_test.go @@ -149,14 +149,12 @@ var _ = Describe("HTTPRoute path matches", func() { }) AfterEach(func() { - testFramework.ExpectDeleted(ctx, pathMatchHttpRoute) - testFramework.SleepForRouteDeletion() testFramework.ExpectDeletedThenNotFound(ctx, + pathMatchHttpRoute, deployment1, deployment2, service1, service2, - pathMatchHttpRoute, ) }) }) diff --git a/test/suites/integration/httproute_update_test.go b/test/suites/integration/httproute_update_test.go index 81132ae0..07123e21 100644 --- a/test/suites/integration/httproute_update_test.go +++ b/test/suites/integration/httproute_update_test.go @@ -1,9 +1,9 @@ package integration import ( - "github.com/samber/lo" - "k8s.io/apimachinery/pkg/types" - "log" + model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" + "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" + "github.com/aws/aws-sdk-go/aws" "time" . "github.com/onsi/ginkgo/v2" @@ -13,100 +13,95 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/gateway-api/apis/v1beta1" - "github.com/aws/aws-application-networking-k8s/pkg/latticestore" "github.com/aws/aws-application-networking-k8s/test/pkg/test" "github.com/aws/aws-sdk-go/service/vpclattice" - "strings" ) var _ = Describe("HTTPRoute Update", func() { var ( - pathMatchHttpRouteOne *v1beta1.HTTPRoute - pathMatchHttpRouteTwo *v1beta1.HTTPRoute - deployment1 *appsv1.Deployment - service1 *corev1.Service - deployment2 *appsv1.Deployment - service2 *corev1.Service + route1 *v1beta1.HTTPRoute + route2 *v1beta1.HTTPRoute + deployment1 *appsv1.Deployment + service1 *corev1.Service + tg1 *vpclattice.TargetGroupSummary + tg2 *vpclattice.TargetGroupSummary + err error ) - Context("Create a HTTPRoute with backendref to service1, then update the HTTPRoute with backendref to service1 "+ - "and service2, then update the HTTPRoute with backendref to just service2", func() { - - It("Updates rules correctly with corresponding target groups after each update", func() { + Context("BackendRefs to the same service use different target groups", func() { + It("Target groups for the same service are different for different routes", func() { deployment1, service1 = testFramework.NewHttpApp(test.HTTPAppOptions{Name: "test-v1", Namespace: k8snamespace}) - deployment2, service2 = testFramework.NewHttpApp(test.HTTPAppOptions{Name: "test-v2", Namespace: k8snamespace}) - - pathMatchHttpRouteOne = testFramework.NewPathMatchHttpRoute(testGateway, []client.Object{service1}, "http", - "", k8snamespace) - pathMatchHttpRouteTwo = testFramework.NewPathMatchHttpRoute(testGateway, []client.Object{service1, service2}, "http", - "", k8snamespace) + route1 = testFramework.NewPathMatchHttpRoute(testGateway, []client.Object{service1}, "http", + "route-one", k8snamespace) + route2 = testFramework.NewPathMatchHttpRoute(testGateway, []client.Object{service1}, "http", + "route-two", k8snamespace) + + r1TgSpec := model.TargetGroupSpec{ + Protocol: vpclattice.TargetGroupProtocolHttp, + ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service1.Name, + K8SServiceNamespace: service1.Namespace, + K8SRouteName: route1.Name, + K8SRouteNamespace: route1.Namespace, + }, + } + r2TgSpec := model.TargetGroupSpec{ + Protocol: vpclattice.TargetGroupProtocolHttp, + ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, + TargetGroupTagFields: model.TargetGroupTagFields{ + K8SServiceName: service1.Name, + K8SServiceNamespace: service1.Namespace, + K8SRouteName: route2.Name, + K8SRouteNamespace: route2.Namespace, + }, + } // Create Kubernetes Resources testFramework.ExpectCreated(ctx, - pathMatchHttpRouteOne, - deployment1, service1, - deployment2, - service2, - ) - - log.Println("Set the pathMatchHttpRoute to backendRefs to just service1") - checkTgs(service1, service2, true, false) - - testFramework.ExpectCreated(ctx, - pathMatchHttpRouteTwo, + deployment1, + route1, + route2, ) - testFramework.Get(ctx, types.NamespacedName{Name: pathMatchHttpRouteTwo.Name, Namespace: pathMatchHttpRouteTwo.Namespace}, pathMatchHttpRouteTwo) - testFramework.Update(ctx, pathMatchHttpRouteTwo) - log.Println("Updated the pathMatchHttpRoute to backendRefs to service1 and service2") - checkTgs(service1, service2, true, true) - - testFramework.Get(ctx, types.NamespacedName{Name: pathMatchHttpRouteOne.Name, Namespace: pathMatchHttpRouteOne.Namespace}, pathMatchHttpRouteOne) - testFramework.Update(ctx, pathMatchHttpRouteOne) // Remove pathMatchHttpRouteTwo for service2 so service is free to use again - testFramework.ExpectDeleted(ctx, pathMatchHttpRouteTwo) - testFramework.EventuallyExpectNotFound(ctx, pathMatchHttpRouteTwo) - pathMatchHttpRouteOne.Spec.Rules[0].BackendRefs[0].BackendObjectReference.Name = v1beta1.ObjectName(service2.Name) - testFramework.Update(ctx, pathMatchHttpRouteOne) - - log.Println("Updated the pathMatchHttpRoute to backendRefs to just service2") - checkTgs(service1, service2, false, true) + // we want two separate target groups + Eventually(func(g Gomega) { + tg1, err = testFramework.FindTargetGroupFromSpec(ctx, r1TgSpec) + g.Expect(err).To(BeNil()) + g.Expect(tg1).ToNot(BeNil()) + tg2, err = testFramework.FindTargetGroupFromSpec(ctx, r2TgSpec) + g.Expect(err).To(BeNil()) + g.Expect(tg2).ToNot(BeNil()) + + // without this we end up trying to delete while the tgs are still creating + g.Expect(*tg1.Status).To(Equal(vpclattice.TargetGroupStatusActive)) + g.Expect(*tg2.Status).To(Equal(vpclattice.TargetGroupStatusActive)) + }).WithPolling(15 * time.Second).WithTimeout(2 * time.Minute).Should(Succeed()) + + gwlog.FallbackLogger.Infof("Found TG1 %s and TG2 %s", aws.StringValue(tg1.Id), aws.StringValue(tg2.Id)) + Expect(aws.StringValue(tg1.Id) != aws.StringValue(tg2.Id)).To(BeTrue()) + + // deletion of one should not affect the other + testFramework.ExpectDeleted(ctx, route1) + Eventually(func(g Gomega) { + tg1, err = testFramework.FindTargetGroupFromSpec(ctx, r1TgSpec) + g.Expect(err).To(BeNil()) + g.Expect(tg1).To(BeNil()) + tg2, err = testFramework.FindTargetGroupFromSpec(ctx, r2TgSpec) + g.Expect(err).To(BeNil()) + g.Expect(tg2).ToNot(BeNil()) + }).WithPolling(15 * time.Second).WithTimeout(2 * time.Minute).Should(Succeed()) }) }) AfterEach(func() { - testFramework.ExpectDeleted(ctx, pathMatchHttpRouteOne, pathMatchHttpRouteTwo) - testFramework.SleepForRouteDeletion() - testFramework.ExpectDeletedThenNotFound(ctx, - pathMatchHttpRouteOne, - pathMatchHttpRouteTwo, + route1, + route2, deployment1, service1, - deployment2, - service2, ) }) }) - -func checkTgs(service1 *corev1.Service, service2 *corev1.Service, expectedService1TgFound bool, expectedService2TgFound bool) { - Eventually(func(g Gomega) bool { - var service1TgFound = false - var service2TgFound = false - - targetGroups, err := testFramework.LatticeClient.ListTargetGroupsAsList(ctx, &vpclattice.ListTargetGroupsInput{}) - Expect(err).To(BeNil()) - - for _, targetGroup := range targetGroups { - if strings.HasPrefix(lo.FromPtr(targetGroup.Name), latticestore.TargetGroupName(service1.Name, service1.Namespace)) { - service1TgFound = true - } - if strings.HasPrefix(lo.FromPtr(targetGroup.Name), latticestore.TargetGroupName(service2.Name, service2.Namespace)) { - service2TgFound = true - } - } - - return service1TgFound == expectedService1TgFound && service2TgFound == expectedService2TgFound - }).WithPolling(15 * time.Second).WithTimeout(2 * time.Minute).Should(BeTrue()) -} diff --git a/test/suites/integration/https_listener_weighted_rule_with_service_export_import_test.go b/test/suites/integration/https_listener_weighted_rule_with_service_export_import_test.go index 2e71a1b4..73869e62 100644 --- a/test/suites/integration/https_listener_weighted_rule_with_service_export_import_test.go +++ b/test/suites/integration/https_listener_weighted_rule_with_service_export_import_test.go @@ -145,17 +145,14 @@ var _ = Describe("Test 2 listeners with weighted httproute rules and service exp }) AfterEach(func() { - testFramework.ExpectDeleted(ctx, httpRoute) - testFramework.SleepForRouteDeletion() - testFramework.ExpectDeletedThenNotFound(ctx, + httpRoute, deployment0, service0, deployment1, service1, serviceExport1, serviceImport1, - httpRoute, ) }) }) From da5e62f6b9a5f449eb9b6ac574fcd0295187587c Mon Sep 17 00:00:00 2001 From: Erik F <16261515+erikfuller@users.noreply.github.com> Date: Tue, 31 Oct 2023 10:41:08 -0700 Subject: [PATCH 2/6] updates based on PR feedback --- controllers/route_controller.go | 3 +- controllers/route_controller_test.go | 3 +- pkg/deploy/lattice/listener_synthesizer.go | 21 +-- pkg/deploy/lattice/rule_manager.go | 8 +- pkg/deploy/lattice/rule_synthesizer.go | 52 ++---- pkg/deploy/lattice/rule_synthesizer_test.go | 2 +- pkg/deploy/lattice/service_synthesizer.go | 24 ++- pkg/deploy/lattice/target_group_manager.go | 19 +-- .../lattice/target_group_manager_test.go | 27 ++-- .../lattice/target_group_synthesizer.go | 148 +++++++++--------- .../lattice/target_group_synthesizer_test.go | 28 ++-- pkg/deploy/lattice/targets_manager.go | 17 +- pkg/deploy/lattice/targets_manager_test.go | 4 +- pkg/deploy/lattice/targets_synthesizer.go | 9 +- pkg/deploy/stack_deployer.go | 32 ++-- pkg/gateway/model_build_lattice_service.go | 8 +- pkg/gateway/model_build_targetgroup.go | 32 ++-- pkg/gateway/model_build_targetgroup_test.go | 8 +- pkg/model/core/stack.go | 21 ++- pkg/model/core/stack_mock.go | 13 +- pkg/model/core/stack_test.go | 26 ++- pkg/model/lattice/targetgroup.go | 59 ++++--- test/pkg/test/framework.go | 4 +- 23 files changed, 286 insertions(+), 282 deletions(-) diff --git a/controllers/route_controller.go b/controllers/route_controller.go index c5743a2a..b38bb7f0 100644 --- a/controllers/route_controller.go +++ b/controllers/route_controller.go @@ -92,6 +92,7 @@ func RegisterAllRouteControllers( } for _, routeInfo := range routeInfos { + brTgBuilder := gateway.NewBackendRefTargetGroupBuilder(log, mgrClient) reconciler := routeReconciler{ routeType: routeInfo.routeType, log: log, @@ -99,7 +100,7 @@ func RegisterAllRouteControllers( scheme: mgr.GetScheme(), finalizerManager: finalizerManager, eventRecorder: mgr.GetEventRecorderFor(string(routeInfo.routeType) + "route"), - modelBuilder: gateway.NewLatticeServiceBuilder(log, mgrClient), + modelBuilder: gateway.NewLatticeServiceBuilder(log, mgrClient, brTgBuilder), stackDeployer: deploy.NewLatticeServiceStackDeploy(log, cloud, mgrClient), stackMarshaller: deploy.NewDefaultStackMarshaller(), cloud: cloud, diff --git a/controllers/route_controller_test.go b/controllers/route_controller_test.go index 9ababf70..29e7aad3 100644 --- a/controllers/route_controller_test.go +++ b/controllers/route_controller_test.go @@ -254,6 +254,7 @@ func TestRouteReconciler_ReconcileCreates(t *testing.T) { mockFinalizer.EXPECT().AddFinalizers(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() mockFinalizer.EXPECT().RemoveFinalizers(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes() + brTgBuilder := gateway.NewBackendRefTargetGroupBuilder(gwlog.FallbackLogger, k8sClient) rc := routeReconciler{ routeType: core.HttpRouteType, log: gwlog.FallbackLogger, @@ -261,7 +262,7 @@ func TestRouteReconciler_ReconcileCreates(t *testing.T) { scheme: k8sScheme, finalizerManager: mockFinalizer, eventRecorder: mockEventRecorder, - modelBuilder: gateway.NewLatticeServiceBuilder(gwlog.FallbackLogger, k8sClient), + modelBuilder: gateway.NewLatticeServiceBuilder(gwlog.FallbackLogger, k8sClient, brTgBuilder), stackDeployer: deploy.NewLatticeServiceStackDeploy(gwlog.FallbackLogger, mockCloud, k8sClient), stackMarshaller: deploy.NewDefaultStackMarshaller(), cloud: mockCloud, diff --git a/pkg/deploy/lattice/listener_synthesizer.go b/pkg/deploy/lattice/listener_synthesizer.go index 43d65baf..4a4d29db 100644 --- a/pkg/deploy/lattice/listener_synthesizer.go +++ b/pkg/deploy/lattice/listener_synthesizer.go @@ -37,35 +37,38 @@ func (l *listenerSynthesizer) Synthesize(ctx context.Context) error { return err } + var listenerErr error for _, listener := range stackListeners { - resSvc, err := l.stack.GetResource(listener.Spec.StackServiceId, &model.Service{}) + svc := &model.Service{} + err := l.stack.GetResource(listener.Spec.StackServiceId, svc) if err != nil { return err } - svc, ok := resSvc.(*model.Service) - if !ok { - return errors.New("unexpected type conversion failure for service stack object") - } - // deletes are deferred to the later logic comparing existing listeners // to current listeners. if !listener.IsDeleted { status, err := l.listenerMgr.Upsert(ctx, listener, svc) if err != nil { - return fmt.Errorf("Failed ListenerManager.Upsert %s-%s due to err %s", - listener.Spec.K8SRouteName, listener.Spec.K8SRouteNamespace, err) + listenerErr = errors.Join(listenerErr, + fmt.Errorf("failed ListenerManager.Upsert %s-%s due to err %s", + listener.Spec.K8SRouteName, listener.Spec.K8SRouteNamespace, err)) + continue } listener.Status = &status } } + if listenerErr != nil { + return listenerErr + } + // All deletions happen here, we fetch all listeners for NON-deleted // services, since service deletion will delete its listeners latticeListenersAsModel, err := l.getLatticeListenersAsModels(ctx) if err != nil { - return err + listenerErr = errors.Join(listenerErr, err) } for _, latticeListenerAsModel := range latticeListenersAsModel { diff --git a/pkg/deploy/lattice/rule_manager.go b/pkg/deploy/lattice/rule_manager.go index 8ce88829..29015c25 100644 --- a/pkg/deploy/lattice/rule_manager.go +++ b/pkg/deploy/lattice/rule_manager.go @@ -80,8 +80,7 @@ func (r *defaultRuleManager) UpdatePriorities(ctx context.Context, svcId string, _, err := r.cloud.Lattice().BatchUpdateRuleWithContext(ctx, &batchRuleInput) if err != nil { - r.log.Infof("Failed BatchUpdateRule %s, %s, due to %s", svcId, listenerId, err) - return err + return fmt.Errorf("failed BatchUpdateRule %s, %s, due to %s", svcId, listenerId, err) } r.log.Infof("Success BatchUpdateRule %s, %s", svcId, listenerId) @@ -141,8 +140,7 @@ func (r *defaultRuleManager) update(ctx context.Context, res, err := r.cloud.Lattice().UpdateRuleWithContext(ctx, &uri) if err != nil { - r.log.Infof("Failed UpdateRule %s due to %s", aws.StringValue(latticeRule.Id), err) - return err + return fmt.Errorf("failed UpdateRule %s due to %s", aws.StringValue(latticeRule.Id), err) } r.log.Infof("Succcess UpdateRule %s, %s", aws.StringValue(res.Name), aws.StringValue(res.Id)) @@ -276,7 +274,7 @@ func (r *defaultRuleManager) create( res, err := r.cloud.Lattice().CreateRuleWithContext(ctx, &cri) if err != nil { - return model.RuleStatus{}, fmt.Errorf("Failed CreateRule %s, %s due to %s", latticeListenerId, latticeSvcId, err) + return model.RuleStatus{}, fmt.Errorf("failed CreateRule %s, %s due to %s", latticeListenerId, latticeSvcId, err) } r.log.Infof("Succcess CreateRule %s, %s", aws.StringValue(res.Name), aws.StringValue(res.Id)) diff --git a/pkg/deploy/lattice/rule_synthesizer.go b/pkg/deploy/lattice/rule_synthesizer.go index b2226c37..fef00044 100644 --- a/pkg/deploy/lattice/rule_synthesizer.go +++ b/pkg/deploy/lattice/rule_synthesizer.go @@ -51,16 +51,12 @@ func (r *ruleSynthesizer) resolveRuleTgIds(ctx context.Context, modelRule *model if rtg.StackTargetGroupId != "" { r.log.Debugf("Fetching TG %d from the stack (ID %s)", i, rtg.StackTargetGroupId) - resTg, err := r.stack.GetResource(rtg.StackTargetGroupId, &model.TargetGroup{}) + stackTg := &model.TargetGroup{} + err := r.stack.GetResource(rtg.StackTargetGroupId, stackTg) if err != nil { return err } - stackTg, ok := resTg.(*model.TargetGroup) - if !ok { - return errors.New("unexpected type conversion failure for target group stack object") - } - if stackTg.Status == nil { return errors.New("stack target group is missing Status field") } @@ -96,10 +92,10 @@ func (r *ruleSynthesizer) findSvcExportTG(ctx context.Context, svcImportTg model tgTags := model.TGTagFieldsFromTags(tg.targetGroupTags.Tags) - svcMatch := tgTags.IsServiceExport() && (tgTags.K8SServiceName == svcImportTg.K8SServiceName) && + svcMatch := tgTags.IsSourceTypeServiceExport() && (tgTags.K8SServiceName == svcImportTg.K8SServiceName) && (tgTags.K8SServiceNamespace == svcImportTg.K8SServiceNamespace) - clusterMatch := (svcImportTg.EKSClusterName == "") || (tgTags.EKSClusterName == svcImportTg.EKSClusterName) + clusterMatch := (svcImportTg.EKSClusterName == "") || (tgTags.ClusterName == svcImportTg.EKSClusterName) vpcMatch := (svcImportTg.VpcId == "") || (svcImportTg.VpcId == aws.StringValue(tg.getTargetGroupOutput.Config.VpcIdentifier)) @@ -188,11 +184,11 @@ func (r *ruleSynthesizer) createOrUpdateRules(ctx context.Context, rule *model.R } func (r *ruleSynthesizer) deleteStaleLatticeRules(ctx context.Context, snlRules map[snlKey]ruleIdMap) error { - var lastDelErr error + var delErr error for snl := range snlRules { allLatticeRules, err := r.ruleManager.List(ctx, snl.SvcId, snl.ListenerId) if err != nil { - return fmt.Errorf("Failed RuleManager.List %s/%s, due to %s", snl.SvcId, snl.ListenerId, err) + return fmt.Errorf("failed RuleManager.List %s/%s, due to %s", snl.SvcId, snl.ListenerId, err) } activeRules, _ := snlRules[snl] @@ -207,20 +203,17 @@ func (r *ruleSynthesizer) deleteStaleLatticeRules(ctx context.Context, snlRules if _, ok := activeRules[ruleId]; !ok { err := r.ruleManager.Delete(ctx, ruleId, snl.SvcId, snl.ListenerId) if err != nil { - r.log.Infof("Failed RuleManager.Delete %s/%s/%s, due to %s", snl.SvcId, snl.ListenerId, ruleId, err) - err = lastDelErr + delErr = errors.Join(delErr, + fmt.Errorf("failed RuleManager.Delete %s/%s/%s, due to %s", snl.SvcId, snl.ListenerId, ruleId, err)) } } } } - if lastDelErr != nil { - return lastDelErr - } - return nil + return delErr } func (r *ruleSynthesizer) adjustPriorities(ctx context.Context, snlStackRules map[snlKey]ruleIdMap, resRule []*model.Rule) error { - var lastUpdateErr error + var updateErr error for snl := range snlStackRules { activeRules, _ := snlStackRules[snl] for _, rule := range activeRules { @@ -235,41 +228,30 @@ func (r *ruleSynthesizer) adjustPriorities(ctx context.Context, snlStackRules ma err := r.ruleManager.UpdatePriorities(ctx, snl.SvcId, snl.ListenerId, rulesToUpdate) if err != nil { - r.log.Infof("Failed RuleManager.UpdatePriorities for rules %+v due to %s", resRule, err) - lastUpdateErr = err + updateErr = errors.Join(updateErr, + fmt.Errorf("failed RuleManager.UpdatePriorities for rules %+v due to %s", resRule, err)) } break } } } - if lastUpdateErr != nil { - return lastUpdateErr - } - return nil + return updateErr } func (r *ruleSynthesizer) getStackObjects(rule *model.Rule) (*model.Listener, *model.Service, error) { - resListener, err := r.stack.GetResource(rule.Spec.StackListenerId, &model.Listener{}) + listener := &model.Listener{} + err := r.stack.GetResource(rule.Spec.StackListenerId, listener) if err != nil { return nil, nil, err } - listener, ok := resListener.(*model.Listener) - if !ok { - return nil, nil, errors.New("unexpected type conversion failure for listener stack object") - } - - resSvc, err := r.stack.GetResource(listener.Spec.StackServiceId, &model.Service{}) + svc := &model.Service{} + err = r.stack.GetResource(listener.Spec.StackServiceId, svc) if err != nil { return nil, nil, err } - svc, ok := resSvc.(*model.Service) - if !ok { - return nil, nil, errors.New("unexpected type conversion failure for service stack object") - } - return listener, svc, nil } diff --git a/pkg/deploy/lattice/rule_synthesizer_test.go b/pkg/deploy/lattice/rule_synthesizer_test.go index b42d4178..c2b91288 100644 --- a/pkg/deploy/lattice/rule_synthesizer_test.go +++ b/pkg/deploy/lattice/rule_synthesizer_test.go @@ -187,7 +187,7 @@ func Test_resolveRuleTgs(t *testing.T) { model.K8SServiceNameKey: aws.String("svc-name"), model.K8SServiceNamespaceKey: aws.String("ns"), model.EKSClusterNameKey: aws.String("cluster-name"), - model.K8SParentRefTypeKey: aws.String(string(model.ParentRefTypeSvcExport)), + model.K8SSourceTypeKey: aws.String(string(model.SourceTypeSvcExport)), }}, }, }, nil) diff --git a/pkg/deploy/lattice/service_synthesizer.go b/pkg/deploy/lattice/service_synthesizer.go index 035eb538..1ef6514b 100644 --- a/pkg/deploy/lattice/service_synthesizer.go +++ b/pkg/deploy/lattice/service_synthesizer.go @@ -2,6 +2,8 @@ package lattice import ( "context" + "errors" + "fmt" "github.com/aws/aws-application-networking-k8s/pkg/deploy/externaldns" "github.com/aws/aws-application-networking-k8s/pkg/model/core" @@ -34,41 +36,37 @@ func (s *serviceSynthesizer) Synthesize(ctx context.Context) error { var resServices []*model.Service s.stack.ListResources(&resServices) - var lastErr error + var svcErr error for _, resService := range resServices { s.log.Debugf("Synthesizing service: %s-%s", resService.Spec.Name, resService.Spec.Namespace) if resService.IsDeleted { err := s.serviceManager.Delete(ctx, resService) if err != nil { - s.log.Infof("Failed ServiceManager.Delete %s-%s due to %s", - resService.Spec.Name, resService.Spec.Namespace, err) - - lastErr = err + svcErr = errors.Join(svcErr, + fmt.Errorf("failed ServiceManager.Delete %s-%s due to %s", resService.Spec.Name, resService.Spec.Namespace, err)) continue } } else { serviceStatus, err := s.serviceManager.Upsert(ctx, resService) if err != nil { - s.log.Infof("Failed ServiceManager.Upsert %s-%s due to %s", - resService.Spec.Name, resService.Spec.Namespace, err) - - lastErr = err + svcErr = errors.Join(svcErr, + fmt.Errorf("failed ServiceManager.Upsert %s-%s due to %s", resService.Spec.Name, resService.Spec.Namespace, err)) continue } resService.Status = &serviceStatus err = s.dnsEndpointManager.Create(ctx, resService) if err != nil { - s.log.Infof("Failed DnsEndpointManager.Create %s-%s due to %s", - resService.Spec.Name, resService.Spec.Namespace, err) + svcErr = errors.Join(svcErr, + fmt.Errorf("failed DnsEndpointManager.Create %s-%s due to %s", resService.Spec.Name, resService.Spec.Namespace, err)) - lastErr = err + svcErr = err continue } } } - return lastErr + return svcErr } func (s *serviceSynthesizer) PostSynthesize(ctx context.Context) error { diff --git a/pkg/deploy/lattice/target_group_manager.go b/pkg/deploy/lattice/target_group_manager.go index 33b5a63c..b7b7ee02 100644 --- a/pkg/deploy/lattice/target_group_manager.go +++ b/pkg/deploy/lattice/target_group_manager.go @@ -46,13 +46,13 @@ func (s *defaultTargetGroupManager) Upsert( } if latticeTgSummary == nil { - return s.create(ctx, modelTg, err) + return s.create(ctx, modelTg) } else { return s.update(ctx, modelTg, latticeTgSummary) } } -func (s *defaultTargetGroupManager) create(ctx context.Context, modelTg *model.TargetGroup, err error) (model.TargetGroupStatus, error) { +func (s *defaultTargetGroupManager) create(ctx context.Context, modelTg *model.TargetGroup) (model.TargetGroupStatus, error) { var ipAddressType *string if modelTg.Spec.IpAddressType != "" { ipAddressType = &modelTg.Spec.IpAddressType @@ -76,12 +76,12 @@ func (s *defaultTargetGroupManager) create(ctx context.Context, modelTg *model.T Type: &latticeTgType, Tags: s.cloud.DefaultTags(), } - createInput.Tags[model.EKSClusterNameKey] = &modelTg.Spec.EKSClusterName + createInput.Tags[model.EKSClusterNameKey] = &modelTg.Spec.ClusterName createInput.Tags[model.K8SServiceNameKey] = &modelTg.Spec.K8SServiceName createInput.Tags[model.K8SServiceNamespaceKey] = &modelTg.Spec.K8SServiceNamespace - createInput.Tags[model.K8SParentRefTypeKey] = aws.String(string(modelTg.Spec.K8SParentRefType)) + createInput.Tags[model.K8SSourceTypeKey] = aws.String(string(modelTg.Spec.K8SSourceType)) - if modelTg.Spec.IsRoute() { + if modelTg.Spec.IsSourceTypeRoute() { createInput.Tags[model.K8SRouteNameKey] = &modelTg.Spec.K8SRouteName createInput.Tags[model.K8SRouteNamespaceKey] = &modelTg.Spec.K8SRouteNamespace } @@ -207,9 +207,8 @@ func (s *defaultTargetGroupManager) Delete(ctx context.Context, modelTg *model.T isDeRegRespUnsuccessful := len(deRegResp.Unsuccessful) > 0 if isDeRegRespUnsuccessful { - s.log.Infof("Unsuccessful (%d total) DeregisterTargets %s (0->%s), will retry", + return fmt.Errorf("Unsuccessful (%d total) DeregisterTargets %s (%s), will retry", len(deRegResp.Unsuccessful), modelTg.Status.Id, aws.StringValue(deRegResp.Unsuccessful[0].FailureMessage)) - return errors.New(LATTICE_RETRY) } s.log.Infof("Success DeregisterTargets %s", modelTg.Status.Id) } @@ -223,7 +222,7 @@ func (s *defaultTargetGroupManager) Delete(ctx context.Context, modelTg *model.T s.log.Infof("Target group %s was already deleted", modelTg.Status.Id) return nil } else { - return fmt.Errorf("Failed DeleteTargetGroup %s due to %s", modelTg.Status.Id, err) + return fmt.Errorf("failed DeleteTargetGroup %s due to %s", modelTg.Status.Id, err) } } @@ -281,7 +280,9 @@ func (s *defaultTargetGroupManager) findTargetGroup( ctx context.Context, modelTargetGroup *model.TargetGroup, ) (*vpclattice.TargetGroupSummary, error) { - listInput := vpclattice.ListTargetGroupsInput{} + listInput := vpclattice.ListTargetGroupsInput{ + VpcIdentifier: aws.String(modelTargetGroup.Spec.VpcId), + } resp, err := s.cloud.Lattice().ListTargetGroupsAsList(ctx, &listInput) if err != nil { return nil, err diff --git a/pkg/deploy/lattice/target_group_manager_test.go b/pkg/deploy/lattice/target_group_manager_test.go index 6344bd13..7a0c190d 100644 --- a/pkg/deploy/lattice/target_group_manager_test.go +++ b/pkg/deploy/lattice/target_group_manager_test.go @@ -42,8 +42,8 @@ func Test_CreateTargetGroup_TGNotExist_Active(t *testing.T) { ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, } tgSpec.VpcId = config.VpcID - tgSpec.EKSClusterName = config.ClusterName - tgSpec.K8SParentRefType = model.ParentRefTypeSvcExport + tgSpec.ClusterName = config.ClusterName + tgSpec.K8SSourceType = model.SourceTypeSvcExport tgSpec.K8SServiceName = "exportsvc1" tgSpec.K8SServiceNamespace = "default" tgSpec.Type = model.TargetGroupTypeIP @@ -55,8 +55,8 @@ func Test_CreateTargetGroup_TGNotExist_Active(t *testing.T) { ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, } tgSpec.VpcId = config.VpcID - tgSpec.EKSClusterName = config.ClusterName - tgSpec.K8SParentRefType = model.ParentRefTypeHTTPRoute + tgSpec.ClusterName = config.ClusterName + tgSpec.K8SSourceType = model.SourceTypeHTTPRoute tgSpec.K8SServiceName = "backend-svc1" tgSpec.K8SServiceNamespace = "default" tgSpec.K8SRouteName = "httproute1" @@ -71,14 +71,14 @@ func Test_CreateTargetGroup_TGNotExist_Active(t *testing.T) { expectedTags := cloud.DefaultTags() expectedTags[model.K8SServiceNameKey] = &tgSpec.K8SServiceName expectedTags[model.K8SServiceNamespaceKey] = &tgSpec.K8SServiceNamespace - expectedTags[model.EKSClusterNameKey] = &tgSpec.EKSClusterName + expectedTags[model.EKSClusterNameKey] = &tgSpec.ClusterName if tgType == "by-serviceexport" { - value := string(model.ParentRefTypeSvcExport) - expectedTags[model.K8SParentRefTypeKey] = &value + value := string(model.SourceTypeSvcExport) + expectedTags[model.K8SSourceTypeKey] = &value } else if tgType == "by-backendref" { - value := string(model.ParentRefTypeHTTPRoute) - expectedTags[model.K8SParentRefTypeKey] = &value + value := string(model.SourceTypeHTTPRoute) + expectedTags[model.K8SSourceTypeKey] = &value expectedTags[model.K8SRouteNameKey] = &tgSpec.K8SRouteName expectedTags[model.K8SRouteNamespaceKey] = &tgSpec.K8SRouteNamespace } @@ -682,7 +682,6 @@ func Test_DeleteTG_DeRegisterTargetsUnsuccessfully(t *testing.T) { err := tgManager.Delete(ctx, &tgDeleteInput) assert.NotNil(t, err) - assert.Equal(t, err, errors.New(LATTICE_RETRY)) } // Delete target group fails @@ -948,7 +947,7 @@ func Test_IsTargetGroupMatch(t *testing.T) { }, }, latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, - tags: &model.TargetGroupTagFields{EKSClusterName: "foo"}, + tags: &model.TargetGroupTagFields{ClusterName: "foo"}, }, { name: "fetch tags not equal", @@ -993,13 +992,13 @@ func Test_IsTargetGroupMatch(t *testing.T) { Port: 443, ProtocolVersion: "HTTP1", TargetGroupTagFields: model.TargetGroupTagFields{ - EKSClusterName: "cluster", + ClusterName: "cluster", }, }, }, latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, tags: &model.TargetGroupTagFields{ - EKSClusterName: "cluster", + ClusterName: "cluster", }, getTgOut: &vpclattice.GetTargetGroupOutput{ Config: &vpclattice.TargetGroupConfig{ @@ -1016,7 +1015,7 @@ func Test_IsTargetGroupMatch(t *testing.T) { Port: 443, ProtocolVersion: "HTTP1", TargetGroupTagFields: model.TargetGroupTagFields{ - EKSClusterName: "cluster", + ClusterName: "cluster", }, }, }, diff --git a/pkg/deploy/lattice/target_group_synthesizer.go b/pkg/deploy/lattice/target_group_synthesizer.go index 88f3577a..6de41a84 100644 --- a/pkg/deploy/lattice/target_group_synthesizer.go +++ b/pkg/deploy/lattice/target_group_synthesizer.go @@ -3,6 +3,7 @@ package lattice import ( "context" "errors" + "fmt" "github.com/aws/aws-application-networking-k8s/pkg/gateway" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/vpclattice" @@ -21,16 +22,6 @@ import ( model "github.com/aws/aws-application-networking-k8s/pkg/model/lattice" ) -type ActionDirective bool - -const ( - // just helps a bit with readability - PerformUpserts ActionDirective = true - DoNotPerformUpserts ActionDirective = false - PerformDeletes ActionDirective = true - DoNotPerformDeletes ActionDirective = false -) - // helpful for testing/mocking func NewTargetGroupSynthesizer( log gwlog.Logger, @@ -63,16 +54,11 @@ type TargetGroupSynthesizer struct { } func (t *TargetGroupSynthesizer) Synthesize(ctx context.Context) error { - return t.synthesize(ctx, PerformUpserts, PerformDeletes) + err1 := t.SynthesizeCreate(ctx) + err2 := t.SynthesizeDelete(ctx) + return errors.Join(err1, err2) } func (t *TargetGroupSynthesizer) SynthesizeCreate(ctx context.Context) error { - return t.synthesize(ctx, PerformUpserts, DoNotPerformDeletes) -} -func (t *TargetGroupSynthesizer) SynthesizeDelete(ctx context.Context) error { - return t.synthesize(ctx, DoNotPerformUpserts, PerformDeletes) -} - -func (t *TargetGroupSynthesizer) synthesize(ctx context.Context, performUpserts ActionDirective, performDeletes ActionDirective) error { var resTargetGroups []*model.TargetGroup var returnErr = false @@ -81,42 +67,54 @@ func (t *TargetGroupSynthesizer) synthesize(ctx context.Context, performUpserts return err } - if bool(performDeletes) { - for _, resTargetGroup := range resTargetGroups { - if resTargetGroup.IsDeleted { - prefix := model.TgNamePrefix(resTargetGroup.Spec) - - err := t.targetGroupManager.Delete(ctx, resTargetGroup) - if err != nil { - t.log.Infof("Failed TargetGroupManager.Delete %s due to %s", prefix, err) - returnErr = true - } - } + for _, resTargetGroup := range resTargetGroups { + if resTargetGroup.IsDeleted { + continue } - } - if bool(performUpserts) { - for _, resTargetGroup := range resTargetGroups { - if !resTargetGroup.IsDeleted { - prefix := model.TgNamePrefix(resTargetGroup.Spec) - - tgStatus, err := t.targetGroupManager.Upsert(ctx, resTargetGroup) - if err == nil { - resTargetGroup.Status = &tgStatus - } else { - t.log.Debugf("Failed TargetGroupManager.Upsert %s due to %s", prefix, err) - returnErr = true - } - } + + prefix := model.TgNamePrefix(resTargetGroup.Spec) + + tgStatus, err := t.targetGroupManager.Upsert(ctx, resTargetGroup) + if err == nil { + resTargetGroup.Status = &tgStatus + } else { + t.log.Debugf("Failed TargetGroupManager.Upsert %s due to %s", prefix, err) + returnErr = true } } if returnErr { - t.log.Infof("Error during target group synthesis, will retry") - return errors.New(LATTICE_RETRY) + return fmt.Errorf("error during target group synthesis, will retry") } return nil } +func (t *TargetGroupSynthesizer) SynthesizeDelete(ctx context.Context) error { + var resTargetGroups []*model.TargetGroup + + err := t.stack.ListResources(&resTargetGroups) + if err != nil { + return err + } + + var retErr error + for _, resTargetGroup := range resTargetGroups { + if !resTargetGroup.IsDeleted { + continue + } + + err := t.targetGroupManager.Delete(ctx, resTargetGroup) + if err != nil { + prefix := model.TgNamePrefix(resTargetGroup.Spec) + retErr = errors.Join(retErr, fmt.Errorf("failed TargetGroupManager.Delete %s due to %s", prefix, err)) + } + } + + if retErr != nil { + return retErr + } + return nil +} // this method assumes all synthesis func (t *TargetGroupSynthesizer) SynthesizeUnusedDelete(ctx context.Context) error { @@ -125,7 +123,7 @@ func (t *TargetGroupSynthesizer) SynthesizeUnusedDelete(ctx context.Context) err return err } - retErr := false + var retErr error for _, tg := range tgsToDelete { modelStatus := model.TargetGroupStatus{ Name: aws.StringValue(tg.getTargetGroupOutput.Name), @@ -139,23 +137,21 @@ func (t *TargetGroupSynthesizer) SynthesizeUnusedDelete(ctx context.Context) err err := t.targetGroupManager.Delete(ctx, &modelTg) if err != nil { - t.log.Infof("Failed TargetGroupManager.Delete %s due to %s", modelStatus.Id, err) - retErr = true + retErr = errors.Join(retErr, fmt.Errorf("failed TargetGroupManager.Delete %s due to %s", modelStatus.Id, err)) } } - if retErr { - return errors.New(LATTICE_RETRY) - } else { - return nil + if retErr != nil { + return retErr } + + return nil } func (t *TargetGroupSynthesizer) calculateTargetGroupsToDelete(ctx context.Context) ([]tgListOutput, error) { latticeTgs, err := t.targetGroupManager.List(ctx) if err != nil { - t.log.Infof("Failed TargetGroupManager.List due to %s", err) - return latticeTgs, err + return latticeTgs, fmt.Errorf("failed TargetGroupManager.List due to %s", err) } var tgsToDelete []tgListOutput @@ -164,8 +160,15 @@ func (t *TargetGroupSynthesizer) calculateTargetGroupsToDelete(ctx context.Conte // some changes to existing service exports or routes will simply create new target groups, // for example on protocol changes for _, latticeTg := range latticeTgs { - tagFields, controllerManaged := t.isControllerManaged(latticeTg) - if !controllerManaged { + if !t.hasTags(latticeTg) || !t.vpcMatchesConfig(latticeTg) { + continue + } + + // TGs from earlier releases will require 1-time manual cleanup + // this method of validation only covers TGs created by this build + // of the controller forward + tagFields := model.TGTagFieldsFromTags(latticeTg.targetGroupTags.Tags) + if !t.hasExpectedTags(latticeTg, tagFields) { continue } @@ -176,7 +179,7 @@ func (t *TargetGroupSynthesizer) calculateTargetGroupsToDelete(ctx context.Conte continue } - if tagFields.K8SParentRefType == model.ParentRefTypeSvcExport { + if tagFields.K8SSourceType == model.SourceTypeSvcExport { if t.shouldDeleteSvcExportTg(ctx, latticeTg, tagFields) { tgsToDelete = append(tgsToDelete, latticeTg) } @@ -351,46 +354,47 @@ func (t *TargetGroupSynthesizer) shouldDeleteRouteTg( return false } -func (t *TargetGroupSynthesizer) isControllerManaged(latticeTg tgListOutput) (model.TargetGroupTagFields, bool) { +func (t *TargetGroupSynthesizer) hasTags(latticeTg tgListOutput) bool { if latticeTg.targetGroupTags == nil { t.log.Debugf("Ignoring target group %s (%s) because tag fetch was not successful", *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) - return model.TargetGroupTagFields{}, false + return false } + return true +} - // TGs from earlier releases will require 1-time manual cleanup - // this method of validation only covers TGs created by this build - // of the controller forward +func (t *TargetGroupSynthesizer) vpcMatchesConfig(latticeTg tgListOutput) bool { if aws.StringValue(latticeTg.getTargetGroupOutput.Config.VpcIdentifier) != config.VpcID { t.log.Debugf("Ignoring target group %s (%s) because it is not configured for this VPC", *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) - return model.TargetGroupTagFields{}, false + return false } + return true +} - tagFields := model.TGTagFieldsFromTags(latticeTg.targetGroupTags.Tags) - - if tagFields.EKSClusterName != config.ClusterName { +func (t *TargetGroupSynthesizer) hasExpectedTags(latticeTg tgListOutput, tagFields model.TargetGroupTagFields) bool { + if tagFields.ClusterName != config.ClusterName { t.log.Debugf("Ignoring target group %s (%s) because it is not configured for this Cluster", *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) - return model.TargetGroupTagFields{}, false + return false } - if tagFields.K8SParentRefType == model.ParentRefTypeInvalid || + if tagFields.K8SSourceType == model.SourceTypeInvalid || tagFields.K8SServiceName == "" || tagFields.K8SServiceNamespace == "" { t.log.Infof("Ignoring target group %s (%s) as one or more required tags are missing", *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) - return model.TargetGroupTagFields{}, false + return false } // route-based TGs should have the additional route keys - if tagFields.IsRoute() && (tagFields.K8SRouteName == "" || tagFields.K8SRouteNamespace == "") { + if tagFields.IsSourceTypeRoute() && (tagFields.K8SRouteName == "" || tagFields.K8SRouteNamespace == "") { t.log.Infof("Ignoring route-based target group %s (%s) as one or more required tags are missing", *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) - return model.TargetGroupTagFields{}, false + return false } - return tagFields, true + return true } func (t *TargetGroupSynthesizer) PostSynthesize(ctx context.Context) error { diff --git a/pkg/deploy/lattice/target_group_synthesizer_test.go b/pkg/deploy/lattice/target_group_synthesizer_test.go index b8834db0..ec6dd1a1 100644 --- a/pkg/deploy/lattice/target_group_synthesizer_test.go +++ b/pkg/deploy/lattice/target_group_synthesizer_test.go @@ -133,11 +133,11 @@ func Test_SynthesizeUnusedDeleteIgnoreNotManagedByController(t *testing.T) { tgInvalidParentRef := copy(tgWrongCluster) tgInvalidParentRef.targetGroupTags.Tags[model.EKSClusterNameKey] = aws.String("cluster-name") - tgInvalidParentRef.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeInvalid)) + tgInvalidParentRef.targetGroupTags.Tags[model.K8SSourceTypeKey] = aws.String(string(model.SourceTypeInvalid)) nonManagedTgs = append(nonManagedTgs, tgInvalidParentRef) tgMissingK8SServiceName := copy(tgInvalidParentRef) - tgMissingK8SServiceName.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeSvcExport)) + tgMissingK8SServiceName.targetGroupTags.Tags[model.K8SSourceTypeKey] = aws.String(string(model.SourceTypeSvcExport)) nonManagedTgs = append(nonManagedTgs, tgMissingK8SServiceName) tgMissingK8SServiceNamespace := copy(tgMissingK8SServiceName) @@ -145,7 +145,7 @@ func Test_SynthesizeUnusedDeleteIgnoreNotManagedByController(t *testing.T) { nonManagedTgs = append(nonManagedTgs, tgMissingK8SServiceNamespace) tgMissingRouteName := copy(tgMissingK8SServiceNamespace) - tgMissingRouteName.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeHTTPRoute)) + tgMissingRouteName.targetGroupTags.Tags[model.K8SSourceTypeKey] = aws.String(string(model.SourceTypeHTTPRoute)) tgMissingRouteName.targetGroupTags.Tags[model.K8SServiceNamespaceKey] = aws.String("ns-1") nonManagedTgs = append(nonManagedTgs, tgMissingRouteName) @@ -212,12 +212,12 @@ func Test_DoNotDeleteCases(t *testing.T) { tgSvcExportUpToDate := copy(baseTg) tgSvcExportUpToDate.getTargetGroupOutput.Arn = aws.String("tg-svc-export-arn") - tgSvcExportUpToDate.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeSvcExport)) + tgSvcExportUpToDate.targetGroupTags.Tags[model.K8SSourceTypeKey] = aws.String(string(model.SourceTypeSvcExport)) noDeleteTgs = append(noDeleteTgs, tgSvcExportUpToDate) tgSvcUpToDate := copy(baseTg) tgSvcUpToDate.getTargetGroupOutput.Arn = aws.String("tg-svc-arn") - tgSvcUpToDate.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeHTTPRoute)) + tgSvcUpToDate.targetGroupTags.Tags[model.K8SSourceTypeKey] = aws.String(string(model.SourceTypeHTTPRoute)) tgSvcUpToDate.targetGroupTags.Tags[model.K8SRouteNameKey] = aws.String("route") tgSvcUpToDate.targetGroupTags.Tags[model.K8SRouteNamespaceKey] = aws.String("route-ns") noDeleteTgs = append(noDeleteTgs, tgSvcUpToDate) @@ -241,21 +241,21 @@ func Test_DoNotDeleteCases(t *testing.T) { ProtocolVersion: "HTTP1", IpAddressType: "IPV4", TargetGroupTagFields: model.TargetGroupTagFields{ - EKSClusterName: "cluster-name", + ClusterName: "cluster-name", K8SServiceName: "svc", K8SServiceNamespace: "ns", }, }, } svcExportModelTg := baseModelTg - svcExportModelTg.Spec.TargetGroupTagFields.K8SParentRefType = model.ParentRefTypeSvcExport + svcExportModelTg.Spec.TargetGroupTagFields.K8SSourceType = model.SourceTypeSvcExport mockSvcExportTgBuilder.EXPECT().BuildTargetGroup(ctx, gomock.Any()).Return(&svcExportModelTg, nil) stack := core.NewDefaultStack(core.StackID{Name: "foo", Namespace: "bar"}) svcModelTg := baseModelTg svcModelTg.ResourceMeta = core.NewResourceMeta(stack, "AWS:VPCServiceNetwork::TargetGroup", "tg-id") - svcModelTg.Spec.TargetGroupTagFields.K8SParentRefType = model.ParentRefTypeHTTPRoute + svcModelTg.Spec.TargetGroupTagFields.K8SSourceType = model.SourceTypeHTTPRoute svcModelTg.Spec.TargetGroupTagFields.K8SRouteName = "route" svcModelTg.Spec.TargetGroupTagFields.K8SRouteNamespace = "route-ns" stack.AddResource(&svcModelTg) @@ -287,7 +287,7 @@ func Test_DeleteServiceExport_DeleteCases(t *testing.T) { var deleteTgs []tgListOutput tgSvcExport := copy(baseTg) tgSvcExport.getTargetGroupOutput.Arn = aws.String("tg-svc-export-arn") - tgSvcExport.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeSvcExport)) + tgSvcExport.targetGroupTags.Tags[model.K8SSourceTypeKey] = aws.String(string(model.SourceTypeSvcExport)) deleteTgs = append(deleteTgs, tgSvcExport) t.Run("Service Export does not exist", func(t *testing.T) { @@ -342,10 +342,10 @@ func Test_DeleteServiceExport_DeleteCases(t *testing.T) { ProtocolVersion: "HTTP1", IpAddressType: "IPV4", TargetGroupTagFields: model.TargetGroupTagFields{ - EKSClusterName: "cluster-name", + ClusterName: "cluster-name", K8SServiceName: "svc", K8SServiceNamespace: "ns", - K8SParentRefType: model.ParentRefTypeSvcExport, + K8SSourceType: model.SourceTypeSvcExport, }, }, } @@ -387,7 +387,7 @@ func Test_DeleteRoute_DeleteCases(t *testing.T) { var deleteTgs []tgListOutput tgSvc := copy(baseTg) tgSvc.getTargetGroupOutput.Arn = aws.String("tg-svc-arn") - tgSvc.targetGroupTags.Tags[model.K8SParentRefTypeKey] = aws.String(string(model.ParentRefTypeHTTPRoute)) + tgSvc.targetGroupTags.Tags[model.K8SSourceTypeKey] = aws.String(string(model.SourceTypeHTTPRoute)) tgSvc.targetGroupTags.Tags[model.K8SRouteNameKey] = aws.String("route") tgSvc.targetGroupTags.Tags[model.K8SRouteNamespaceKey] = aws.String("route-ns") deleteTgs = append(deleteTgs, tgSvc) @@ -446,12 +446,12 @@ func Test_DeleteRoute_DeleteCases(t *testing.T) { ProtocolVersion: "HTTP1", IpAddressType: "IPV4", TargetGroupTagFields: model.TargetGroupTagFields{ - EKSClusterName: "cluster-name", + ClusterName: "cluster-name", K8SServiceName: "svc", K8SServiceNamespace: "ns", K8SRouteName: "route-name", K8SRouteNamespace: "route-ns", - K8SParentRefType: model.ParentRefTypeSvcExport, + K8SSourceType: model.SourceTypeSvcExport, }, }, } diff --git a/pkg/deploy/lattice/targets_manager.go b/pkg/deploy/lattice/targets_manager.go index 6a0159df..3d998c80 100644 --- a/pkg/deploy/lattice/targets_manager.go +++ b/pkg/deploy/lattice/targets_manager.go @@ -51,8 +51,9 @@ func (s *defaultTargetsManager) Update(ctx context.Context, modelTargets *model. return err } - s.deregisterStaleTargets(ctx, modelTargets, modelTg, listTargetsOutput) - return s.registerTargets(ctx, modelTargets, modelTg) + err1 := s.deregisterStaleTargets(ctx, modelTargets, modelTg, listTargetsOutput) + err2 := s.registerTargets(ctx, modelTargets, modelTg) + return errors.Join(err1, err2) } func (s *defaultTargetsManager) registerTargets( @@ -87,9 +88,8 @@ func (s *defaultTargetsManager) registerTargets( } if len(resp.Unsuccessful) > 0 { - s.log.Infof("Failed RegisterTargets (Unsuccessful=%d) %s, will retry", + return fmt.Errorf("Failed RegisterTargets (Unsuccessful=%d) %s, will retry", len(resp.Unsuccessful), modelTg.Status.Id) - return errors.New(LATTICE_RETRY) } s.log.Infof("Success RegisterTargets %d, %s", len(resp.Successful), modelTg.Status.Id) @@ -101,7 +101,7 @@ func (s *defaultTargetsManager) deregisterStaleTargets( modelTargets *model.Targets, modelTg *model.TargetGroup, listTargetsOutput []*vpclattice.TargetSummary, -) { +) error { var targetsToDeregister []*vpclattice.Target for _, latticeTarget := range listTargetsOutput { isStale := true @@ -124,9 +124,10 @@ func (s *defaultTargetsManager) deregisterStaleTargets( } _, err := s.cloud.Lattice().DeregisterTargetsWithContext(ctx, &deregisterTargetsInput) if err != nil { - s.log.Infof("Failed DeregisterTargets %s due to %s", modelTg.Status.Id, err) - } else { - s.log.Infof("Success DeregisterTargets %s", modelTg.Status.Id) + return fmt.Errorf("failed DeregisterTargets %s due to %s", modelTg.Status.Id, err) } + s.log.Infof("Success DeregisterTargets %s", modelTg.Status.Id) } + + return nil } diff --git a/pkg/deploy/lattice/targets_manager_test.go b/pkg/deploy/lattice/targets_manager_test.go index b5ddfec6..12c680d7 100644 --- a/pkg/deploy/lattice/targets_manager_test.go +++ b/pkg/deploy/lattice/targets_manager_test.go @@ -116,7 +116,7 @@ func TestTargetsManager(t *testing.T) { err = targetsManager.Update(ctx, &modelTargets, &modelTg) assert.NotNil(t, err) - // error on DeregisterTargetsWithContext - currently this DOES NOT cause overall failure + // error on DeregisterTargetsWithContext existingTarget := &vpclattice.TargetSummary{ Id: aws.String("192.0.2.250"), Port: aws.Int64(80), @@ -128,7 +128,7 @@ func TestTargetsManager(t *testing.T) { targetsManager = NewTargetsManager(gwlog.FallbackLogger, mockCloud) err = targetsManager.Update(ctx, &modelTargets, &modelTg) - assert.Nil(t, err) + assert.NotNil(t, err) }) t.Run("basic validation", func(t *testing.T) { diff --git a/pkg/deploy/lattice/targets_synthesizer.go b/pkg/deploy/lattice/targets_synthesizer.go index 9af03664..e51ea680 100644 --- a/pkg/deploy/lattice/targets_synthesizer.go +++ b/pkg/deploy/lattice/targets_synthesizer.go @@ -2,7 +2,6 @@ package lattice import ( "context" - "errors" "fmt" pkg_aws "github.com/aws/aws-application-networking-k8s/pkg/aws" @@ -40,16 +39,12 @@ func (t *targetsSynthesizer) Synthesize(ctx context.Context) error { } for _, targets := range resTargets { - resTg, err := t.stack.GetResource(targets.Spec.StackTargetGroupId, &model.TargetGroup{}) + tg := &model.TargetGroup{} + err := t.stack.GetResource(targets.Spec.StackTargetGroupId, tg) if err != nil { return err } - tg, ok := resTg.(*model.TargetGroup) - if !ok { - return errors.New("unexpected type conversion failure for target group stack object") - } - err = t.targetsManager.Update(ctx, targets, tg) if err != nil { identifier := model.TgNamePrefix(tg.Spec) diff --git a/pkg/deploy/stack_deployer.go b/pkg/deploy/stack_deployer.go index ff5a780e..be07b0d4 100644 --- a/pkg/deploy/stack_deployer.go +++ b/pkg/deploy/stack_deployer.go @@ -2,6 +2,7 @@ package deploy import ( "context" + "fmt" "github.com/aws/aws-application-networking-k8s/pkg/gateway" "github.com/aws/aws-application-networking-k8s/pkg/utils/gwlog" @@ -80,6 +81,8 @@ func NewLatticeServiceStackDeploy( cloud pkg_aws.Cloud, k8sClient client.Client, ) *latticeServiceStackDeployer { + brTgBuilder := gateway.NewBackendRefTargetGroupBuilder(log, k8sClient) + return &latticeServiceStackDeployer{ log: log, cloud: cloud, @@ -91,7 +94,7 @@ func NewLatticeServiceStackDeploy( ruleManager: lattice.NewRuleManager(log, cloud), dnsEndpointManager: externaldns.NewDnsEndpointManager(log, k8sClient), svcExportTgBuilder: gateway.NewSvcExportTargetGroupBuilder(log, k8sClient), - svcBuilder: gateway.NewLatticeServiceBuilder(log, k8sClient), + svcBuilder: gateway.NewLatticeServiceBuilder(log, k8sClient, brTgBuilder), } } @@ -104,45 +107,38 @@ func (d *latticeServiceStackDeployer) Deploy(ctx context.Context, stack core.Sta //Handle targetGroups creation request if err := targetGroupSynthesizer.SynthesizeCreate(ctx); err != nil { - d.log.Infof("Error during tg synthesis %s", err) - return err + return fmt.Errorf("error during tg synthesis %s", err) } //Handle targets "reconciliation" request (register intend-to-be-registered targets and deregister intend-to-be-registered targets) if err := targetsSynthesizer.Synthesize(ctx); err != nil { - d.log.Infof("Error during target synthesis %s", err) - return err + return fmt.Errorf("error during target synthesis %s", err) } // Handle latticeService "reconciliation" request if err := serviceSynthesizer.Synthesize(ctx); err != nil { - d.log.Infof("Error during service synthesis %s", err) - return err + return fmt.Errorf("error during service synthesis %s", err) } //Handle latticeService listeners "reconciliation" request if err := listenerSynthesizer.Synthesize(ctx); err != nil { - d.log.Infof("Error during listener synthesis %s", err) - return err + return fmt.Errorf("error during listener synthesis %s", err) } //Handle latticeService listener's rules "reconciliation" request if err := ruleSynthesizer.Synthesize(ctx); err != nil { - d.log.Infof("Error during rule synthesis %s", err) - return err + return fmt.Errorf("error during rule synthesis %s", err) } //Handle targetGroup deletion request if err := targetGroupSynthesizer.SynthesizeDelete(ctx); err != nil { - d.log.Infof("Error during tg delete synthesis %s", err) - return err + return fmt.Errorf("error during tg delete synthesis %s", err) } // Do garbage collection for not-in-use targetGroups - //TODO: run SynthesizeSDKTargetGroups(ctx) as a global garbage collector scheduled backgroud task (i.e., run it as a goroutine in main.go) + //TODO: run SynthesizeSDKTargetGroups(ctx) as a global garbage collector scheduled background task (i.e., run it as a goroutine in main.go) if err := targetGroupSynthesizer.SynthesizeUnusedDelete(ctx); err != nil { - d.log.Infof("Error during tg unused delete synthesis %s", err) - return err + return fmt.Errorf("error during tg unused delete synthesis %s", err) } return nil @@ -163,13 +159,15 @@ func NewTargetGroupStackDeploy( cloud pkg_aws.Cloud, k8sClient client.Client, ) *latticeTargetGroupStackDeployer { + brTgBuilder := gateway.NewBackendRefTargetGroupBuilder(log, k8sClient) + return &latticeTargetGroupStackDeployer{ log: log, cloud: cloud, k8sclient: k8sClient, targetGroupManager: lattice.NewTargetGroupManager(log, cloud), svcExportTgBuilder: gateway.NewSvcExportTargetGroupBuilder(log, k8sClient), - svcBuilder: gateway.NewLatticeServiceBuilder(log, k8sClient), + svcBuilder: gateway.NewLatticeServiceBuilder(log, k8sClient, brTgBuilder), } } diff --git a/pkg/gateway/model_build_lattice_service.go b/pkg/gateway/model_build_lattice_service.go index d1b2b359..769344f4 100644 --- a/pkg/gateway/model_build_lattice_service.go +++ b/pkg/gateway/model_build_lattice_service.go @@ -29,9 +29,8 @@ type LatticeServiceModelBuilder struct { func NewLatticeServiceBuilder( log gwlog.Logger, client client.Client, + brTgBuilder BackendRefTargetGroupModelBuilder, ) *LatticeServiceModelBuilder { - brTgBuilder := NewBackendRefTargetGroupBuilder(log, client) - return &LatticeServiceModelBuilder{ log: log, client: client, @@ -45,11 +44,6 @@ func (b *LatticeServiceModelBuilder) Build( ) (core.Stack, error) { stack := core.NewDefaultStack(core.StackID(k8s.NamespacedName(route.K8sObject()))) - if b.brTgBuilder == nil { - b.log.Debugf("brTgBuilder is nil, initializing") - b.brTgBuilder = NewBackendRefTargetGroupBuilder(b.log, b.client) - } - task := &latticeServiceModelBuildTask{ log: b.log, route: route, diff --git a/pkg/gateway/model_build_targetgroup.go b/pkg/gateway/model_build_targetgroup.go index bd0f6b2f..782d9c81 100644 --- a/pkg/gateway/model_build_targetgroup.go +++ b/pkg/gateway/model_build_targetgroup.go @@ -163,8 +163,8 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargetGroup(ctx context.Contex HealthCheckConfig: healthCheckConfig, } spec.VpcId = config.VpcID - spec.K8SParentRefType = model.ParentRefTypeSvcExport - spec.EKSClusterName = config.ClusterName + spec.K8SSourceType = model.SourceTypeSvcExport + spec.ClusterName = config.ClusterName spec.K8SServiceName = t.serviceExport.Name spec.K8SServiceNamespace = t.serviceExport.Namespace @@ -260,10 +260,11 @@ func (t *backendRefTargetGroupModelBuildTask) buildTargets(ctx context.Context, backendRefNsName := getBackendRefNsName(t.route, t.backendRef) svc := &corev1.Service{} if err := t.client.Get(ctx, backendRefNsName, svc); err != nil { - t.log.Infof("Error finding backend service %s due to %s", backendRefNsName, err) - if t.route.DeletionTimestamp().IsZero() { - // Ignore error for deletion request - return err + if apierrors.IsNotFound(err) && !t.route.DeletionTimestamp().IsZero() { + t.log.Infof("Ignoring not found error for service %s on deleted route %s", + t.backendRef.Name(), t.route.Name()) + } else { + return fmt.Errorf("error finding backend service %s due to %s", backendRefNsName, err) } } @@ -286,13 +287,14 @@ func (t *backendRefTargetGroupModelBuildTask) buildTargetGroupSpec(ctx context.C routeIsDeleted := !t.route.DeletionTimestamp().IsZero() backendRefNsName := getBackendRefNsName(t.route, t.backendRef) - svc := &corev1.Service{} if err := t.client.Get(ctx, backendRefNsName, svc); err != nil { - t.log.Infof("Error finding backend service %s due to %s", backendRefNsName, err) - if !routeIsDeleted { - // Ignore error for deletion request - return model.TargetGroupSpec{}, err + if routeIsDeleted && apierrors.IsNotFound(err) { + t.log.Infof("Ignoring not found error for service %s on deleted route %s", + t.backendRef.Name(), t.route.Name()) + } else if !routeIsDeleted { + return model.TargetGroupSpec{}, + fmt.Errorf("error finding backend service %s due to %s", backendRefNsName, err) } } @@ -332,10 +334,10 @@ func (t *backendRefTargetGroupModelBuildTask) buildTargetGroupSpec(ctx context.C } // GRPC takes precedence over other protocolVersions. - parentRefType := model.ParentRefTypeHTTPRoute + parentRefType := model.SourceTypeHTTPRoute if _, ok := t.route.(*core.GRPCRoute); ok { protocolVersion = vpclattice.TargetGroupProtocolVersionGrpc - parentRefType = model.ParentRefTypeGRPCRoute + parentRefType = model.SourceTypeGRPCRoute } spec := model.TargetGroupSpec{ @@ -347,8 +349,8 @@ func (t *backendRefTargetGroupModelBuildTask) buildTargetGroupSpec(ctx context.C HealthCheckConfig: healthCheckConfig, } spec.VpcId = vpc - spec.K8SParentRefType = parentRefType - spec.EKSClusterName = eksCluster + spec.K8SSourceType = parentRefType + spec.ClusterName = eksCluster spec.K8SServiceName = backendRefNsName.Name spec.K8SServiceNamespace = backendRefNsName.Namespace spec.K8SRouteName = t.route.Name() diff --git a/pkg/gateway/model_build_targetgroup_test.go b/pkg/gateway/model_build_targetgroup_test.go index fd489ddb..02aac5e3 100644 --- a/pkg/gateway/model_build_targetgroup_test.go +++ b/pkg/gateway/model_build_targetgroup_test.go @@ -258,9 +258,9 @@ func Test_TGModelByServicExportBuild(t *testing.T) { assert.Equal(t, vpclattice.IpAddressTypeIpv4, stackTg.Spec.IpAddressType) } - assert.Equal(t, config.ClusterName, stackTg.Spec.EKSClusterName) + assert.Equal(t, config.ClusterName, stackTg.Spec.ClusterName) assert.Equal(t, config.VpcID, stackTg.Spec.VpcId) - assert.Equal(t, model.ParentRefTypeSvcExport, stackTg.Spec.K8SParentRefType) + assert.Equal(t, model.SourceTypeSvcExport, stackTg.Spec.K8SSourceType) assert.Equal(t, tt.svc.Name, stackTg.Spec.K8SServiceName) assert.Equal(t, tt.svc.Namespace, stackTg.Spec.K8SServiceNamespace) assert.Equal(t, "", stackTg.Spec.K8SRouteName) @@ -578,9 +578,9 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { assert.Equal(t, vpclattice.IpAddressTypeIpv4, stackTg.Spec.IpAddressType) } - assert.Equal(t, config.ClusterName, stackTg.Spec.EKSClusterName) + assert.Equal(t, config.ClusterName, stackTg.Spec.ClusterName) assert.Equal(t, config.VpcID, stackTg.Spec.VpcId) - assert.Equal(t, model.ParentRefTypeHTTPRoute, stackTg.Spec.K8SParentRefType) + assert.Equal(t, model.SourceTypeHTTPRoute, stackTg.Spec.K8SSourceType) assert.Equal(t, spec.K8SServiceName, stackTg.Spec.K8SServiceName) assert.Equal(t, spec.K8SServiceNamespace, stackTg.Spec.K8SServiceNamespace) assert.Equal(t, tt.route.Name(), stackTg.Spec.K8SRouteName) diff --git a/pkg/model/core/stack.go b/pkg/model/core/stack.go index 4fe2648a..7af7c0e1 100644 --- a/pkg/model/core/stack.go +++ b/pkg/model/core/stack.go @@ -18,8 +18,8 @@ type Stack interface { // Add a resource into stack. AddResource(res Resource) error - // Get a resource by its id and type - GetResource(id string, resType Resource) (Resource, error) + // Get a resource by its id and type, pointer will be populated after call + GetResource(id string, res Resource) error // Add a dependency relationship between resources. AddDependency(dependee Resource, depender Resource) error @@ -66,17 +66,24 @@ func (s *defaultStack) AddResource(res Resource) error { } // Get a resource from the pointer, then return the result -func (s *defaultStack) GetResource(id string, resType Resource) (Resource, error) { +// will ensure the resource is of the specified type +func (s *defaultStack) GetResource(id string, res Resource) error { + t := reflect.TypeOf(res) resUID := graph.ResourceUID{ - ResType: reflect.TypeOf(resType), + ResType: t, ResID: id, } - if r, ok := s.resources[resUID]; ok { - return r, nil + r, ok := s.resources[resUID] + if !ok { + return fmt.Errorf("resource %s not found", id) } - return nil, fmt.Errorf("resource %s not found", id) + // since the type makes up the key used to retrieve, + // it will be safe to assign the value to the param + v := reflect.ValueOf(res) + v.Elem().Set(reflect.ValueOf(r).Elem()) + return nil } // Add a dependency relationship between resources. diff --git a/pkg/model/core/stack_mock.go b/pkg/model/core/stack_mock.go index 349d20e9..b7b6ed07 100644 --- a/pkg/model/core/stack_mock.go +++ b/pkg/model/core/stack_mock.go @@ -62,18 +62,17 @@ func (mr *MockStackMockRecorder) AddResource(res interface{}) *gomock.Call { } // GetResource mocks base method. -func (m *MockStack) GetResource(id string, resType Resource) (Resource, error) { +func (m *MockStack) GetResource(id string, res Resource) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetResource", id, resType) - ret0, _ := ret[0].(Resource) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret := m.ctrl.Call(m, "GetResource", id, res) + ret0, _ := ret[0].(error) + return ret0 } // GetResource indicates an expected call of GetResource. -func (mr *MockStackMockRecorder) GetResource(id, resType interface{}) *gomock.Call { +func (mr *MockStackMockRecorder) GetResource(id, res interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockStack)(nil).GetResource), id, resType) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetResource", reflect.TypeOf((*MockStack)(nil).GetResource), id, res) } // ListResources mocks base method. diff --git a/pkg/model/core/stack_test.go b/pkg/model/core/stack_test.go index d8dfbea1..f9edd1e3 100644 --- a/pkg/model/core/stack_test.go +++ b/pkg/model/core/stack_test.go @@ -118,7 +118,29 @@ func Test_Get(t *testing.T) { Status: nil, } stack.AddResource(&fr) - gr, err := stack.GetResource(fr.ID(), &FakeResource{}) + frPtr := &FakeResource{} + err := stack.GetResource(fr.ID(), frPtr) assert.NoError(t, err) - assert.Equal(t, &fr, gr) + assert.Equal(t, &fr, frPtr) +} + +type FakeResource2 struct { + FakeResource +} + +func Test_GetWithErr(t *testing.T) { + stack := NewDefaultStack(StackID{Namespace: "namespace", Name: "name"}) + fr := FakeResource{ + ResourceMeta: ResourceMeta{ + resType: "fake", + id: "id-B", + }, + Spec: FakeResourceSpec{}, + Status: nil, + } + stack.AddResource(&fr) + + // getting with the wrong type will error + err := stack.GetResource(fr.ID(), &FakeResource2{}) + assert.Error(t, err) } diff --git a/pkg/model/lattice/targetgroup.go b/pkg/model/lattice/targetgroup.go index ebe9bbd4..3d363d7d 100644 --- a/pkg/model/lattice/targetgroup.go +++ b/pkg/model/lattice/targetgroup.go @@ -11,13 +11,12 @@ import ( ) const ( - EKSClusterNameKey = "EKSClusterName" + EKSClusterNameKey = "ClusterName" K8SServiceNameKey = "K8SServiceName" K8SServiceNamespaceKey = "K8SServiceNamespace" K8SRouteNameKey = "K8SRouteName" K8SRouteNamespaceKey = "K8SRouteNamespace" - - K8SParentRefTypeKey = "K8SParentRefTypeKey" + K8SSourceTypeKey = "K8SSourceTypeKey" MaxNamespaceLength = 55 MaxNameLength = 55 @@ -42,8 +41,8 @@ type TargetGroupSpec struct { TargetGroupTagFields } type TargetGroupTagFields struct { - EKSClusterName string `json:"eksclustername"` - K8SParentRefType ParentRefType `json:"k8sparentreftype"` + ClusterName string `json:"clustername"` + K8SSourceType K8SSourceType `json:"k8ssourcetype"` K8SServiceName string `json:"k8sservicename"` K8SServiceNamespace string `json:"k8sservicenamespace"` K8SRouteName string `json:"k8sroutename"` @@ -57,22 +56,22 @@ type TargetGroupStatus struct { } type TargetGroupType string -type ParentRefType string +type K8SSourceType string type RouteType string const ( TargetGroupTypeIP TargetGroupType = "IP" - ParentRefTypeSvcExport ParentRefType = "ServiceExport" - ParentRefTypeHTTPRoute ParentRefType = "HTTPRoute" - ParentRefTypeGRPCRoute ParentRefType = "GRPCRoute" - ParentRefTypeInvalid ParentRefType = "INVALID" + SourceTypeSvcExport K8SSourceType = "ServiceExport" + SourceTypeHTTPRoute K8SSourceType = "HTTPRoute" + SourceTypeGRPCRoute K8SSourceType = "GRPCRoute" + SourceTypeInvalid K8SSourceType = "INVALID" ) func TGTagFieldsFromTags(tags map[string]*string) TargetGroupTagFields { return TargetGroupTagFields{ - EKSClusterName: getMapValue(tags, EKSClusterNameKey), - K8SParentRefType: GetParentRefType(getMapValue(tags, K8SParentRefTypeKey)), + ClusterName: getMapValue(tags, EKSClusterNameKey), + K8SSourceType: GetParentRefType(getMapValue(tags, K8SSourceTypeKey)), K8SServiceName: getMapValue(tags, K8SServiceNameKey), K8SServiceNamespace: getMapValue(tags, K8SServiceNamespaceKey), K8SRouteName: getMapValue(tags, K8SRouteNameKey), @@ -88,27 +87,27 @@ func getMapValue(m map[string]*string, key string) string { return *v } -func GetParentRefType(s string) ParentRefType { +func GetParentRefType(s string) K8SSourceType { if s == "" { return "" // empty is OK } switch s { - case string(ParentRefTypeHTTPRoute): - return ParentRefTypeHTTPRoute - case string(ParentRefTypeGRPCRoute): - return ParentRefTypeGRPCRoute - case string(ParentRefTypeSvcExport): - return ParentRefTypeSvcExport + case string(SourceTypeHTTPRoute): + return SourceTypeHTTPRoute + case string(SourceTypeGRPCRoute): + return SourceTypeGRPCRoute + case string(SourceTypeSvcExport): + return SourceTypeSvcExport default: - return ParentRefTypeInvalid + return SourceTypeInvalid } } func TagFieldsMatch(spec TargetGroupSpec, tags TargetGroupTagFields) bool { specTags := TargetGroupTagFields{ - EKSClusterName: spec.EKSClusterName, - K8SParentRefType: spec.K8SParentRefType, + ClusterName: spec.ClusterName, + K8SSourceType: spec.K8SSourceType, K8SServiceName: spec.K8SServiceName, K8SServiceNamespace: spec.K8SServiceNamespace, K8SRouteName: spec.K8SRouteName, @@ -138,19 +137,19 @@ func NewTargetGroup(stack core.Stack, spec TargetGroupSpec) (*TargetGroup, error return tg, nil } -func (t *TargetGroupTagFields) IsServiceExport() bool { - return t.K8SParentRefType == ParentRefTypeSvcExport +func (t *TargetGroupTagFields) IsSourceTypeServiceExport() bool { + return t.K8SSourceType == SourceTypeSvcExport } -func (t *TargetGroupTagFields) IsRoute() bool { - return t.K8SParentRefType == ParentRefTypeHTTPRoute || - t.K8SParentRefType == ParentRefTypeGRPCRoute +func (t *TargetGroupTagFields) IsSourceTypeRoute() bool { + return t.K8SSourceType == SourceTypeHTTPRoute || + t.K8SSourceType == SourceTypeGRPCRoute } func (t *TargetGroupSpec) Validate() error { requiredFields := []string{t.K8SServiceName, t.K8SServiceNamespace, - t.Protocol, t.ProtocolVersion, t.VpcId, t.EKSClusterName, t.IpAddressType, - string(t.K8SParentRefType)} + t.Protocol, t.ProtocolVersion, t.VpcId, t.ClusterName, t.IpAddressType, + string(t.K8SSourceType)} for _, s := range requiredFields { if s == "" { @@ -158,7 +157,7 @@ func (t *TargetGroupSpec) Validate() error { } } - if t.IsRoute() { + if t.IsSourceTypeRoute() { if t.K8SRouteName == "" || t.K8SRouteNamespace == "" { return errors.New("route name or namespace missing for route-based target group") } diff --git a/test/pkg/test/framework.go b/test/pkg/test/framework.go index 379c911d..3577357a 100644 --- a/test/pkg/test/framework.go +++ b/test/pkg/test/framework.go @@ -199,8 +199,8 @@ func (env *Framework) ExpectToBeClean(ctx context.Context) { }) if err == nil { env.Log.Infof("Found Tags for tg %v tags: %v", *tg.Name, retrievedTags) - tagValue, ok := retrievedTags.Tags[model.K8SParentRefTypeKey] - if ok && *tagValue == string(model.ParentRefTypeSvcExport) { + tagValue, ok := retrievedTags.Tags[model.K8SSourceTypeKey] + if ok && *tagValue == string(model.SourceTypeSvcExport) { env.Log.Infof("TargetGroup: %s was created by k8s controller, by a ServiceExport", *tg.Id) //This tg is created by k8s controller, by a ServiceExport, //ServiceExport still have a known targetGroup leaking issue, From 0677988f153aaf6292aa483e725c147c476c1022 Mon Sep 17 00:00:00 2001 From: Erik F <16261515+erikfuller@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:46:44 -0700 Subject: [PATCH 3/6] adding standard tag prefix --- pkg/deploy/lattice/rule_synthesizer.go | 4 ++-- pkg/deploy/lattice/rule_synthesizer_test.go | 4 ++-- pkg/deploy/lattice/target_group_manager.go | 2 +- .../lattice/target_group_manager_test.go | 18 ++++++++-------- .../lattice/target_group_synthesizer.go | 2 +- .../lattice/target_group_synthesizer_test.go | 12 +++++------ pkg/gateway/model_build_rule.go | 2 +- pkg/gateway/model_build_targetgroup.go | 4 ++-- pkg/gateway/model_build_targetgroup_test.go | 4 ++-- pkg/model/lattice/rule.go | 2 +- pkg/model/lattice/targetgroup.go | 21 ++++++++++--------- 11 files changed, 38 insertions(+), 37 deletions(-) diff --git a/pkg/deploy/lattice/rule_synthesizer.go b/pkg/deploy/lattice/rule_synthesizer.go index fef00044..da628055 100644 --- a/pkg/deploy/lattice/rule_synthesizer.go +++ b/pkg/deploy/lattice/rule_synthesizer.go @@ -66,7 +66,7 @@ func (r *ruleSynthesizer) resolveRuleTgIds(ctx context.Context, modelRule *model if rtg.SvcImportTG != nil { r.log.Debugf("Getting target group for service import %s %s (%s, %s)", rtg.SvcImportTG.K8SServiceName, rtg.SvcImportTG.K8SServiceNamespace, - rtg.SvcImportTG.EKSClusterName, rtg.SvcImportTG.VpcId) + rtg.SvcImportTG.K8SClusterName, rtg.SvcImportTG.VpcId) tgId, err := r.findSvcExportTG(ctx, *rtg.SvcImportTG) if err != nil { @@ -95,7 +95,7 @@ func (r *ruleSynthesizer) findSvcExportTG(ctx context.Context, svcImportTg model svcMatch := tgTags.IsSourceTypeServiceExport() && (tgTags.K8SServiceName == svcImportTg.K8SServiceName) && (tgTags.K8SServiceNamespace == svcImportTg.K8SServiceNamespace) - clusterMatch := (svcImportTg.EKSClusterName == "") || (tgTags.ClusterName == svcImportTg.EKSClusterName) + clusterMatch := (svcImportTg.K8SClusterName == "") || (tgTags.K8SClusterName == svcImportTg.K8SClusterName) vpcMatch := (svcImportTg.VpcId == "") || (svcImportTg.VpcId == aws.StringValue(tg.getTargetGroupOutput.Config.VpcIdentifier)) diff --git a/pkg/deploy/lattice/rule_synthesizer_test.go b/pkg/deploy/lattice/rule_synthesizer_test.go index c2b91288..0fe1a82b 100644 --- a/pkg/deploy/lattice/rule_synthesizer_test.go +++ b/pkg/deploy/lattice/rule_synthesizer_test.go @@ -157,7 +157,7 @@ func Test_resolveRuleTgs(t *testing.T) { TargetGroups: []*model.RuleTargetGroup{ { SvcImportTG: &model.SvcImportTargetGroup{ - EKSClusterName: "cluster-name", + K8SClusterName: "cluster-name", K8SServiceName: "svc-name", K8SServiceNamespace: "ns", VpcId: "vpc-id", @@ -186,7 +186,7 @@ func Test_resolveRuleTgs(t *testing.T) { targetGroupTags: &vpclattice.ListTagsForResourceOutput{Tags: map[string]*string{ model.K8SServiceNameKey: aws.String("svc-name"), model.K8SServiceNamespaceKey: aws.String("ns"), - model.EKSClusterNameKey: aws.String("cluster-name"), + model.K8SClusterNameKey: aws.String("cluster-name"), model.K8SSourceTypeKey: aws.String(string(model.SourceTypeSvcExport)), }}, }, diff --git a/pkg/deploy/lattice/target_group_manager.go b/pkg/deploy/lattice/target_group_manager.go index b7b7ee02..160aa583 100644 --- a/pkg/deploy/lattice/target_group_manager.go +++ b/pkg/deploy/lattice/target_group_manager.go @@ -76,7 +76,7 @@ func (s *defaultTargetGroupManager) create(ctx context.Context, modelTg *model.T Type: &latticeTgType, Tags: s.cloud.DefaultTags(), } - createInput.Tags[model.EKSClusterNameKey] = &modelTg.Spec.ClusterName + createInput.Tags[model.K8SClusterNameKey] = &modelTg.Spec.K8SClusterName createInput.Tags[model.K8SServiceNameKey] = &modelTg.Spec.K8SServiceName createInput.Tags[model.K8SServiceNamespaceKey] = &modelTg.Spec.K8SServiceNamespace createInput.Tags[model.K8SSourceTypeKey] = aws.String(string(modelTg.Spec.K8SSourceType)) diff --git a/pkg/deploy/lattice/target_group_manager_test.go b/pkg/deploy/lattice/target_group_manager_test.go index 7a0c190d..03ef8e8c 100644 --- a/pkg/deploy/lattice/target_group_manager_test.go +++ b/pkg/deploy/lattice/target_group_manager_test.go @@ -42,7 +42,7 @@ func Test_CreateTargetGroup_TGNotExist_Active(t *testing.T) { ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, } tgSpec.VpcId = config.VpcID - tgSpec.ClusterName = config.ClusterName + tgSpec.K8SClusterName = config.ClusterName tgSpec.K8SSourceType = model.SourceTypeSvcExport tgSpec.K8SServiceName = "exportsvc1" tgSpec.K8SServiceNamespace = "default" @@ -55,7 +55,7 @@ func Test_CreateTargetGroup_TGNotExist_Active(t *testing.T) { ProtocolVersion: vpclattice.TargetGroupProtocolVersionHttp1, } tgSpec.VpcId = config.VpcID - tgSpec.ClusterName = config.ClusterName + tgSpec.K8SClusterName = config.ClusterName tgSpec.K8SSourceType = model.SourceTypeHTTPRoute tgSpec.K8SServiceName = "backend-svc1" tgSpec.K8SServiceNamespace = "default" @@ -71,7 +71,7 @@ func Test_CreateTargetGroup_TGNotExist_Active(t *testing.T) { expectedTags := cloud.DefaultTags() expectedTags[model.K8SServiceNameKey] = &tgSpec.K8SServiceName expectedTags[model.K8SServiceNamespaceKey] = &tgSpec.K8SServiceNamespace - expectedTags[model.EKSClusterNameKey] = &tgSpec.ClusterName + expectedTags[model.K8SClusterNameKey] = &tgSpec.K8SClusterName if tgType == "by-serviceexport" { value := string(model.SourceTypeSvcExport) @@ -947,7 +947,7 @@ func Test_IsTargetGroupMatch(t *testing.T) { }, }, latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, - tags: &model.TargetGroupTagFields{ClusterName: "foo"}, + tags: &model.TargetGroupTagFields{K8SClusterName: "foo"}, }, { name: "fetch tags not equal", @@ -961,7 +961,7 @@ func Test_IsTargetGroupMatch(t *testing.T) { latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, listTagsOut: &vpclattice.ListTagsForResourceOutput{ Tags: map[string]*string{ - model.EKSClusterNameKey: aws.String("foo"), + model.K8SClusterNameKey: aws.String("foo"), }, }, }, @@ -992,13 +992,13 @@ func Test_IsTargetGroupMatch(t *testing.T) { Port: 443, ProtocolVersion: "HTTP1", TargetGroupTagFields: model.TargetGroupTagFields{ - ClusterName: "cluster", + K8SClusterName: "cluster", }, }, }, latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, tags: &model.TargetGroupTagFields{ - ClusterName: "cluster", + K8SClusterName: "cluster", }, getTgOut: &vpclattice.GetTargetGroupOutput{ Config: &vpclattice.TargetGroupConfig{ @@ -1015,14 +1015,14 @@ func Test_IsTargetGroupMatch(t *testing.T) { Port: 443, ProtocolVersion: "HTTP1", TargetGroupTagFields: model.TargetGroupTagFields{ - ClusterName: "cluster", + K8SClusterName: "cluster", }, }, }, latticeTg: &vpclattice.TargetGroupSummary{Port: aws.Int64(443)}, listTagsOut: &vpclattice.ListTagsForResourceOutput{ Tags: map[string]*string{ - model.EKSClusterNameKey: aws.String("cluster"), + model.K8SClusterNameKey: aws.String("cluster"), }, }, getTgOut: &vpclattice.GetTargetGroupOutput{ diff --git a/pkg/deploy/lattice/target_group_synthesizer.go b/pkg/deploy/lattice/target_group_synthesizer.go index 6de41a84..6df23b89 100644 --- a/pkg/deploy/lattice/target_group_synthesizer.go +++ b/pkg/deploy/lattice/target_group_synthesizer.go @@ -373,7 +373,7 @@ func (t *TargetGroupSynthesizer) vpcMatchesConfig(latticeTg tgListOutput) bool { } func (t *TargetGroupSynthesizer) hasExpectedTags(latticeTg tgListOutput, tagFields model.TargetGroupTagFields) bool { - if tagFields.ClusterName != config.ClusterName { + if tagFields.K8SClusterName != config.ClusterName { t.log.Debugf("Ignoring target group %s (%s) because it is not configured for this Cluster", *latticeTg.getTargetGroupOutput.Arn, *latticeTg.getTargetGroupOutput.Name) return false diff --git a/pkg/deploy/lattice/target_group_synthesizer_test.go b/pkg/deploy/lattice/target_group_synthesizer_test.go index ec6dd1a1..1c0e9b8c 100644 --- a/pkg/deploy/lattice/target_group_synthesizer_test.go +++ b/pkg/deploy/lattice/target_group_synthesizer_test.go @@ -128,11 +128,11 @@ func Test_SynthesizeUnusedDeleteIgnoreNotManagedByController(t *testing.T) { tgWrongCluster := copy(tgWrongVpc) tgWrongCluster.getTargetGroupOutput.Config.VpcIdentifier = aws.String("vpc-id") - tgWrongCluster.targetGroupTags.Tags[model.EKSClusterNameKey] = aws.String("another-cluster") + tgWrongCluster.targetGroupTags.Tags[model.K8SClusterNameKey] = aws.String("another-cluster") nonManagedTgs = append(nonManagedTgs, tgWrongCluster) tgInvalidParentRef := copy(tgWrongCluster) - tgInvalidParentRef.targetGroupTags.Tags[model.EKSClusterNameKey] = aws.String("cluster-name") + tgInvalidParentRef.targetGroupTags.Tags[model.K8SClusterNameKey] = aws.String("cluster-name") tgInvalidParentRef.targetGroupTags.Tags[model.K8SSourceTypeKey] = aws.String(string(model.SourceTypeInvalid)) nonManagedTgs = append(nonManagedTgs, tgInvalidParentRef) @@ -176,7 +176,7 @@ func getBaseTg() tgListOutput { }, targetGroupTags: &vpclattice.ListTagsForResourceOutput{Tags: make(map[string]*string)}, } - baseTg.targetGroupTags.Tags[model.EKSClusterNameKey] = aws.String("cluster-name") + baseTg.targetGroupTags.Tags[model.K8SClusterNameKey] = aws.String("cluster-name") baseTg.targetGroupTags.Tags[model.K8SServiceNameKey] = aws.String("svc") baseTg.targetGroupTags.Tags[model.K8SServiceNamespaceKey] = aws.String("ns") return baseTg @@ -241,7 +241,7 @@ func Test_DoNotDeleteCases(t *testing.T) { ProtocolVersion: "HTTP1", IpAddressType: "IPV4", TargetGroupTagFields: model.TargetGroupTagFields{ - ClusterName: "cluster-name", + K8SClusterName: "cluster-name", K8SServiceName: "svc", K8SServiceNamespace: "ns", }, @@ -342,7 +342,7 @@ func Test_DeleteServiceExport_DeleteCases(t *testing.T) { ProtocolVersion: "HTTP1", IpAddressType: "IPV4", TargetGroupTagFields: model.TargetGroupTagFields{ - ClusterName: "cluster-name", + K8SClusterName: "cluster-name", K8SServiceName: "svc", K8SServiceNamespace: "ns", K8SSourceType: model.SourceTypeSvcExport, @@ -446,7 +446,7 @@ func Test_DeleteRoute_DeleteCases(t *testing.T) { ProtocolVersion: "HTTP1", IpAddressType: "IPV4", TargetGroupTagFields: model.TargetGroupTagFields{ - ClusterName: "cluster-name", + K8SClusterName: "cluster-name", K8SServiceName: "svc", K8SServiceNamespace: "ns", K8SRouteName: "route-name", diff --git a/pkg/gateway/model_build_rule.go b/pkg/gateway/model_build_rule.go index c0f692b9..1200e3aa 100644 --- a/pkg/gateway/model_build_rule.go +++ b/pkg/gateway/model_build_rule.go @@ -243,7 +243,7 @@ func (t *latticeServiceModelBuildTask) getTargetGroupsForRuleAction(ctx context. eksCluster, ok := svcImport.Annotations["multicluster.x-k8s.io/aws-eks-cluster-name"] if ok { - svcImportTg.EKSClusterName = eksCluster + svcImportTg.K8SClusterName = eksCluster } } ruleTG.SvcImportTG = &svcImportTg diff --git a/pkg/gateway/model_build_targetgroup.go b/pkg/gateway/model_build_targetgroup.go index 782d9c81..dd5ee4d9 100644 --- a/pkg/gateway/model_build_targetgroup.go +++ b/pkg/gateway/model_build_targetgroup.go @@ -164,7 +164,7 @@ func (t *svcExportTargetGroupModelBuildTask) buildTargetGroup(ctx context.Contex } spec.VpcId = config.VpcID spec.K8SSourceType = model.SourceTypeSvcExport - spec.ClusterName = config.ClusterName + spec.K8SClusterName = config.ClusterName spec.K8SServiceName = t.serviceExport.Name spec.K8SServiceNamespace = t.serviceExport.Namespace @@ -350,7 +350,7 @@ func (t *backendRefTargetGroupModelBuildTask) buildTargetGroupSpec(ctx context.C } spec.VpcId = vpc spec.K8SSourceType = parentRefType - spec.ClusterName = eksCluster + spec.K8SClusterName = eksCluster spec.K8SServiceName = backendRefNsName.Name spec.K8SServiceNamespace = backendRefNsName.Namespace spec.K8SRouteName = t.route.Name() diff --git a/pkg/gateway/model_build_targetgroup_test.go b/pkg/gateway/model_build_targetgroup_test.go index 02aac5e3..0b736c75 100644 --- a/pkg/gateway/model_build_targetgroup_test.go +++ b/pkg/gateway/model_build_targetgroup_test.go @@ -258,7 +258,7 @@ func Test_TGModelByServicExportBuild(t *testing.T) { assert.Equal(t, vpclattice.IpAddressTypeIpv4, stackTg.Spec.IpAddressType) } - assert.Equal(t, config.ClusterName, stackTg.Spec.ClusterName) + assert.Equal(t, config.ClusterName, stackTg.Spec.K8SClusterName) assert.Equal(t, config.VpcID, stackTg.Spec.VpcId) assert.Equal(t, model.SourceTypeSvcExport, stackTg.Spec.K8SSourceType) assert.Equal(t, tt.svc.Name, stackTg.Spec.K8SServiceName) @@ -578,7 +578,7 @@ func Test_TGModelByHTTPRouteBuild(t *testing.T) { assert.Equal(t, vpclattice.IpAddressTypeIpv4, stackTg.Spec.IpAddressType) } - assert.Equal(t, config.ClusterName, stackTg.Spec.ClusterName) + assert.Equal(t, config.ClusterName, stackTg.Spec.K8SClusterName) assert.Equal(t, config.VpcID, stackTg.Spec.VpcId) assert.Equal(t, model.SourceTypeHTTPRoute, stackTg.Spec.K8SSourceType) assert.Equal(t, spec.K8SServiceName, stackTg.Spec.K8SServiceName) diff --git a/pkg/model/lattice/rule.go b/pkg/model/lattice/rule.go index a4267f75..1fe19dd3 100644 --- a/pkg/model/lattice/rule.go +++ b/pkg/model/lattice/rule.go @@ -41,7 +41,7 @@ type RuleTargetGroup struct { } type SvcImportTargetGroup struct { - EKSClusterName string `json:"eksclustername"` + K8SClusterName string `json:"k8sclustername"` K8SServiceName string `json:"k8sservicename"` K8SServiceNamespace string `json:"k8sservicenamespace"` VpcId string `json:"vpcid"` diff --git a/pkg/model/lattice/targetgroup.go b/pkg/model/lattice/targetgroup.go index 3d363d7d..ae61bd95 100644 --- a/pkg/model/lattice/targetgroup.go +++ b/pkg/model/lattice/targetgroup.go @@ -3,6 +3,7 @@ package lattice import ( "errors" "fmt" + "github.com/aws/aws-application-networking-k8s/pkg/aws" "github.com/aws/aws-application-networking-k8s/pkg/model/core" "github.com/aws/aws-application-networking-k8s/pkg/utils" "github.com/aws/aws-sdk-go/service/vpclattice" @@ -11,12 +12,12 @@ import ( ) const ( - EKSClusterNameKey = "ClusterName" - K8SServiceNameKey = "K8SServiceName" - K8SServiceNamespaceKey = "K8SServiceNamespace" - K8SRouteNameKey = "K8SRouteName" - K8SRouteNamespaceKey = "K8SRouteNamespace" - K8SSourceTypeKey = "K8SSourceTypeKey" + K8SClusterNameKey = aws.TagBase + "K8SClusterName" + K8SServiceNameKey = aws.TagBase + "ServiceName" + K8SServiceNamespaceKey = aws.TagBase + "ServiceNamespace" + K8SRouteNameKey = aws.TagBase + "RouteName" + K8SRouteNamespaceKey = aws.TagBase + "RouteNamespace" + K8SSourceTypeKey = aws.TagBase + "SourceTypeKey" MaxNamespaceLength = 55 MaxNameLength = 55 @@ -41,7 +42,7 @@ type TargetGroupSpec struct { TargetGroupTagFields } type TargetGroupTagFields struct { - ClusterName string `json:"clustername"` + K8SClusterName string `json:"k8sclustername"` K8SSourceType K8SSourceType `json:"k8ssourcetype"` K8SServiceName string `json:"k8sservicename"` K8SServiceNamespace string `json:"k8sservicenamespace"` @@ -70,7 +71,7 @@ const ( func TGTagFieldsFromTags(tags map[string]*string) TargetGroupTagFields { return TargetGroupTagFields{ - ClusterName: getMapValue(tags, EKSClusterNameKey), + K8SClusterName: getMapValue(tags, K8SClusterNameKey), K8SSourceType: GetParentRefType(getMapValue(tags, K8SSourceTypeKey)), K8SServiceName: getMapValue(tags, K8SServiceNameKey), K8SServiceNamespace: getMapValue(tags, K8SServiceNamespaceKey), @@ -106,7 +107,7 @@ func GetParentRefType(s string) K8SSourceType { func TagFieldsMatch(spec TargetGroupSpec, tags TargetGroupTagFields) bool { specTags := TargetGroupTagFields{ - ClusterName: spec.ClusterName, + K8SClusterName: spec.K8SClusterName, K8SSourceType: spec.K8SSourceType, K8SServiceName: spec.K8SServiceName, K8SServiceNamespace: spec.K8SServiceNamespace, @@ -148,7 +149,7 @@ func (t *TargetGroupTagFields) IsSourceTypeRoute() bool { func (t *TargetGroupSpec) Validate() error { requiredFields := []string{t.K8SServiceName, t.K8SServiceNamespace, - t.Protocol, t.ProtocolVersion, t.VpcId, t.ClusterName, t.IpAddressType, + t.Protocol, t.ProtocolVersion, t.VpcId, t.K8SClusterName, t.IpAddressType, string(t.K8SSourceType)} for _, s := range requiredFields { From 41c842a6ec01af0caf133515dfb2b13cef709047 Mon Sep 17 00:00:00 2001 From: Erik F <16261515+erikfuller@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:48:20 -0700 Subject: [PATCH 4/6] missed one tag name fix --- pkg/model/lattice/targetgroup.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/model/lattice/targetgroup.go b/pkg/model/lattice/targetgroup.go index ae61bd95..bab8e484 100644 --- a/pkg/model/lattice/targetgroup.go +++ b/pkg/model/lattice/targetgroup.go @@ -12,7 +12,7 @@ import ( ) const ( - K8SClusterNameKey = aws.TagBase + "K8SClusterName" + K8SClusterNameKey = aws.TagBase + "ClusterName" K8SServiceNameKey = aws.TagBase + "ServiceName" K8SServiceNamespaceKey = aws.TagBase + "ServiceNamespace" K8SRouteNameKey = aws.TagBase + "RouteName" From 50c1422e09e7864c19aa1b3fd549feb0d7f3fb44 Mon Sep 17 00:00:00 2001 From: Erik F <16261515+erikfuller@users.noreply.github.com> Date: Tue, 31 Oct 2023 11:56:52 -0700 Subject: [PATCH 5/6] fixed typo 'Succcess' -> 'Success' --- pkg/deploy/lattice/rule_manager.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/deploy/lattice/rule_manager.go b/pkg/deploy/lattice/rule_manager.go index 29015c25..8ee527cf 100644 --- a/pkg/deploy/lattice/rule_manager.go +++ b/pkg/deploy/lattice/rule_manager.go @@ -143,7 +143,7 @@ func (r *defaultRuleManager) update(ctx context.Context, return fmt.Errorf("failed UpdateRule %s due to %s", aws.StringValue(latticeRule.Id), err) } - r.log.Infof("Succcess UpdateRule %s, %s", aws.StringValue(res.Name), aws.StringValue(res.Id)) + r.log.Infof("Success UpdateRule %s, %s", aws.StringValue(res.Name), aws.StringValue(res.Id)) return nil } @@ -277,7 +277,7 @@ func (r *defaultRuleManager) create( return model.RuleStatus{}, fmt.Errorf("failed CreateRule %s, %s due to %s", latticeListenerId, latticeSvcId, err) } - r.log.Infof("Succcess CreateRule %s, %s", aws.StringValue(res.Name), aws.StringValue(res.Id)) + r.log.Infof("Success CreateRule %s, %s", aws.StringValue(res.Name), aws.StringValue(res.Id)) return model.RuleStatus{ Name: aws.StringValue(res.Name), @@ -357,6 +357,6 @@ func (r *defaultRuleManager) Delete(ctx context.Context, ruleId string, serviceI return fmt.Errorf("Failed DeleteRule %s/%s/%s due to %s", serviceId, listenerId, ruleId, err) } - r.log.Infof("Succcess DeleteRule %s/%s/%s", serviceId, listenerId, ruleId) + r.log.Infof("Success DeleteRule %s/%s/%s", serviceId, listenerId, ruleId) return nil } From ce4a868ee9cab85438ab4f6c10f87fa41508fe7f Mon Sep 17 00:00:00 2001 From: Erik F <16261515+erikfuller@users.noreply.github.com> Date: Tue, 31 Oct 2023 13:38:25 -0700 Subject: [PATCH 6/6] Fix for rule matching, also setting case sensitivity for rules according to spec --- pkg/deploy/lattice/rule_manager.go | 33 ++++++++++++--- pkg/deploy/lattice/rule_manager_test.go | 56 ++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 7 deletions(-) diff --git a/pkg/deploy/lattice/rule_manager.go b/pkg/deploy/lattice/rule_manager.go index 8ee527cf..25991ac4 100644 --- a/pkg/deploy/lattice/rule_manager.go +++ b/pkg/deploy/lattice/rule_manager.go @@ -301,23 +301,44 @@ func updateMatchFromRule(httpMatch *vpclattice.HttpMatch, modelRule *model.Rule) } httpMatch.PathMatch = &vpclattice.PathMatch{ - Match: &matchType, + Match: &matchType, + CaseSensitive: aws.Bool(true), // see PathMatchType.PathPrefix in gw spec } } - httpMatch.Method = &modelRule.Spec.Method + if modelRule.Spec.Method != "" { + httpMatch.Method = &modelRule.Spec.Method + } for i := 0; i < len(modelRule.Spec.MatchedHeaders); i++ { headerMatch := vpclattice.HeaderMatch{ - Match: modelRule.Spec.MatchedHeaders[i].Match, - Name: modelRule.Spec.MatchedHeaders[i].Name, + Match: modelRule.Spec.MatchedHeaders[i].Match, + Name: modelRule.Spec.MatchedHeaders[i].Name, + CaseSensitive: aws.Bool(false), // see HTTPHeaderMatch.HTTPHeaderName in gw spec } httpMatch.HeaderMatches = append(httpMatch.HeaderMatches, &headerMatch) } } -func isMatchEqual(lr1, lr2 *vpclattice.GetRuleOutput) bool { - return reflect.DeepEqual(lr1.Match, lr2.Match) +func isMatchEqual(localRule, latticeRule *vpclattice.GetRuleOutput) bool { + // currently lattice API converts nil HeaderMatches to empty list on create + // if we're currently nil, test both just in case it gets fixed later + if localRule.Match != nil && localRule.Match.HttpMatch != nil && + localRule.Match.HttpMatch.HeaderMatches == nil { + firstTry := reflect.DeepEqual(localRule.Match, latticeRule.Match) + if firstTry { + return true + } + // test with empty, then reset to original value + localRule.Match.HttpMatch.HeaderMatches = make([]*vpclattice.HeaderMatch, 0) + secondTry := reflect.DeepEqual(localRule.Match, latticeRule.Match) + localRule.Match.HttpMatch.HeaderMatches = nil + + return secondTry + } + + // otherwise we can rely on normal equality + return reflect.DeepEqual(localRule.Match, latticeRule.Match) } func (r *defaultRuleManager) nextAvailablePriority(latticeRules []*vpclattice.GetRuleOutput) (int64, error) { diff --git a/pkg/deploy/lattice/rule_manager_test.go b/pkg/deploy/lattice/rule_manager_test.go index 0baa2a71..28d535c3 100644 --- a/pkg/deploy/lattice/rule_manager_test.go +++ b/pkg/deploy/lattice/rule_manager_test.go @@ -49,6 +49,22 @@ func Test_Create(t *testing.T) { }, } + r2 := &model.Rule{ + Spec: model.RuleSpec{ + Priority: 1, + Action: model.RuleAction{ + TargetGroups: []*model.RuleTargetGroup{ + { + LatticeTgId: "tg-id", + Weight: 1, + }, + }, + }, + PathMatchPrefix: true, + PathMatchValue: "/foo", + }, + } + t.Run("test create", func(t *testing.T) { mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( []*vpclattice.GetRuleOutput{}, nil) @@ -66,7 +82,7 @@ func Test_Create(t *testing.T) { assert.Equal(t, "arn", ruleStatus.Arn) }) - t.Run("test update", func(t *testing.T) { + t.Run("test update method match", func(t *testing.T) { mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( []*vpclattice.GetRuleOutput{ { @@ -98,6 +114,44 @@ func Test_Create(t *testing.T) { assert.Equal(t, "existing-arn", ruleStatus.Arn) }) + t.Run("test update path match", func(t *testing.T) { + mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( + []*vpclattice.GetRuleOutput{ + { + Id: aws.String("existing-id"), + Arn: aws.String("existing-arn"), + Match: &vpclattice.RuleMatch{ + HttpMatch: &vpclattice.HttpMatch{ + HeaderMatches: make([]*vpclattice.HeaderMatch, 0), // this is what's returned in the Lattice API, not nil + PathMatch: &vpclattice.PathMatch{ + CaseSensitive: aws.Bool(true), // default value + Match: &vpclattice.PathMatchType{ + Prefix: aws.String("/foo"), + }, + }, + }, + }, + Action: &vpclattice.RuleAction{ + FixedResponse: &vpclattice.FixedResponseAction{}, // <-- this will trigger update + }, + Name: aws.String("existing-name"), + Priority: aws.Int64(1), + }, + }, nil) + + mockLattice.EXPECT().UpdateRuleWithContext(ctx, gomock.Any()).Return( + &vpclattice.UpdateRuleOutput{ + Arn: aws.String("existing-arn"), + Id: aws.String("existing-id"), + Name: aws.String("existing-name"), + }, nil) + + rm := NewRuleManager(gwlog.FallbackLogger, cloud) + ruleStatus, err := rm.Upsert(ctx, r2, l, svc) + assert.Nil(t, err) + assert.Equal(t, "existing-arn", ruleStatus.Arn) + }) + t.Run("test update - nothing to do", func(t *testing.T) { mockLattice.EXPECT().GetRulesAsList(ctx, gomock.Any()).Return( []*vpclattice.GetRuleOutput{