From d77b10a91e996d1373f5f42be51223568a230dc6 Mon Sep 17 00:00:00 2001 From: Christian Joun Date: Wed, 15 Apr 2020 10:59:35 -0700 Subject: [PATCH] WIP: Implemented API loadbalancer class, allowing NLB and ELB support on AWS. --- cmd/kops/create_cluster.go | 15 + docs/cluster_spec.md | 2 +- pkg/commands/status_discovery.go | 56 +- pkg/model/awsmodel/api_loadbalancer.go | 183 +++- pkg/model/dns.go | 45 +- pkg/model/names.go | 5 + pkg/model/spotinstmodel/instance_group.go | 4 + upup/pkg/fi/cloudup/awstasks/dnsname.go | 81 +- upup/pkg/fi/cloudup/awstasks/load_balancer.go | 2 + .../cloudup/awstasks/load_balancer_cleanup.go | 446 ++++++++++ .../awstasks/loadbalancercleanup_fitask.go | 75 ++ .../cloudup/awstasks/network_load_balancer.go | 823 ++++++++++++++++++ .../network_load_balancer_attachment.go | 160 ++++ .../networkloadbalancer_attributes.go | 273 ++++++ .../awstasks/networkloadbalancer_fitask.go | 75 ++ .../networkloadbalancer_healthchecks.go | 76 ++ .../networkloadbalancerattachment_fitask.go | 75 ++ .../fi/cloudup/awstasks/securitygrouprule.go | 8 +- upup/pkg/fi/cloudup/awsup/aws_apitarget.go | 51 ++ upup/pkg/fi/cloudup/awsup/aws_cloud.go | 29 + upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go | 4 + .../fi/cloudup/spotinsttasks/elastigroup.go | 37 +- upup/pkg/fi/default_methods.go | 1 + 23 files changed, 2464 insertions(+), 62 deletions(-) create mode 100644 upup/pkg/fi/cloudup/awstasks/load_balancer_cleanup.go create mode 100644 upup/pkg/fi/cloudup/awstasks/loadbalancercleanup_fitask.go create mode 100644 upup/pkg/fi/cloudup/awstasks/network_load_balancer.go create mode 100644 upup/pkg/fi/cloudup/awstasks/network_load_balancer_attachment.go create mode 100644 upup/pkg/fi/cloudup/awstasks/networkloadbalancer_attributes.go create mode 100644 upup/pkg/fi/cloudup/awstasks/networkloadbalancer_fitask.go create mode 100644 upup/pkg/fi/cloudup/awstasks/networkloadbalancer_healthchecks.go create mode 100644 upup/pkg/fi/cloudup/awstasks/networkloadbalancerattachment_fitask.go diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go index f23d952fd89c0..0841c53587eb8 100644 --- a/cmd/kops/create_cluster.go +++ b/cmd/kops/create_cluster.go @@ -80,6 +80,9 @@ type CreateClusterOptions struct { MasterTenancy string NodeTenancy string + // Specify API loadbalancer class as classic or nlb + APILoadBalancerClass string + // Allow custom public master name MasterPublicName string @@ -280,6 +283,7 @@ func NewCmdCreateCluster(f *util.Factory, out io.Writer) *cobra.Command { cmd.Flags().StringVar(&options.MasterTenancy, "master-tenancy", options.MasterTenancy, "The tenancy of the master group on AWS. Can either be default or dedicated.") cmd.Flags().StringVar(&options.NodeTenancy, "node-tenancy", options.NodeTenancy, "The tenancy of the node group on AWS. Can be either default or dedicated.") + cmd.Flags().StringVar(&options.APILoadBalancerClass, "api-loadbalancer-class", options.APILoadBalancerClass, "Currently only supported in AWS. Sets the API loadbalancer class to eiether 'classic' or 'network'") cmd.Flags().StringVar(&options.APILoadBalancerType, "api-loadbalancer-type", options.APILoadBalancerType, "Sets the API loadbalancer type to either 'public' or 'internal'") cmd.Flags().StringVar(&options.APISSLCertificate, "api-ssl-certificate", options.APISSLCertificate, "Currently only supported in AWS. Sets the ARN of the SSL Certificate to use for the API server loadbalancer.") @@ -489,6 +493,17 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr cluster.Spec.MasterPublicName = c.MasterPublicName } + if cluster.Spec.API.LoadBalancer != nil && cluster.Spec.API.LoadBalancer.Class == "" { + switch c.APILoadBalancerClass { + case "", "classic": + cluster.Spec.API.LoadBalancer.Class = api.LoadBalancerClassClassic + case "network": + cluster.Spec.API.LoadBalancer.Class = api.LoadBalancerClassNetwork + default: + return fmt.Errorf("unknown api-loadbalancer-class: %q", c.APILoadBalancerClass) + } + } + if err := commands.SetClusterFields(c.Overrides, cluster, instanceGroups); err != nil { return err } diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md index e6c8c6a95377c..8c5051eb4bbba 100644 --- a/docs/cluster_spec.md +++ b/docs/cluster_spec.md @@ -82,7 +82,7 @@ spec: *AWS only* You can choose to have a Network Load Balancer instead of a Classsic Load Balancer. The `class` -field should be either `Network` or `Classic` (default). Note: Note: changing the class of load balancer in an existing +field should be either `Network` or `Classic` (default). Note: changing the class of load balancer in an existing cluster is a disruptive operation. Until the masters have gone through a rolling update, new connections to the apiserver will fail due to the old master's TLS certificates containing the old load balancer's IP address. ```yaml spec: diff --git a/pkg/commands/status_discovery.go b/pkg/commands/status_discovery.go index 660ec2ea45bd4..b5d19c8cda294 100644 --- a/pkg/commands/status_discovery.go +++ b/pkg/commands/status_discovery.go @@ -18,6 +18,10 @@ package commands import ( "fmt" + "reflect" + + "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/aws/aws-sdk-go/aws" "k8s.io/kops/pkg/apis/kops" @@ -37,6 +41,10 @@ type CloudDiscoveryStatusStore struct { var _ kops.StatusStore = &CloudDiscoveryStatusStore{} +func isNil(v interface{}) bool { + return v == nil || (reflect.ValueOf(v).Kind() == reflect.Ptr && reflect.ValueOf(v).IsNil()) +} + func (s *CloudDiscoveryStatusStore) GetApiIngressStatus(cluster *kops.Cluster) ([]kops.ApiIngressStatus, error) { cloud, err := cloudup.BuildCloud(cluster) if err != nil { @@ -53,19 +61,51 @@ func (s *CloudDiscoveryStatusStore) GetApiIngressStatus(cluster *kops.Cluster) ( if awsCloud, ok := cloud.(awsup.AWSCloud); ok { name := "api." + cluster.Name - lb, err := awstasks.FindLoadBalancerByNameTag(awsCloud, name) - if lb == nil { - return nil, nil + + ELB, NLB := false, false + + lbSpec := cluster.Spec.API.LoadBalancer + switch lbSpec.Class { + case kops.LoadBalancerClassClassic, "": + ELB = true + case kops.LoadBalancerClassNetwork: + NLB = true + default: + return nil, fmt.Errorf("Unknown Cluster.Spec.API.LoadBalancer.Class : %v", lbSpec.Class) } - if err != nil { - return nil, fmt.Errorf("error looking for AWS ELB: %v", err) + + var lb interface{} + + if ELB { + lb, err = awstasks.FindLoadBalancerByNameTag(awsCloud, name) + if isNil(lb) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("error looking for AWS ELB: %v", err) + } + } else if NLB { + lb, err = awstasks.FindNetworkLoadBalancerByNameTag(awsCloud, name) + if isNil(lb) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("error looking for AWS ELB: %v", err) + } + } var ingresses []kops.ApiIngressStatus - if lb != nil { - lbDnsName := aws.StringValue(lb.DNSName) + if !isNil(lb) { + var lbDnsName string + if ELB { + lbDnsName = aws.StringValue(lb.(*elb.LoadBalancerDescription).DNSName) + } else if NLB { + lbDnsName = aws.StringValue(lb.(*elbv2.LoadBalancer).DNSName) + } + //lbDnsName := aws.StringValue(lb.DNSName) if lbDnsName == "" { - return nil, fmt.Errorf("found ELB %q, but it did not have a DNSName", name) + return nil, fmt.Errorf("found api LB %q, but it did not have a DNSName", name) } ingresses = append(ingresses, kops.ApiIngressStatus{Hostname: lbDnsName}) diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go index 0f2d1498e6187..598384bbadb3d 100644 --- a/pkg/model/awsmodel/api_loadbalancer.go +++ b/pkg/model/awsmodel/api_loadbalancer.go @@ -45,6 +45,40 @@ var _ fi.ModelBuilder = &APILoadBalancerBuilder{} // Build is responsible for building the KubeAPI tasks for the aws model func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { + NLB, ELB := false, false //TOOD: change ot using lbSpec.Type == kops.LoadBalancerClassClassic && lbSpec.Type == kops.LoadBalancerClassNetwork below + { + if b.UseLoadBalancerForAPI() { + lbSpec := b.Cluster.Spec.API.LoadBalancer + switch lbSpec.Class { + case kops.LoadBalancerClassClassic, "": + ELB = true + case kops.LoadBalancerClassNetwork: + NLB = true + default: + return fmt.Errorf("Unknown Cluster.Spec.API.LoadBalancer.Class : %v", lbSpec.Class) + } + } + + var agNames []*string + + if !featureflag.Spotinst.Enabled() { + for _, ig := range b.MasterInstanceGroups() { + agNames = append(agNames, b.LinkToAutoscalingGroup(ig).GetName()) + } + } + + cleanup := &awstasks.LoadBalancerCleanup{ + Name: fi.String("cleanup.api." + b.ClusterName()), + AgNames: agNames, + Lifecycle: b.Lifecycle, //what does this even do? + UseELBForAPI: fi.Bool(ELB), + UseNLBForAPI: fi.Bool(NLB), + NLBName: fi.String("api." + b.ClusterName()), + ELBName: fi.String("api." + b.ClusterName()), //TODO: think about changing their names? + } + c.AddTask(cleanup) + } + // Configuration where an ELB fronts the API if !b.UseLoadBalancerForAPI() { return nil @@ -65,7 +99,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } // Compute the subnets - only one per zone, and then break ties based on chooseBestSubnetForELB - var elbSubnets []*awstasks.Subnet + var lbSubnets []*awstasks.Subnet { subnetsByZone := make(map[string][]*kops.ClusterSubnetSpec) for i := range b.Cluster.Spec.Subnets { @@ -92,11 +126,12 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { for zone, subnets := range subnetsByZone { subnet := b.chooseBestSubnetForELB(zone, subnets) - elbSubnets = append(elbSubnets, b.LinkToSubnet(subnet)) + lbSubnets = append(lbSubnets, b.LinkToSubnet(subnet)) } } var elb *awstasks.LoadBalancer + var nlb *awstasks.NetworkLoadBalancer { loadBalancerName := b.GetELBName32("api") @@ -109,6 +144,10 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { "443": {InstancePort: 443}, } + nlbListeners := map[string]*awstasks.NetworkLoadBalancerListener{ + "443": {InstancePort: 443}, + } + if lbSpec.SSLCertificate != "" { listeners["443"] = &awstasks.LoadBalancerListener{InstancePort: 443, SSLCertificateID: lbSpec.SSLCertificate} } @@ -124,6 +163,26 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { // Override the returned name to be the expected ELB name tags["Name"] = "api." + b.ClusterName() + nlb = &awstasks.NetworkLoadBalancer{ + Name: fi.String("api." + b.ClusterName()), + Lifecycle: b.Lifecycle, + + LoadBalancerName: fi.String(loadBalancerName), + Subnets: lbSubnets, + Listeners: nlbListeners, + + // Configure fast-recovery health-checks + HealthCheck: &awstasks.NetworkLoadBalancerHealthCheck{ + HealthyThreshold: fi.Int64(2), + UnhealthyThreshold: fi.Int64(2), + Port: fi.String("443"), + }, + + Tags: tags, + VPC: b.LinkToVPC(), + Type: fi.String("network"), + } + elb = &awstasks.LoadBalancer{ Name: fi.String("api." + b.ClusterName()), Lifecycle: b.Lifecycle, @@ -132,7 +191,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { SecurityGroups: []*awstasks.SecurityGroup{ b.LinkToELBSecurityGroup("api"), }, - Subnets: elbSubnets, + Subnets: lbSubnets, Listeners: listeners, // Configure fast-recovery health-checks @@ -159,18 +218,34 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { Enabled: lbSpec.CrossZoneLoadBalancing, } + nlb.CrossZoneLoadBalancing = &awstasks.NetworkLoadBalancerCrossZoneLoadBalancing{ + Enabled: lbSpec.CrossZoneLoadBalancing, + } + switch lbSpec.Type { case kops.LoadBalancerTypeInternal: elb.Scheme = fi.String("internal") + nlb.Scheme = fi.String("internal") case kops.LoadBalancerTypePublic: elb.Scheme = nil + nlb.Scheme = nil default: - return fmt.Errorf("unknown elb Type: %q", lbSpec.Type) + return fmt.Errorf("unknown elb/nlb Type: %q", lbSpec.Type) + } + + if ELB { + c.AddTask(elb) + } else if NLB { + c.AddTask(nlb) } - c.AddTask(elb) } + // TODO: figure out if we need to do removeExtraRules + // TODO: this is referenced in other parts of code, need to figure out how to stop it from being + // referenced + // I0412 16:45:28.182976 73951 loader.go:292] tmpnlbcluster.k8s.local-addons-storage-aws.addons.k8s.io-v1.7.0 + // error building tasks: unexpected error resolving task "LoadBalancer/api.tmpnlbcluster.k8s.local": unable to find task "SecurityGroup/api-elb.tmpnlbcluster.k8s.local", referenced from LoadBalancer/api.tmpnlbcluster.k8s.local:.SecurityGroups[0] // Create security group for API ELB var lbSG *awstasks.SecurityGroup { @@ -192,7 +267,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } // Allow traffic from ELB to egress freely - { + if ELB { t := &awstasks.SecurityGroupRule{ Name: fi.String("api-elb-egress"), Lifecycle: b.SecurityLifecycle, @@ -204,7 +279,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } // Allow traffic into the ELB from KubernetesAPIAccess CIDRs - { + if ELB { for _, cidr := range b.Cluster.Spec.KubernetesAPIAccess { t := &awstasks.SecurityGroupRule{ Name: fi.String("https-api-elb-" + cidr), @@ -230,8 +305,44 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } } + masterGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleMaster) + if err != nil { + return err + } + + if NLB { + for _, cidr := range b.Cluster.Spec.KubernetesAPIAccess { + + for _, masterGroup := range masterGroups { + suffix := masterGroup.Suffix + t := &awstasks.SecurityGroupRule{ + Name: fi.String(fmt.Sprintf("https-nlb-to-master%s-%s", suffix, cidr)), //TODO: what is a good name for this? + Lifecycle: b.SecurityLifecycle, + CIDR: fi.String(cidr), + FromPort: fi.Int64(443), + Protocol: fi.String("tcp"), + SecurityGroup: masterGroup.Task, + ToPort: fi.Int64(443), + } + c.AddTask(t) + + // Allow ICMP traffic required for PMTU discovery + c.AddTask(&awstasks.SecurityGroupRule{ + Name: fi.String(fmt.Sprintf("icmp-pmtu-api-master%s-%s", suffix, cidr)), + Lifecycle: b.SecurityLifecycle, + CIDR: fi.String(cidr), + FromPort: fi.Int64(3), + Protocol: fi.String("icmp"), + SecurityGroup: masterGroup.Task, + ToPort: fi.Int64(4), + }) + } + } + } + + //TODO: not implemented for NLB, would the security groups go to the master nodes? // Add precreated additional security groups to the ELB - { + if ELB { for _, id := range b.Cluster.Spec.API.LoadBalancer.AdditionalSecurityGroups { t := &awstasks.SecurityGroup{ Name: fi.String(id), @@ -246,13 +357,8 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } } - masterGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleMaster) - if err != nil { - return err - } - // Allow HTTPS to the master instances from the ELB - { + if ELB { for _, masterGroup := range masterGroups { suffix := masterGroup.Suffix c.AddTask(&awstasks.SecurityGroupRule{ @@ -267,10 +373,35 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { } } + if NLB { + // Can tighten security by allowing only https access from the private ip's of the eni's associated with the nlb's nodes in each availability zone. + // Recommended approach is the whole vpc cidr https://docs.aws.amazon.com/elasticloadbalancing/latest/network/target-group-register-targets.html#target-security-groups + // work around suggested here https://forums.aws.amazon.com/thread.jspa?threadID=263245&start=0&tstart=0 + // https://docs.aws.amazon.com/sdk-for-go/api/aws/arn/ + for _, masterGroup := range masterGroups { + suffix := masterGroup.Suffix + c.AddTask(&awstasks.SecurityGroupRule{ + Name: fi.String(fmt.Sprintf("nlb-health-check-to-master%s", suffix)), + Lifecycle: b.SecurityLifecycle, + FromPort: fi.Int64(443), + Protocol: fi.String("tcp"), + SecurityGroup: masterGroup.Task, + ToPort: fi.Int64(443), + VPC: b.LinkToVPC(), + }) + } + } + if dns.IsGossipHostname(b.Cluster.Name) || b.UsePrivateDNS() { // Ensure the ELB hostname is included in the TLS certificate, // if we're not going to use an alias for it - elb.ForAPIServer = true + if ELB { + elb.ForAPIServer = true + } else if NLB { + fmt.Println("not yet implemented") + //nlb.ForAPIServer = true + } + } // When Spotinst Elastigroups are used, there is no need to create @@ -278,12 +409,22 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error { // is already done as part of the Elastigroup's creation, if needed. if !featureflag.Spotinst.Enabled() { for _, ig := range b.MasterInstanceGroups() { - c.AddTask(&awstasks.LoadBalancerAttachment{ - Name: fi.String("api-" + ig.ObjectMeta.Name), - Lifecycle: b.Lifecycle, - AutoscalingGroup: b.LinkToAutoscalingGroup(ig), - LoadBalancer: b.LinkToELB("api"), - }) + if ELB { + c.AddTask(&awstasks.LoadBalancerAttachment{ + Name: fi.String("api-" + ig.ObjectMeta.Name), + Lifecycle: b.Lifecycle, + AutoscalingGroup: b.LinkToAutoscalingGroup(ig), + LoadBalancer: b.LinkToELB("api"), + }) + } + if NLB { + c.AddTask(&awstasks.NetworkLoadBalancerAttachment{ + Name: fi.String("api-" + ig.ObjectMeta.Name), + Lifecycle: b.Lifecycle, + AutoscalingGroup: b.LinkToAutoscalingGroup(ig), + LoadBalancer: b.LinkToNLB("api"), + }) + } } } diff --git a/pkg/model/dns.go b/pkg/model/dns.go index 3de3e5fda81e3..90f521f13e632 100644 --- a/pkg/model/dns.go +++ b/pkg/model/dns.go @@ -91,6 +91,29 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { } } + //TODO: use interface (but what about bastion?), should I create a new member instead of targetLoadBalancer for the interface? + + ELB, NLB := false, false + lbSpec := b.Cluster.Spec.API.LoadBalancer + switch lbSpec.Class { + case kops.LoadBalancerClassClassic, "": + ELB = true + case kops.LoadBalancerClassNetwork: + NLB = true + default: + return fmt.Errorf("Unknown Cluster.Spec.API.LoadBalancer.Class : %v", lbSpec.Class) + } + + var targetLoadBalancer *awstasks.LoadBalancer + var targetNetworkLoadBalancer *awstasks.NetworkLoadBalancer + + if ELB { + targetLoadBalancer = b.LinkToELB("api") + } + if NLB { + targetNetworkLoadBalancer = b.LinkToNLB("api") + } + if b.UseLoadBalancerForAPI() { // This will point our external DNS record to the load balancer, and put the // pieces together for kubectl to work @@ -101,11 +124,12 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { } apiDnsName := &awstasks.DNSName{ - Name: s(b.Cluster.Spec.MasterPublicName), - Lifecycle: b.Lifecycle, - Zone: b.LinkToDNSZone(), - ResourceType: s("A"), - TargetLoadBalancer: awstasks.DNSTarget(b.LinkToELB("api")), + Name: s(b.Cluster.Spec.MasterPublicName), + Lifecycle: b.Lifecycle, + Zone: b.LinkToDNSZone(), + ResourceType: s("A"), + TargetLoadBalancer: awstasks.DNSTarget(b.LinkToELB("api")), + TargetNetworkLoadBalancer: targetNetworkLoadBalancer, } c.AddTask(apiDnsName) } @@ -121,11 +145,12 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error { } internalApiDnsName := &awstasks.DNSName{ - Name: s(b.Cluster.Spec.MasterInternalName), - Lifecycle: b.Lifecycle, - Zone: b.LinkToDNSZone(), - ResourceType: s("A"), - TargetLoadBalancer: b.LinkToELB("api"), + Name: s(b.Cluster.Spec.MasterInternalName), + Lifecycle: b.Lifecycle, + Zone: b.LinkToDNSZone(), + ResourceType: s("A"), + TargetLoadBalancer: targetLoadBalancer, + TargetNetworkLoadBalancer: targetNetworkLoadBalancer, } // Using EnsureTask as MasterInternalName and MasterPublicName could be the same c.EnsureTask(internalApiDnsName) diff --git a/pkg/model/names.go b/pkg/model/names.go index 04251c4477590..4e2ff921fa16f 100644 --- a/pkg/model/names.go +++ b/pkg/model/names.go @@ -90,6 +90,11 @@ func (b *KopsModelContext) LinkToELB(prefix string) *awstasks.LoadBalancer { return &awstasks.LoadBalancer{Name: &name} } +func (b *KopsModelContext) LinkToNLB(prefix string) *awstasks.NetworkLoadBalancer { + name := b.ELBName(prefix) //TODO: does this need to change? + return &awstasks.NetworkLoadBalancer{Name: &name} +} + func (b *KopsModelContext) LinkToVPC() *awstasks.VPC { name := b.ClusterName() return &awstasks.VPC{Name: &name} diff --git a/pkg/model/spotinstmodel/instance_group.go b/pkg/model/spotinstmodel/instance_group.go index 855e9f1f63ed3..a943d76c0dd45 100644 --- a/pkg/model/spotinstmodel/instance_group.go +++ b/pkg/model/spotinstmodel/instance_group.go @@ -253,6 +253,10 @@ func (b *InstanceGroupModelBuilder) buildElastigroup(c *fi.ModelBuilderContext, return fmt.Errorf("error building ssh key: %v", err) } + if true { + panic("this code is not ready for NLB") + } + // Load balancer. var lb *awstasks.LoadBalancer switch ig.Spec.Role { diff --git a/upup/pkg/fi/cloudup/awstasks/dnsname.go b/upup/pkg/fi/cloudup/awstasks/dnsname.go index e65609ce0580d..81b0b543227d4 100644 --- a/upup/pkg/fi/cloudup/awstasks/dnsname.go +++ b/upup/pkg/fi/cloudup/awstasks/dnsname.go @@ -24,6 +24,7 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/route53" "k8s.io/klog/v2" + "k8s.io/kops/pkg/apis/kops" "k8s.io/kops/upup/pkg/fi" "k8s.io/kops/upup/pkg/fi/cloudup/awsup" "k8s.io/kops/upup/pkg/fi/cloudup/cloudformation" @@ -39,7 +40,8 @@ type DNSName struct { Zone *DNSZone ResourceType *string - TargetLoadBalancer DNSTarget + TargetLoadBalancer DNSTarget + TargetNetworkLoadBalancer *NetworkLoadBalancer } type DNSTarget interface { @@ -119,26 +121,65 @@ func (e *DNSName) Find(c *fi.Context) (*DNSName, error) { dnsName := aws.StringValue(found.AliasTarget.DNSName) klog.Infof("AliasTarget for %q is %q", aws.StringValue(found.Name), dnsName) if dnsName != "" { - // TODO: check "looks like" an ELB? - lb, err := findLoadBalancerByAlias(cloud, found.AliasTarget) - if err != nil { - return nil, fmt.Errorf("error mapping DNSName %q to LoadBalancer: %v", dnsName, err) + + //TODO: use interface (but what about bastion?), should I create a new member instead of targetLoadBalancer for the interface? + ELB, NLB := false, false + lbSpec := c.Cluster.Spec.API.LoadBalancer + switch lbSpec.Class { + case kops.LoadBalancerClassClassic, "": + ELB = true + case kops.LoadBalancerClassNetwork: + NLB = true + default: + return nil, fmt.Errorf("Unknown Cluster.Spec.API.LoadBalancer.Class : %v", lbSpec.Class) + } + + if NLB { + // TODO: check "looks like" an ELB? + lb, err := findNetworkLoadBalancerByAlias(cloud, found.AliasTarget) + if err != nil { + return nil, fmt.Errorf("error mapping DNSName %q to LoadBalancer: %v", dnsName, err) + } + if lb == nil { + klog.Warningf("Unable to find load balancer with DNS name: %q", dnsName) + } else { + loadBalancerName := aws.StringValue(lb.LoadBalancerName) //TOOD: can we keep these on object + loadBalancerArn := aws.StringValue(lb.LoadBalancerArn) //TODO: can we keep these on object + tagMap, err := describeNetworkLoadBalancerTags(cloud, []string{loadBalancerArn}) + if err != nil { + return nil, err + } + tags := tagMap[loadBalancerName] + nameTag, _ := awsup.FindELBV2Tag(tags, "Name") + if nameTag == "" { + return nil, fmt.Errorf("Found NLB %q linked to DNS name %q, but it did not have a Name tag", loadBalancerName, fi.StringValue(e.Name)) + } + actual.TargetNetworkLoadBalancer = &NetworkLoadBalancer{Name: fi.String(nameTag)} + } } - if lb == nil { - klog.Warningf("Unable to find load balancer with DNS name: %q", dnsName) - } else { - loadBalancerName := aws.StringValue(lb.LoadBalancerName) - tagMap, err := describeLoadBalancerTags(cloud, []string{loadBalancerName}) + if ELB { + // TODO: check "looks like" an ELB? + lb, err := findLoadBalancerByAlias(cloud, found.AliasTarget) if err != nil { - return nil, err + return nil, fmt.Errorf("error mapping DNSName %q to LoadBalancer: %v", dnsName, err) } - tags := tagMap[loadBalancerName] - nameTag, _ := awsup.FindELBTag(tags, "Name") - if nameTag == "" { - return nil, fmt.Errorf("Found ELB %q linked to DNS name %q, but it did not have a Name tag", loadBalancerName, fi.StringValue(e.Name)) + if lb == nil { + klog.Warningf("Unable to find load balancer with DNS name: %q", dnsName) + } else { + loadBalancerName := aws.StringValue(lb.LoadBalancerName) + tagMap, err := describeLoadBalancerTags(cloud, []string{loadBalancerName}) + if err != nil { + return nil, err + } + tags := tagMap[loadBalancerName] + nameTag, _ := awsup.FindELBTag(tags, "Name") + if nameTag == "" { + return nil, fmt.Errorf("Found ELB %q linked to DNS name %q, but it did not have a Name tag", loadBalancerName, fi.StringValue(e.Name)) + } + actual.TargetLoadBalancer = &LoadBalancer{Name: fi.String(nameTag)} } - actual.TargetLoadBalancer = &LoadBalancer{Name: fi.String(nameTag)} } + } } @@ -172,6 +213,14 @@ func (_ *DNSName) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *DNSName) error } } + if e.TargetNetworkLoadBalancer != nil { + rrs.AliasTarget = &route53.AliasTarget{ + DNSName: e.TargetNetworkLoadBalancer.DNSName, + EvaluateTargetHealth: aws.Bool(false), + HostedZoneId: e.TargetNetworkLoadBalancer.HostedZoneId, + } + } + change := &route53.Change{ Action: aws.String("UPSERT"), ResourceRecordSet: rrs, diff --git a/upup/pkg/fi/cloudup/awstasks/load_balancer.go b/upup/pkg/fi/cloudup/awstasks/load_balancer.go index cfdc970a42892..bb3578a9435f4 100644 --- a/upup/pkg/fi/cloudup/awstasks/load_balancer.go +++ b/upup/pkg/fi/cloudup/awstasks/load_balancer.go @@ -72,6 +72,7 @@ type LoadBalancer struct { } var _ fi.CompareWithID = &LoadBalancer{} +var _ fi.ProducesDeletions = &LaunchConfiguration{} func (e *LoadBalancer) CompareWithID() *string { return e.Name @@ -490,6 +491,7 @@ func (s *LoadBalancer) CheckChanges(a, e, changes *LoadBalancer) error { } func (_ *LoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *LoadBalancer) error { + var loadBalancerName string if a == nil { if e.LoadBalancerName == nil { diff --git a/upup/pkg/fi/cloudup/awstasks/load_balancer_cleanup.go b/upup/pkg/fi/cloudup/awstasks/load_balancer_cleanup.go new file mode 100644 index 0000000000000..7290d9f13b365 --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/load_balancer_cleanup.go @@ -0,0 +1,446 @@ +/* + +Copyright 2016 The Kubernetes Authors. + + + +Licensed under the Apache License, Version 2.0 (the "License"); + +you may not use this file except in compliance with the License. + +You may obtain a copy of the License at + + + + http://www.apache.org/licenses/LICENSE-2.0 + + + +Unless required by applicable law or agreed to in writing, software + +distributed under the License is distributed on an "AS IS" BASIS, + +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + +See the License for the specific language governing permissions and + +limitations under the License. + +*/ + +package awstasks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/elbv2" + "k8s.io/klog" + "k8s.io/kops/upup/pkg/fi" + + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" +) + +// LoadBalancer manages an ELB. We find the existing ELB using the Name tag. + +//go:generate fitask -type=LoadBalancerCleanup + +type LoadBalancerCleanup struct { + + // We use the Name tag to find the existing ELB, because we are (more or less) unrestricted when + + // it comes to tag values, but the LoadBalancerName is length limited + + Name *string + UseNLBForAPI *bool + UseELBForAPI *bool + AgNames []*string + NLBName *string + ELBName *string + + Lifecycle *fi.Lifecycle +} + +type deleteLoadBalancer struct { + request *elb.DeleteLoadBalancerInput +} + +var _ fi.Deletion = &deleteLoadBalancer{} + +func (d *deleteLoadBalancer) TaskName() string { + return "LoadBalancer" +} + +func (d *deleteLoadBalancer) Item() string { + return aws.StringValue(d.request.LoadBalancerName) +} + +func (d *deleteLoadBalancer) Delete(t fi.Target) error { + klog.V(2).Infof("deleting elb %v", d) + + awsTarget, ok := t.(*awsup.AWSAPITarget) + if !ok { + return fmt.Errorf("unexpected target type for deletion: %T", t) + } + + name := aws.StringValue(d.request.LoadBalancerName) + klog.V(2).Infof("Calling elb DeleteLoadBalancer for %s", name) + + _, err := awsTarget.Cloud.ELB().DeleteLoadBalancer(d.request) + + if err != nil { + return fmt.Errorf("error deleting elb %s: %v", name, err) + } + + return nil +} + +func (d *deleteLoadBalancer) String() string { + return d.TaskName() + "-" + d.Item() +} + +type detachLoadBalancer struct { + request *autoscaling.DetachLoadBalancersInput +} + +var _ fi.Deletion = &detachLoadBalancer{} + +func (d *detachLoadBalancer) TaskName() string { + return "Autoscaling LoadBalancerAttachment" +} + +func (d *detachLoadBalancer) Item() string { + tmp := *d.request.LoadBalancerNames[0] + " -> " + *d.request.AutoScalingGroupName + return aws.StringValue(&tmp) +} + +func (d *detachLoadBalancer) Delete(t fi.Target) error { + klog.V(2).Infof("deleting elb %v", d) + + awsTarget, ok := t.(*awsup.AWSAPITarget) + if !ok { + return fmt.Errorf("unexpected target type for deletion: %T", t) + } + + name := aws.StringValue(d.request.AutoScalingGroupName) + klog.V(2).Infof("Calling autoscaling Detach LoadBalancer for autoscaling group %s", name) + + _, err := awsTarget.Cloud.Autoscaling().DetachLoadBalancers(d.request) + + if err != nil { + return fmt.Errorf("Error Detaching LoadBalancers from Autoscaling group : %v", err) + } + + return nil +} + +func (d *detachLoadBalancer) String() string { + return d.TaskName() + "-" + d.Item() +} + +type deleteTargetGroup struct { + request *elbv2.DeleteTargetGroupInput +} + +var _ fi.Deletion = &deleteTargetGroup{} + +func (d *deleteTargetGroup) TaskName() string { + return "TargetGroup" +} + +func (d *deleteTargetGroup) Item() string { + return aws.StringValue(d.request.TargetGroupArn) +} + +func (d *deleteTargetGroup) Delete(t fi.Target) error { + klog.V(2).Infof("deleting target group %v", d) + + awsTarget, ok := t.(*awsup.AWSAPITarget) + if !ok { + return fmt.Errorf("unexpected target type for deletion: %T", t) + } + + name := aws.StringValue(d.request.TargetGroupArn) + klog.V(2).Infof("Calling Nlb DeleteTargetGroup for %s", name) + + _, err := awsTarget.Cloud.ELBV2().DeleteTargetGroup(d.request) + + if err != nil { + return fmt.Errorf("error Deleting TargetGroup from NLB: %v", err) + } + + return nil +} + +func (d *deleteTargetGroup) String() string { + return d.TaskName() + "-" + d.Item() +} + +type deleteNetworkLoadBalancer struct { + request *elbv2.DeleteLoadBalancerInput +} + +var _ fi.Deletion = &deleteNetworkLoadBalancer{} + +func (d *deleteNetworkLoadBalancer) TaskName() string { + return "LoadBalancer" +} + +func (d *deleteNetworkLoadBalancer) Item() string { + return aws.StringValue(d.request.LoadBalancerArn) +} + +func (d *deleteNetworkLoadBalancer) Delete(t fi.Target) error { + klog.V(2).Infof("deleting nlb %v", d) + + awsTarget, ok := t.(*awsup.AWSAPITarget) + if !ok { + return fmt.Errorf("unexpected target type for deletion: %T", t) + } + + name := aws.StringValue(d.request.LoadBalancerArn) + klog.V(2).Infof("Calling elb DeleteLoadBalancer for %s", name) + + _, err := awsTarget.Cloud.ELBV2().DeleteLoadBalancer(d.request) + + if err != nil { + return fmt.Errorf("error deleting nlb %s: %v", name, err) + } + + return nil +} + +func (d *deleteNetworkLoadBalancer) String() string { + return d.TaskName() + "-" + d.Item() +} + +type detachNetworkLoadBalancer struct { + request *autoscaling.DetachLoadBalancerTargetGroupsInput +} + +var _ fi.Deletion = &detachNetworkLoadBalancer{} + +func (d *detachNetworkLoadBalancer) TaskName() string { + return "Autoscaling LoadBalancerTargetGroupAttachment" +} + +func (d *detachNetworkLoadBalancer) Item() string { + tmp := *d.request.TargetGroupARNs[0] + " -> " + *d.request.AutoScalingGroupName + return aws.StringValue(&tmp) +} + +func (d *detachNetworkLoadBalancer) Delete(t fi.Target) error { + klog.V(2).Infof("deleting elb %v", d) + + awsTarget, ok := t.(*awsup.AWSAPITarget) + if !ok { + return fmt.Errorf("unexpected target type for deletion: %T", t) + } + + name := aws.StringValue(d.request.AutoScalingGroupName) + klog.V(2).Infof("Calling autoscaling Detach LoadBalancer for autoscaling group %s", name) + + _, err := awsTarget.Cloud.Autoscaling().DetachLoadBalancerTargetGroups(d.request) + + if err != nil { + return fmt.Errorf("Error Detaching LoadBalancer TargetGroup from Autoscaling group : %v", err) + } + + return nil +} + +func (d *detachNetworkLoadBalancer) String() string { + return d.TaskName() + "-" + d.Item() +} + +func (e *LoadBalancerCleanup) FindELBDeletions(c *fi.Context) ([]fi.Deletion, error) { + var removals []fi.Deletion + + cloud := c.Cloud.(awsup.AWSCloud) + + lb, err := FindLoadBalancerByNameTag(cloud, fi.StringValue(e.ELBName)) + + if err != nil { + return nil, err + } + + if lb != nil { + + request := &elb.DeleteLoadBalancerInput{ + LoadBalancerName: lb.LoadBalancerName, + } + + removals = append(removals, &deleteLoadBalancer{request: request}) + klog.V(2).Infof("will delete load balancer: %v", lb.LoadBalancerName) + } + + for _, autoScalingGroupName := range e.AgNames { + + request := &autoscaling.DescribeLoadBalancersInput{ + AutoScalingGroupName: autoScalingGroupName, + } + response, err := cloud.Autoscaling().DescribeLoadBalancers(request) + + if err != nil { + return nil, nil + } + + for _, LoadBalancerState := range response.LoadBalancers { //detach all elbs from autoscaling group + + loadBalancerName := LoadBalancerState.LoadBalancerName + + request := &autoscaling.DetachLoadBalancersInput{ + AutoScalingGroupName: autoScalingGroupName, + LoadBalancerNames: []*string{ + loadBalancerName, + }, + } + + removals = append(removals, &detachLoadBalancer{request: request}) + klog.V(2).Infof("will detach load balancer: %v from autoscalinggroup %v", loadBalancerName, autoScalingGroupName) + } + } + + return removals, nil +} + +func (e *LoadBalancerCleanup) FindNLBDeletions(c *fi.Context) ([]fi.Deletion, error) { + var removals []fi.Deletion + + cloud := c.Cloud.(awsup.AWSCloud) + + lb, err := FindNetworkLoadBalancerByNameTag(cloud, fi.StringValue(e.NLBName)) + + if err != nil { + return nil, err + } + + if lb != nil { + + request := &elbv2.DeleteLoadBalancerInput{ + LoadBalancerArn: lb.LoadBalancerArn, + } + + removals = append(removals, &deleteNetworkLoadBalancer{request: request}) + klog.V(2).Infof("will delete network load balancer: %v", lb.LoadBalancerName) + + //TODO: Could be useful to check the autoscaling group for associated target groups and delete them as well. + { + klog.V(2).Infof("Describing Target Groups for loadBalancerArn : %v\n", lb.LoadBalancerArn) + request := &elbv2.DescribeTargetGroupsInput{ + LoadBalancerArn: lb.LoadBalancerArn, + } + response, err := cloud.ELBV2().DescribeTargetGroups(request) + if err != nil { + return nil, fmt.Errorf("error querying for NLB Target groups :%v", err) + } + + if len(response.TargetGroups) == 0 { + return nil, fmt.Errorf("Found no Target Groups for NLB don't think this is a normal condition : %q", lb.LoadBalancerArn) + } + + if len(response.TargetGroups) != 1 { + return nil, fmt.Errorf("Found multiple Target groups for NLB with arn %q", lb.LoadBalancerArn) + } + + targetGroupArn := response.TargetGroups[0].TargetGroupArn + + { + + klog.V(2).Infof("Deleting Target Group with arn : %v\n", targetGroupArn) + + request := &elbv2.DeleteTargetGroupInput{ + TargetGroupArn: targetGroupArn, + } + + removals = append(removals, &deleteTargetGroup{request: request}) + } + } + + } + + for _, autoScalingGroupName := range e.AgNames { + + request := &autoscaling.DescribeLoadBalancerTargetGroupsInput{ + AutoScalingGroupName: autoScalingGroupName, + } + + response, err := cloud.Autoscaling().DescribeLoadBalancerTargetGroups(request) + + if err != nil { + return nil, nil + } + + for _, LoadBalancerState := range response.LoadBalancerTargetGroups { //detach all elbs from autoscaling group + + targetGroupArn := LoadBalancerState.LoadBalancerTargetGroupARN + + request := &autoscaling.DetachLoadBalancerTargetGroupsInput{ + AutoScalingGroupName: autoScalingGroupName, + TargetGroupARNs: []*string{ + targetGroupArn, + }, + } + + removals = append(removals, &detachNetworkLoadBalancer{request: request}) + klog.V(2).Infof("will detach targetGroup from autoscalinggroup %v", targetGroupArn, autoScalingGroupName) + + } + } + + return removals, nil +} + +func (e *LoadBalancerCleanup) FindDeletions(c *fi.Context) ([]fi.Deletion, error) { + + if *e.UseELBForAPI { + return e.FindNLBDeletions(c) + } else if *e.UseNLBForAPI { + return e.FindELBDeletions(c) + } else { + nlbDeletions, err := e.FindNLBDeletions(c) + if err != nil { + return nil, err + } + elbDeletions, err := e.FindELBDeletions(c) + if err != nil { + return nil, err + } + return append(nlbDeletions, elbDeletions...), nil + } + +} + +var _ fi.CompareWithID = &LoadBalancerCleanup{} + +func (e *LoadBalancerCleanup) CompareWithID() *string { + return e.Name +} + +func (e *LoadBalancerCleanup) Find(c *fi.Context) (*LoadBalancerCleanup, error) { + //avoid spurious mismatches + actual := &LoadBalancerCleanup{} + actual.Name = e.Name + actual.Lifecycle = e.Lifecycle + actual.AgNames = e.AgNames + actual.UseNLBForAPI = e.UseNLBForAPI + actual.UseELBForAPI = e.UseELBForAPI + actual.NLBName = e.NLBName + actual.ELBName = e.ELBName + return actual, nil + +} + +func (e *LoadBalancerCleanup) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (s *LoadBalancerCleanup) CheckChanges(a, e, changes *LoadBalancerCleanup) error { + return nil +} + +func (_ *LoadBalancerCleanup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *LoadBalancerCleanup) error { + return nil +} diff --git a/upup/pkg/fi/cloudup/awstasks/loadbalancercleanup_fitask.go b/upup/pkg/fi/cloudup/awstasks/loadbalancercleanup_fitask.go new file mode 100644 index 0000000000000..f2fd848e7efb0 --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/loadbalancercleanup_fitask.go @@ -0,0 +1,75 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by ""fitask" -type=LoadBalancerCleanup"; DO NOT EDIT + +package awstasks + +import ( + "encoding/json" + + "k8s.io/kops/upup/pkg/fi" +) + +// LoadBalancerCleanup + +// JSON marshaling boilerplate +type realLoadBalancerCleanup LoadBalancerCleanup + +// UnmarshalJSON implements conversion to JSON, supporting an alternate specification of the object as a string +func (o *LoadBalancerCleanup) UnmarshalJSON(data []byte) error { + var jsonName string + if err := json.Unmarshal(data, &jsonName); err == nil { + o.Name = &jsonName + return nil + } + + var r realLoadBalancerCleanup + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *o = LoadBalancerCleanup(r) + return nil +} + +var _ fi.HasLifecycle = &LoadBalancerCleanup{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *LoadBalancerCleanup) GetLifecycle() *fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *LoadBalancerCleanup) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = &lifecycle +} + +var _ fi.HasName = &LoadBalancerCleanup{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *LoadBalancerCleanup) GetName() *string { + return o.Name +} + +// SetName sets the Name of the object, implementing fi.SetName +func (o *LoadBalancerCleanup) SetName(name string) { + o.Name = &name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *LoadBalancerCleanup) String() string { + return fi.TaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go b/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go new file mode 100644 index 0000000000000..4ca18f5a7a0ab --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/network_load_balancer.go @@ -0,0 +1,823 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awstasks + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/service/elbv2" + "github.com/aws/aws-sdk-go/service/route53" + "k8s.io/klog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" + "k8s.io/kops/util/pkg/slice" +) + +// LoadBalancer manages an ELB. We find the existing ELB using the Name tag. + +//go:generate fitask -type=NetworkLoadBalancer +type NetworkLoadBalancer struct { + // We use the Name tag to find the existing ELB, because we are (more or less) unrestricted when + // it comes to tag values, but the LoadBalancerName is length limited + Name *string + Lifecycle *fi.Lifecycle + + // LoadBalancerName is the name in ELB, possibly different from our name + // (ELB is restricted as to names, so we have limited choices!) + // We use the Name tag to find the existing ELB. + LoadBalancerName *string + + DNSName *string + HostedZoneId *string + + Subnets []*Subnet + SecurityGroups []*SecurityGroup + + Listeners map[string]*NetworkLoadBalancerListener + + Scheme *string + + HealthCheck *NetworkLoadBalancerHealthCheck + AccessLog *NetworkLoadBalancerAccessLog + CrossZoneLoadBalancing *NetworkLoadBalancerCrossZoneLoadBalancing + SSLCertificateID string + + Tags map[string]string + ForAPIServer bool + + Type *string + + VPC *VPC + DeletionProtection *NetworkLoadBalancerDeletionProtection + ProxyProtocolV2 *TargetGroupProxyProtocolV2 + Stickiness *TargetGroupStickiness + DeregistationDelay *TargetGroupDeregistrationDelay +} + +var _ fi.CompareWithID = &NetworkLoadBalancer{} + +func (e *NetworkLoadBalancer) CompareWithID() *string { + return e.Name +} + +type NetworkLoadBalancerListener struct { + InstancePort int //TODO: Change this to LoadBalancerPort + SSLCertificateID string +} + +func (e *NetworkLoadBalancerListener) mapToAWS(loadBalancerPort int64, targetGroupArn string, loadBalancerArn string) *elbv2.CreateListenerInput { + + l := &elbv2.CreateListenerInput{ + DefaultActions: []*elbv2.Action{ + { + TargetGroupArn: aws.String(targetGroupArn), + Type: aws.String("forward"), + }, + }, + LoadBalancerArn: aws.String(loadBalancerArn), + Port: aws.Int64(loadBalancerPort), + } + + if e.SSLCertificateID != "" { + l.Certificates = []*elbv2.Certificate{} + l.Certificates = append(l.Certificates, &elbv2.Certificate{ + CertificateArn: aws.String(e.SSLCertificateID), + }) + l.Protocol = aws.String("SSL") + } else { + l.Protocol = aws.String("TCP") + } + + return l +} + +var _ fi.HasDependencies = &NetworkLoadBalancerListener{} + +func (e *NetworkLoadBalancerListener) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +func findTargetGroupByLoadBalancerArn(cloud awsup.AWSCloud, loadBalancerArn string) (*elbv2.TargetGroup, error) { + request := &elbv2.DescribeTargetGroupsInput{ + LoadBalancerArn: aws.String(loadBalancerArn), + } + + response, err := cloud.ELBV2().DescribeTargetGroups(request) + + if err != nil { + return nil, fmt.Errorf("Error retrieving target groups for loadBalancerArn %v with err : %v", loadBalancerArn, err) + } + + if len(response.TargetGroups) != 1 { + return nil, fmt.Errorf("Wrong # of target groups returned in findTargetGroupByLoadBalancerName for name %v", loadBalancerArn) + } + + return response.TargetGroups[0], nil +} + +func findTargetGroupByLoadBalancerName(cloud awsup.AWSCloud, loadBalancerNameTag string) (*elbv2.TargetGroup, error) { + + lb, err := FindNetworkLoadBalancerByNameTag(cloud, loadBalancerNameTag) + if err != nil { + return nil, fmt.Errorf("Can't locate NLB with Name Tag %v in findTargetGroupByLoadBalancerName : %v", loadBalancerNameTag, err) + } + + if lb == nil { + return nil, nil + } + + return findTargetGroupByLoadBalancerArn(cloud, *lb.LoadBalancerArn) +} + +//The load balancer name 'api.renamenlbcluster.k8s.local' can only contain characters that are alphanumeric characters and hyphens(-)\n\tstatus code: 400, +func findNetworkLoadBalancerByLoadBalancerName(cloud awsup.AWSCloud, loadBalancerName string) (*elbv2.LoadBalancer, error) { + request := &elbv2.DescribeLoadBalancersInput{ + Names: []*string{&loadBalancerName}, + } + found, err := describeNetworkLoadBalancers(cloud, request, func(lb *elbv2.LoadBalancer) bool { + // TODO: Filter by cluster? + + if aws.StringValue(lb.LoadBalancerName) == loadBalancerName { + return true + } + + klog.Warningf("Got NLB with unexpected name: %q", aws.StringValue(lb.LoadBalancerName)) + return false + }) + + if err != nil { + if awsError, ok := err.(awserr.Error); ok { + if awsError.Code() == "LoadBalancerNotFound" { + return nil, nil + } + } + + return nil, fmt.Errorf("error listing NLBs: %v", err) + } + + if len(found) == 0 { + return nil, nil + } + + if len(found) != 1 { + return nil, fmt.Errorf("Found multiple NLBs with name %q", loadBalancerName) + } + + return found[0], nil +} + +func findNetworkLoadBalancerByAlias(cloud awsup.AWSCloud, alias *route53.AliasTarget) (*elbv2.LoadBalancer, error) { + // TODO: Any way to avoid listing all ELBs? + request := &elbv2.DescribeLoadBalancersInput{} + + dnsName := aws.StringValue(alias.DNSName) + matchDnsName := strings.TrimSuffix(dnsName, ".") + if matchDnsName == "" { + return nil, fmt.Errorf("DNSName not set on AliasTarget") + } + + matchHostedZoneId := aws.StringValue(alias.HostedZoneId) + + found, err := describeNetworkLoadBalancers(cloud, request, func(lb *elbv2.LoadBalancer) bool { + // TODO: Filter by cluster? + + if matchHostedZoneId != aws.StringValue(lb.CanonicalHostedZoneId) { + return false + } + + lbDnsName := aws.StringValue(lb.DNSName) + lbDnsName = strings.TrimSuffix(lbDnsName, ".") + return lbDnsName == matchDnsName || "dualstack."+lbDnsName == matchDnsName + }) + + if err != nil { + return nil, fmt.Errorf("error listing NLBs: %v", err) + } + + if len(found) == 0 { + return nil, nil + } + + if len(found) != 1 { + return nil, fmt.Errorf("Found multiple NLBs with DNSName %q", dnsName) + } + + return found[0], nil +} + +func FindNetworkLoadBalancerByNameTag(cloud awsup.AWSCloud, findNameTag string) (*elbv2.LoadBalancer, error) { + // TODO: Any way around this? + klog.V(2).Infof("Listing all ELBs for findNetworkLoadBalancerByNameTag") + + request := &elbv2.DescribeLoadBalancersInput{} + // ELB DescribeTags has a limit of 20 names, so we set the page size here to 20 also + request.PageSize = aws.Int64(20) + + var found []*elbv2.LoadBalancer + + var innerError error + err := cloud.ELBV2().DescribeLoadBalancersPages(request, func(p *elbv2.DescribeLoadBalancersOutput, lastPage bool) bool { + if len(p.LoadBalancers) == 0 { + return true + } + + // TODO: Filter by cluster? + + var arns []string + arnToELB := make(map[string]*elbv2.LoadBalancer) + for _, elb := range p.LoadBalancers { + arn := aws.StringValue(elb.LoadBalancerArn) + arnToELB[arn] = elb + arns = append(arns, arn) + } + + tagMap, err := describeNetworkLoadBalancerTags(cloud, arns) + if err != nil { + innerError = err + return false + } + + for loadBalancerArn, tags := range tagMap { + name, foundNameTag := awsup.FindELBV2Tag(tags, "Name") + if !foundNameTag || name != findNameTag { + continue + } + elb, _ := arnToELB[loadBalancerArn] + found = append(found, elb) + } + return true + }) + if err != nil { + return nil, fmt.Errorf("error describing LoadBalancers: %v", err) + } + if innerError != nil { + return nil, fmt.Errorf("error describing LoadBalancers: %v", innerError) + } + + if len(found) == 0 { + return nil, nil + } + + if len(found) != 1 { + return nil, fmt.Errorf("Found multiple ELBs with Name %q", findNameTag) + } + + return found[0], nil +} + +func describeNetworkLoadBalancers(cloud awsup.AWSCloud, request *elbv2.DescribeLoadBalancersInput, filter func(*elbv2.LoadBalancer) bool) ([]*elbv2.LoadBalancer, error) { + var found []*elbv2.LoadBalancer + err := cloud.ELBV2().DescribeLoadBalancersPages(request, func(p *elbv2.DescribeLoadBalancersOutput, lastPage bool) (shouldContinue bool) { + for _, lb := range p.LoadBalancers { + if filter(lb) { + found = append(found, lb) + } + } + + return true + }) + + if err != nil { + return nil, fmt.Errorf("error listing NLBs: %v", err) + } + + return found, nil +} + +func describeNetworkLoadBalancerTags(cloud awsup.AWSCloud, loadBalancerArns []string) (map[string][]*elbv2.Tag, error) { + // TODO: Filter by cluster? + + request := &elbv2.DescribeTagsInput{} + request.ResourceArns = aws.StringSlice(loadBalancerArns) + + // TODO: Cache? + klog.V(2).Infof("Querying ELBV2 api for tags for %s", loadBalancerArns) + response, err := cloud.ELBV2().DescribeTags(request) + if err != nil { + return nil, err + } + + tagMap := make(map[string][]*elbv2.Tag) + for _, tagset := range response.TagDescriptions { + tagMap[aws.StringValue(tagset.ResourceArn)] = tagset.Tags + } + return tagMap, nil +} + +func (e *NetworkLoadBalancer) Find(c *fi.Context) (*NetworkLoadBalancer, error) { + cloud := c.Cloud.(awsup.AWSCloud) + + lb, err := FindNetworkLoadBalancerByNameTag(cloud, fi.StringValue(e.Name)) + if err != nil { + return nil, err + } + if lb == nil { + return nil, nil + } + + loadBalancerArn := lb.LoadBalancerArn + var targetGroupArn *string + + actual := &NetworkLoadBalancer{} + actual.Name = e.Name + actual.Lifecycle = e.Lifecycle + actual.LoadBalancerName = lb.LoadBalancerName + actual.DNSName = lb.DNSName + actual.HostedZoneId = lb.CanonicalHostedZoneId //CanonicalHostedZoneNameID + actual.Scheme = lb.Scheme + actual.VPC = &VPC{ID: lb.VpcId} + actual.Type = lb.Type + + tagMap, err := describeNetworkLoadBalancerTags(cloud, []string{*loadBalancerArn}) + if err != nil { + return nil, err + } + actual.Tags = make(map[string]string) + for _, tag := range tagMap[*loadBalancerArn] { + actual.Tags[aws.StringValue(tag.Key)] = aws.StringValue(tag.Value) + } + + for _, az := range lb.AvailabilityZones { + actual.Subnets = append(actual.Subnets, &Subnet{ID: az.SubnetId}) + } + + /*for _, sg := range lb.SecurityGroups { + actual.SecurityGroups = append(actual.SecurityGroups, &SecurityGroup{ID: sg}) + }*/ + + { + //What happens if someone manually creates additional target groups for this LB? + request := &elbv2.DescribeTargetGroupsInput{ + LoadBalancerArn: loadBalancerArn, + } + response, err := cloud.ELBV2().DescribeTargetGroups(request) + if err != nil { + return nil, fmt.Errorf("error querying for NLB Target groups :%v", err) + } + + if len(response.TargetGroups) == 0 { + return nil, fmt.Errorf("Found no Target Groups for NLB - misconfiguration : %q", loadBalancerArn) + } + + if len(response.TargetGroups) != 1 { + return nil, fmt.Errorf("Found multiple Target groups for NLB with arn %q", loadBalancerArn) + } + + targetGroupArn = response.TargetGroups[0].TargetGroupArn + } + + { + request := &elbv2.DescribeListenersInput{ + LoadBalancerArn: loadBalancerArn, + } + response, err := cloud.ELBV2().DescribeListeners(request) + if err != nil { + return nil, fmt.Errorf("error querying for NLB listeners :%v", err) + } + + actual.Listeners = make(map[string]*NetworkLoadBalancerListener) + + for _, l := range response.Listeners { + loadBalancerPort := strconv.FormatInt(aws.Int64Value(l.Port), 10) + + actualListener := &NetworkLoadBalancerListener{} + actualListener.InstancePort = int(aws.Int64Value(l.Port)) + if len(l.Certificates) != 0 { + actualListener.SSLCertificateID = aws.StringValue(l.Certificates[0].CertificateArn) // What if there is more then one certificate, can we just grab the default certificate? we don't set it as default, we only set the one. + } + actual.Listeners[loadBalancerPort] = actualListener + } + + } + + healthcheck, err := findNLBHealthCheck(cloud, lb) + if err != nil { + return nil, err + } + actual.HealthCheck = healthcheck + + { + lbAttributes, err := findNetworkLoadBalancerAttributes(cloud, aws.StringValue(loadBalancerArn)) + if err != nil { + return nil, err + } + klog.V(4).Infof("NLB Load Balancer attributes: %+v", lbAttributes) + + actual.AccessLog = &NetworkLoadBalancerAccessLog{} + actual.DeletionProtection = &NetworkLoadBalancerDeletionProtection{} + actual.CrossZoneLoadBalancing = &NetworkLoadBalancerCrossZoneLoadBalancing{} + for _, attribute := range lbAttributes { + if attribute.Value == nil { + continue + } + switch key, value := attribute.Key, attribute.Value; *key { + case "access_logs.s3.enabled": + b, err := strconv.ParseBool(*value) + if err != nil { + return nil, err + } + actual.AccessLog.Enabled = fi.Bool(b) + case "access_logs.s3.bucket": + actual.AccessLog.S3BucketName = value + case "access_logs.s3.prefix": + actual.AccessLog.S3BucketPrefix = value + case "deletion_protection.enabled": + b, err := strconv.ParseBool(*value) + if err != nil { + return nil, err + } + actual.DeletionProtection.Enabled = fi.Bool(b) + case "load_balancing.cross_zone.enabled": + b, err := strconv.ParseBool(*value) + if err != nil { + return nil, err + } + actual.CrossZoneLoadBalancing.Enabled = fi.Bool(b) + default: + klog.V(2).Infof("unsupported key -- ignoring, %v.\n", key) + } + } + } + + { + tgAttributes, err := findTargetGroupAttributes(cloud, aws.StringValue(targetGroupArn)) + if err != nil { + return nil, err + } + klog.V(4).Infof("Target Group attributes: %+v", tgAttributes) + + actual.ProxyProtocolV2 = &TargetGroupProxyProtocolV2{} + actual.Stickiness = &TargetGroupStickiness{} + actual.DeregistationDelay = &TargetGroupDeregistrationDelay{} + for _, attribute := range tgAttributes { + if attribute.Value == nil { + continue + } + switch key, value := attribute.Key, attribute.Value; *key { + case "proxy_protocol_v2.enabled": + b, err := strconv.ParseBool(*value) + if err != nil { + return nil, err + } + actual.ProxyProtocolV2.Enabled = fi.Bool(b) + case "stickiness.type": + actual.Stickiness.Type = value + case "stickiness.enabled": + b, err := strconv.ParseBool(*value) + if err != nil { + return nil, err + } + actual.Stickiness.Enabled = fi.Bool(b) + case "deregistration_delay.timeout_seconds": + if n, err := strconv.Atoi(*value); err == nil { + m := int64(n) + actual.DeregistationDelay.TimeoutSeconds = fi.Int64(m) + } else { + return nil, err + } + + default: + klog.V(2).Infof("unsupported key -- ignoring, %v.\n", key) + } + } + } + + // Avoid spurious mismatches + if subnetSlicesEqualIgnoreOrder(actual.Subnets, e.Subnets) { + actual.Subnets = e.Subnets + } + if e.DNSName == nil { + e.DNSName = actual.DNSName + } + if e.HostedZoneId == nil { + e.HostedZoneId = actual.HostedZoneId + } + if e.LoadBalancerName == nil { + e.LoadBalancerName = actual.LoadBalancerName + } + + // We allow for the LoadBalancerName to be wrong: + // 1. We don't want to force a rename of the ELB, because that is a destructive operation + // 2. We were creating ELBs with insufficiently qualified names previously + if fi.StringValue(e.LoadBalancerName) != fi.StringValue(actual.LoadBalancerName) { + klog.V(2).Infof("Reusing existing load balancer with name: %q", aws.StringValue(actual.LoadBalancerName)) + e.LoadBalancerName = actual.LoadBalancerName + } + + // TODO: Make Normalize a standard method + actual.Normalize() + + klog.V(4).Infof("Found NLB %+v", actual) + + return actual, nil +} + +var _ fi.HasAddress = &NetworkLoadBalancer{} + +func (e *NetworkLoadBalancer) IsForAPIServer() bool { + return e.ForAPIServer +} + +func (e *NetworkLoadBalancer) FindIPAddress(context *fi.Context) (*string, error) { + cloud := context.Cloud.(awsup.AWSCloud) + + lb, err := FindNetworkLoadBalancerByNameTag(cloud, fi.StringValue(e.Name)) + if err != nil { + return nil, err + } + if lb == nil { + return nil, nil + } + + lbDnsName := fi.StringValue(lb.DNSName) + if lbDnsName == "" { + return nil, nil + } + return &lbDnsName, nil +} + +func (e *NetworkLoadBalancer) Run(c *fi.Context) error { + // TODO: Make Normalize a standard method + e.Normalize() + + return fi.DefaultDeltaRunMethod(e, c) +} + +func (e *NetworkLoadBalancer) Normalize() { + // We need to sort our arrays consistently, so we don't get spurious changes + sort.Stable(OrderSubnetsById(e.Subnets)) + sort.Stable(OrderSecurityGroupsById(e.SecurityGroups)) +} + +func (s *NetworkLoadBalancer) CheckChanges(a, e, changes *NetworkLoadBalancer) error { + if a == nil { + if fi.StringValue(e.Name) == "" { + return fi.RequiredField("Name") + } + // if len(e.SecurityGroups) == 0 { + // return fi.RequiredField("SecurityGroups") + // } + if len(e.Subnets) == 0 { + return fi.RequiredField("Subnets") + } + + if e.CrossZoneLoadBalancing != nil { + if e.CrossZoneLoadBalancing.Enabled == nil { + return fi.RequiredField("CrossZoneLoadBalancing.Enabled") + } + } + } + + return nil +} + +func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *NetworkLoadBalancer) error { + var loadBalancerName string + var loadBalancerArn string + var targetGroupArn string + + if a == nil { + if e.LoadBalancerName == nil { + return fi.RequiredField("LoadBalancerName") + } + loadBalancerName = *e.LoadBalancerName + + request := &elbv2.CreateLoadBalancerInput{} + request.Name = e.LoadBalancerName + request.Scheme = e.Scheme + request.Type = e.Type + + for _, subnet := range e.Subnets { + request.Subnets = append(request.Subnets, subnet.ID) + } + + //request.SecurityGroups = append(request.SecurityGroups, sg.ID) + + /*for _, sg := range e.SecurityGroups { + request.SecurityGroups = append(request.SecurityGroups, sg.ID) + }*/ + + { + klog.V(2).Infof("Creating NLB with Name:%q", loadBalancerName) + + response, err := t.Cloud.ELBV2().CreateLoadBalancer(request) + if err != nil { + return fmt.Errorf("error creating NLB: %v", err) + } + + if len(response.LoadBalancers) != 1 { + return fmt.Errorf("Either too many or too little NBLs were created, wanted to find %q", loadBalancerName) + } else { + lb := response.LoadBalancers[0] //TODO: how to avoid doing this + e.DNSName = lb.DNSName + e.HostedZoneId = lb.CanonicalHostedZoneId + loadBalancerArn = fi.StringValue(lb.LoadBalancerArn) + } + } + + { + first15Char := loadBalancerName[:15] + targetGroupName := first15Char + "-targets" + //TODO: GET 443/TCP FROM e.loadbalancer + request := &elbv2.CreateTargetGroupInput{ + Name: aws.String(targetGroupName), + Port: aws.Int64(443), + Protocol: aws.String("TCP"), + VpcId: e.VPC.ID, + } + + klog.V(2).Infof("Creating Target Group for NLB") + response, err := t.Cloud.ELBV2().CreateTargetGroup(request) + if err != nil { + return fmt.Errorf("Error creating target group for NLB : %v", err) + } + + targetGroupArn = *response.TargetGroups[0].TargetGroupArn + + if err := t.AddELBV2Tags(targetGroupArn, e.Tags); err != nil { + return err + } + } + + { + for loadBalancerPort, listener := range e.Listeners { + loadBalancerPortInt, err := strconv.ParseInt(loadBalancerPort, 10, 64) + if err != nil { + return fmt.Errorf("error parsing load balancer listener port: %q", loadBalancerPort) + } + awsListener := listener.mapToAWS(loadBalancerPortInt, targetGroupArn, loadBalancerArn) + + klog.V(2).Infof("Creating Listener for NLB") + _, err = t.Cloud.ELBV2().CreateListener(awsListener) + if err != nil { + return fmt.Errorf("Error creating listener for NLB: %v", err) + } + } + } + } else { + loadBalancerName = fi.StringValue(a.LoadBalancerName) + + lb, err := findNetworkLoadBalancerByLoadBalancerName(t.Cloud, loadBalancerName) + if err != nil { + return fmt.Errorf("error getting load balancer by name: %v", err) + } + + // if lb == nil { + // return fmt.Errorf("error querying nlb: %v", err) + // } + + loadBalancerArn = *lb.LoadBalancerArn + tg, err := findTargetGroupByLoadBalancerArn(t.Cloud, loadBalancerArn) + if err != nil { + return fmt.Errorf("error getting target group by lb arn %v", loadBalancerArn) + } + + targetGroupArn = *tg.TargetGroupArn + + if changes.Subnets != nil { + var expectedSubnets []string + for _, s := range e.Subnets { + expectedSubnets = append(expectedSubnets, fi.StringValue(s.ID)) + } + + var actualSubnets []string + for _, s := range a.Subnets { + actualSubnets = append(actualSubnets, fi.StringValue(s.ID)) + } + + oldSubnetIDs := slice.GetUniqueStrings(expectedSubnets, actualSubnets) + if len(oldSubnetIDs) > 0 { + /*request := &elb.DetachLoadBalancerFromSubnetsInput{} + request.SetLoadBalancerName(loadBalancerName) + request.SetSubnets(aws.StringSlice(oldSubnetIDs)) + + klog.V(2).Infof("Detaching Load Balancer from old subnets") + if _, err := t.Cloud.ELB().DetachLoadBalancerFromSubnets(request); err != nil { + return fmt.Errorf("Error detaching Load Balancer from old subnets: %v", err) + }*/ + return fmt.Errorf("Error, NLB's don't support detatching subnets, peraps we need to recreate the NLB") + } + + newSubnetIDs := slice.GetUniqueStrings(actualSubnets, expectedSubnets) + if len(newSubnetIDs) > 0 { + + request := &elbv2.SetSubnetsInput{} + request.SetLoadBalancerArn(loadBalancerArn) + request.SetSubnets(aws.StringSlice(append(actualSubnets, newSubnetIDs...))) + + klog.V(2).Infof("Attaching Load Balancer to new subnets") + if _, err := t.Cloud.ELBV2().SetSubnets(request); err != nil { + return fmt.Errorf("Error attaching Load Balancer to new subnets: %v", err) + } + } + } + + //TODO: decide if security groups should be applied to master nodes + /*if changes.SecurityGroups != nil { + request := &elb.ApplySecurityGroupsToLoadBalancerInput{} + request.LoadBalancerName = aws.String(loadBalancerName) + for _, sg := range e.SecurityGroups { + request.SecurityGroups = append(request.SecurityGroups, sg.ID) + } + + klog.V(2).Infof("Updating Load Balancer Security Groups") + if _, err := t.Cloud.ELB().ApplySecurityGroupsToLoadBalancer(request); err != nil { + return fmt.Errorf("Error updating security groups on Load Balancer: %v", err) + } + }*/ + + if changes.Listeners != nil { + + if lb != nil { + + request := &elbv2.DescribeListenersInput{ + LoadBalancerArn: lb.LoadBalancerArn, + } + response, err := t.Cloud.ELBV2().DescribeListeners(request) + if err != nil { + return fmt.Errorf("error querying for NLB listeners :%v", err) + } + + for _, l := range response.Listeners { + // deleting the listener before recreating it + _, err := t.Cloud.ELBV2().DeleteListener(&elbv2.DeleteListenerInput{ + ListenerArn: l.ListenerArn, + }) + if err != nil { + return fmt.Errorf("error deleting load balancer listener with arn = : %q : %v", l.ListenerArn, err) + } + } + } + + for loadBalancerPort, listener := range changes.Listeners { + loadBalancerPortInt, err := strconv.ParseInt(loadBalancerPort, 10, 64) + if err != nil { + return fmt.Errorf("error parsing load balancer listener port: %q", loadBalancerPort) + } + + awsListener := listener.mapToAWS(loadBalancerPortInt, targetGroupArn, loadBalancerArn) + + klog.V(2).Infof("Creating Listener for NLB") + _, err = t.Cloud.ELBV2().CreateListener(awsListener) + if err != nil { + return fmt.Errorf("Error creating listener for NLB: %v", err) + } + } + } + } + + if err := t.AddELBV2Tags(loadBalancerArn, e.Tags); err != nil { + return err + } + + //TODO: why is this used in load_balancer.go seems unecessary to remove tags right after adding them. + /*if err := t.RemoveELBV2Tags(loadBalancerArn, e.Tags); err != nil { + return err + }*/ + + if changes.HealthCheck != nil && e.HealthCheck != nil { + request := &elbv2.ModifyTargetGroupInput{ + HealthCheckPort: e.HealthCheck.Port, + TargetGroupArn: aws.String(targetGroupArn), + HealthyThresholdCount: e.HealthCheck.HealthyThreshold, + UnhealthyThresholdCount: e.HealthCheck.UnhealthyThreshold, + } + + klog.V(2).Infof("Configuring health checks on NLB %q", loadBalancerName) + _, err := t.Cloud.ELBV2().ModifyTargetGroup(request) + if err != nil { + return fmt.Errorf("error configuring health checks on NLB: %v's target group", err) + } + } + + if err := e.modifyLoadBalancerAttributes(t, a, e, changes, loadBalancerArn); err != nil { + klog.Infof("error modifying NLB attributes: %v", err) + return err + } + + if err := e.modifyTargetGroupAttributes(t, a, e, changes, targetGroupArn); err != nil { + klog.Infof("error modifying NLB Target Group attributes: %v", err) + return err + } + + return nil +} diff --git a/upup/pkg/fi/cloudup/awstasks/network_load_balancer_attachment.go b/upup/pkg/fi/cloudup/awstasks/network_load_balancer_attachment.go new file mode 100644 index 0000000000000..e44cbee8c26db --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/network_load_balancer_attachment.go @@ -0,0 +1,160 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awstasks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/autoscaling" + "github.com/aws/aws-sdk-go/service/elbv2" + "k8s.io/klog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" +) + +//go:generate fitask -type=NetworkLoadBalancerAttachment +type NetworkLoadBalancerAttachment struct { + Name *string + Lifecycle *fi.Lifecycle + + LoadBalancer *NetworkLoadBalancer + + // LoadBalancerAttachments now support ASGs or direct instances + AutoscalingGroup *AutoscalingGroup + Subnet *Subnet + + // Here be dragons.. + // This will *NOT* unmarshal.. for some reason this pointer is initiated as nil + // instead of a pointer to Instance with nil members.. + Instance *Instance +} + +func (e *NetworkLoadBalancerAttachment) Find(c *fi.Context) (*NetworkLoadBalancerAttachment, error) { + cloud := c.Cloud.(awsup.AWSCloud) + + // Instance only + if e.Instance != nil && e.AutoscalingGroup == nil { + i, err := e.Instance.Find(c) + if err != nil { + return nil, fmt.Errorf("unable to find instance: %v", err) + } + actual := &NetworkLoadBalancerAttachment{} + actual.LoadBalancer = e.LoadBalancer + actual.Instance = i + return actual, nil + // ASG only + } else if e.AutoscalingGroup != nil && e.Instance == nil { + if aws.StringValue(e.LoadBalancer.LoadBalancerName) == "" { + return nil, fmt.Errorf("LoadBalancer did not have LoadBalancerName set") + } + + g, err := findAutoscalingGroup(cloud, *e.AutoscalingGroup.Name) + if err != nil { + return nil, err + } + if g == nil { + return nil, nil + } + + tg, err := findTargetGroupByLoadBalancerName(cloud, *e.LoadBalancer.Name) + if err != nil { + return nil, err + } + if tg == nil { //should this return e.AutoscalingGroup w/ e.LoadBalancer? + return nil, nil + } + + for _, arn := range g.TargetGroupARNs { + if aws.StringValue(arn) != *tg.TargetGroupArn { + continue + } + + actual := &NetworkLoadBalancerAttachment{} + actual.LoadBalancer = e.LoadBalancer + actual.AutoscalingGroup = e.AutoscalingGroup + + // Prevent spurious changes + actual.Name = e.Name // ELB attachments don't have tags + actual.Lifecycle = e.Lifecycle + + return actual, nil + } + } else { + // Invalid request + return nil, fmt.Errorf("Must specify either an instance or an ASG") + } + + return nil, nil +} + +func (e *NetworkLoadBalancerAttachment) Run(c *fi.Context) error { + return fi.DefaultDeltaRunMethod(e, c) +} + +func (s *NetworkLoadBalancerAttachment) CheckChanges(a, e, changes *NetworkLoadBalancerAttachment) error { + if a == nil { + if e.LoadBalancer == nil { + return fi.RequiredField("LoadBalancer") + } + if e.AutoscalingGroup == nil { + return fi.RequiredField("AutoscalingGroup") + } + } + return nil +} + +func (_ *NetworkLoadBalancerAttachment) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *NetworkLoadBalancerAttachment) error { + if e.LoadBalancer == nil { + return fi.RequiredField("LoadBalancer") + } + loadBalancerName := fi.StringValue(e.LoadBalancer.LoadBalancerName) + if loadBalancerName == "" { + return fi.RequiredField("LoadBalancer.LoadBalancerName") + } + + tg, err := findTargetGroupByLoadBalancerName(t.Cloud, *e.LoadBalancer.Name) + if err != nil { + return err + } + if tg == nil { //should this return e.AutoscalingGroup w/ e.LoadBalancer? + return nil + } + + if e.AutoscalingGroup != nil && e.Instance == nil { + request := &autoscaling.AttachLoadBalancerTargetGroupsInput{} + request.TargetGroupARNs = []*string{tg.TargetGroupArn} + request.AutoScalingGroupName = e.AutoscalingGroup.Name + + klog.V(2).Infof("Attaching autoscaling group %q to NLB %q's target group", fi.StringValue(e.AutoscalingGroup.Name), loadBalancerName) + _, err = t.Cloud.Autoscaling().AttachLoadBalancerTargetGroups(request) + if err != nil { + return fmt.Errorf("error attaching autoscaling group to NLB's target group: %v", err) + } + } else if e.AutoscalingGroup == nil && e.Instance != nil { + request := &elbv2.RegisterTargetsInput{} + request.TargetGroupArn = tg.TargetGroupArn + request.Targets = []*elbv2.TargetDescription{{Id: e.Instance.ID}} + + klog.V(2).Infof("Attaching instance %q to NLB %q", fi.StringValue(e.Instance.ID), loadBalancerName) + _, err := t.Cloud.ELBV2().RegisterTargets(request) + if err != nil { + return fmt.Errorf("error attaching instance to NLB: %v", err) + } + } + return nil +} diff --git a/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_attributes.go b/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_attributes.go new file mode 100644 index 0000000000000..43c2c5849861b --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_attributes.go @@ -0,0 +1,273 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awstasks + +import ( + "fmt" + "strconv" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/elbv2" + "k8s.io/klog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" +) + +type NetworkLoadBalancerAccessLog struct { + EmitInterval *int64 + Enabled *bool //TODO: change to S3Enabled + S3BucketName *string //TODO: change to S3Bucket + S3BucketPrefix *string //TODO: change to S3Prefix +} + +func (_ *NetworkLoadBalancerAccessLog) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +//type LoadBalancerAdditionalAttribute struct { +// Key *string +// Value *string +//} +// +//func (_ *LoadBalancerAdditionalAttribute) GetDependencies(tasks map[string]fi.Task) []fi.Task { +// return nil +//} + +type TargetGroupProxyProtocolV2 struct { + Enabled *bool +} + +func (_ *TargetGroupProxyProtocolV2) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +type TargetGroupStickiness struct { + Enabled *bool + Type *string +} + +func (_ *TargetGroupStickiness) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +type TargetGroupDeregistrationDelay struct { + TimeoutSeconds *int64 +} + +func (_ *TargetGroupDeregistrationDelay) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +type NetworkLoadBalancerCrossZoneLoadBalancing struct { + Enabled *bool +} + +func (_ *NetworkLoadBalancerCrossZoneLoadBalancing) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +type NetworkLoadBalancerDeletionProtection struct { + Enabled *bool +} + +func (_ *NetworkLoadBalancerDeletionProtection) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +func findNetworkLoadBalancerAttributes(cloud awsup.AWSCloud, LoadBalancerArn string) ([]*elbv2.LoadBalancerAttribute, error) { + + request := &elbv2.DescribeLoadBalancerAttributesInput{ + LoadBalancerArn: aws.String(LoadBalancerArn), + } + + response, err := cloud.ELBV2().DescribeLoadBalancerAttributes(request) + if err != nil { + return nil, err + } + if response == nil { + return nil, nil + } + + //we get back an array of attributes + + /* + Key *string `type:"string"` + Value *string `type:"string"` + */ + + return response.Attributes, nil +} + +func findTargetGroupAttributes(cloud awsup.AWSCloud, TargetGroupArn string) ([]*elbv2.TargetGroupAttribute, error) { + + request := &elbv2.DescribeTargetGroupAttributesInput{ + TargetGroupArn: aws.String(TargetGroupArn), + } + + response, err := cloud.ELBV2().DescribeTargetGroupAttributes(request) + if err != nil { + return nil, err + } + if response == nil { + return nil, nil + } + + //we get back an array of attributes + + /* + Key *string `type:"string"` + Value *string `type:"string"` + */ + + return response.Attributes, nil +} + +func (_ *NetworkLoadBalancer) modifyLoadBalancerAttributes(t *awsup.AWSAPITarget, a, e, changes *NetworkLoadBalancer, loadBalancerArn string) error { + if changes.AccessLog == nil && + changes.DeletionProtection == nil && + changes.CrossZoneLoadBalancing == nil { + klog.V(4).Infof("No LoadBalancerAttribute changes; skipping update") + return nil + } + + loadBalancerName := fi.StringValue(e.LoadBalancerName) + + request := &elbv2.ModifyLoadBalancerAttributesInput{ + LoadBalancerArn: aws.String(loadBalancerArn), + } + + var attributes []*elbv2.LoadBalancerAttribute + + attribute := &elbv2.LoadBalancerAttribute{} + attribute.Key = aws.String("access_logs.s3.enabled") + if e.AccessLog == nil || e.AccessLog.Enabled == nil { + attribute.Value = aws.String("false") + } else { + attribute.Value = aws.String(strconv.FormatBool(aws.BoolValue(e.AccessLog.Enabled))) + } + attributes = append(attributes, attribute) + + /*if *e.AccessLog.Enabled { //TODO: Should we capture these? -- These are not settable from spec. + attribute = &elbv2.LoadBalancerAttribute{} + attribute.Key = aws.String("access_logs.s3.bucket") + if e.AccessLog == nil || e.AccessLog.S3BucketName == nil { + attribute.Value = aws.String("") //TOOD: ValidationError: The value of 'access_logs.s3.bucket' cannot be empty + } else { + attribute.Value = e.AccessLog.S3BucketName + attributes = append(attributes, attribute) + } + + attribute = &elbv2.LoadBalancerAttribute{} + attribute.Key = aws.String("access_logs.s3.prefix") + if e.AccessLog == nil || e.AccessLog.S3BucketPrefix == nil { + attribute.Value = aws.String("") //TODO: ValidationError: The value of 'access_logs.s3.bucket' cannot be empty + } else { + attribute.Value = e.AccessLog.S3BucketPrefix + attributes = append(attributes, attribute) + } + }*/ + + attribute = &elbv2.LoadBalancerAttribute{} + attribute.Key = aws.String("deletion_protection.enabled") + if e.DeletionProtection == nil || e.DeletionProtection.Enabled == nil { + attribute.Value = aws.String("false") + } else { + attribute.Value = aws.String(strconv.FormatBool(aws.BoolValue(e.DeletionProtection.Enabled))) + } + attributes = append(attributes, attribute) + + attribute = &elbv2.LoadBalancerAttribute{} + attribute.Key = aws.String("load_balancing.cross_zone.enabled") + if e.CrossZoneLoadBalancing == nil || e.CrossZoneLoadBalancing.Enabled == nil { + attribute.Value = aws.String("false") + } else { + attribute.Value = aws.String(strconv.FormatBool(aws.BoolValue(e.CrossZoneLoadBalancing.Enabled))) + } + attributes = append(attributes, attribute) + + request.Attributes = attributes + + klog.V(2).Infof("Configuring NLB attributes for NLB %q", loadBalancerName) + + response, err := t.Cloud.ELBV2().ModifyLoadBalancerAttributes(request) + if err != nil { + return fmt.Errorf("error configuring NLB attributes for NLB %q: %v", loadBalancerName, err) + } + + klog.V(4).Infof("modified NLB attributes for NLB %q, response %+v", loadBalancerName, response) + + return nil +} + +func (_ *NetworkLoadBalancer) modifyTargetGroupAttributes(t *awsup.AWSAPITarget, a, e, changes *NetworkLoadBalancer, targetGroupArn string) error { + if changes.ProxyProtocolV2 == nil && + changes.Stickiness == nil && + changes.DeregistationDelay == nil { + klog.V(4).Infof("No TargetGroup changes; skipping update") + return nil + } + + loadBalancerName := fi.StringValue(e.LoadBalancerName) + request := &elbv2.ModifyTargetGroupAttributesInput{ + TargetGroupArn: aws.String(targetGroupArn), + } + + var attributes []*elbv2.TargetGroupAttribute + + attribute := &elbv2.TargetGroupAttribute{} + attribute.Key = aws.String("deregistration_delay.timeout_seconds") + if e.DeregistationDelay == nil || e.DeregistationDelay.TimeoutSeconds == nil { + attribute.Value = aws.String("300") + } else { + attribute.Value = aws.String(strconv.Itoa(int(*e.DeregistationDelay.TimeoutSeconds))) + } + attributes = append(attributes, attribute) + + attribute = &elbv2.TargetGroupAttribute{} + attribute.Key = aws.String("stickiness.enabled") + if e.Stickiness == nil || e.Stickiness.Enabled == nil { + attribute.Value = aws.String("false") + } else { + attribute.Value = aws.String(strconv.FormatBool(aws.BoolValue(e.Stickiness.Enabled))) + } + attributes = append(attributes, attribute) + + attribute = &elbv2.TargetGroupAttribute{} + attribute.Key = aws.String("stickiness.type ") + attribute.Value = aws.String("source_ip") //TODO: can we set this even if enabled = false? + attributes = append(attributes, attribute) + + attribute = &elbv2.TargetGroupAttribute{} + attribute.Key = aws.String("proxy_protocol_v2.enabled") + if e.ProxyProtocolV2 == nil || e.ProxyProtocolV2.Enabled == nil { + attribute.Value = aws.String("false") + } else { + attribute.Value = aws.String(strconv.FormatBool(aws.BoolValue(e.ProxyProtocolV2.Enabled))) + } + attributes = append(attributes, attribute) + + request.Attributes = attributes + + responseTG, err := t.Cloud.ELBV2().ModifyTargetGroupAttributes(request) + if err != nil { + return fmt.Errorf("error configuring NLB target group attributes for NLB %q: %v", loadBalancerName, err) + } + + klog.V(4).Infof("modified NLB target group attributes for NLB %q, response %+v", loadBalancerName, responseTG) + + return nil +} diff --git a/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_fitask.go b/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_fitask.go new file mode 100644 index 0000000000000..48e05045d3a9f --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_fitask.go @@ -0,0 +1,75 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by ""fitask" -type=NetworkLoadBalancer"; DO NOT EDIT + +package awstasks + +import ( + "encoding/json" + + "k8s.io/kops/upup/pkg/fi" +) + +// NetworkLoadBalancer + +// JSON marshaling boilerplate +type realNetworkLoadBalancer NetworkLoadBalancer + +// UnmarshalJSON implements conversion to JSON, supporting an alternate specification of the object as a string +func (o *NetworkLoadBalancer) UnmarshalJSON(data []byte) error { + var jsonName string + if err := json.Unmarshal(data, &jsonName); err == nil { + o.Name = &jsonName + return nil + } + + var r realNetworkLoadBalancer + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *o = NetworkLoadBalancer(r) + return nil +} + +var _ fi.HasLifecycle = &NetworkLoadBalancer{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *NetworkLoadBalancer) GetLifecycle() *fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *NetworkLoadBalancer) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = &lifecycle +} + +var _ fi.HasName = &NetworkLoadBalancer{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *NetworkLoadBalancer) GetName() *string { + return o.Name +} + +// SetName sets the Name of the object, implementing fi.SetName +func (o *NetworkLoadBalancer) SetName(name string) { + o.Name = &name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *NetworkLoadBalancer) String() string { + return fi.TaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_healthchecks.go b/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_healthchecks.go new file mode 100644 index 0000000000000..16563d9e26f69 --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/networkloadbalancer_healthchecks.go @@ -0,0 +1,76 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package awstasks + +import ( + "fmt" + + "github.com/aws/aws-sdk-go/service/elbv2" + "k8s.io/klog" + "k8s.io/kops/upup/pkg/fi" + "k8s.io/kops/upup/pkg/fi/cloudup/awsup" +) + +type NetworkLoadBalancerHealthCheck struct { + Target *string + + HealthyThreshold *int64 + UnhealthyThreshold *int64 + + Interval *int64 + Timeout *int64 + + Port *string + Protocol *string +} + +var _ fi.HasDependencies = &LoadBalancerListener{} + +func (e *NetworkLoadBalancerHealthCheck) GetDependencies(tasks map[string]fi.Task) []fi.Task { + return nil +} + +func findNLBHealthCheck(cloud awsup.AWSCloud, lb *elbv2.LoadBalancer) (*NetworkLoadBalancerHealthCheck, error) { + + klog.V(2).Infof("Requesting Target Group for NLB with Name:%q", lb.LoadBalancerName) + request := &elbv2.DescribeTargetGroupsInput{ + LoadBalancerArn: lb.LoadBalancerArn, + } + response, err := cloud.ELBV2().DescribeTargetGroups(request) + if err != nil { + return nil, fmt.Errorf("error querying for target groups associated with LoadBalancerArn:%+v", lb.LoadBalancerArn) + } + + if len(response.TargetGroups) != 1 { + return nil, fmt.Errorf("error wrong # of target groups returned while querying for target groups associated with LoadBalancerArn:%+v", lb.LoadBalancerArn) + } + + tg := response.TargetGroups[0] + + if lb == nil || tg == nil { + return nil, nil + } + + actual := &NetworkLoadBalancerHealthCheck{} + if tg != nil { + actual.UnhealthyThreshold = tg.UnhealthyThresholdCount + actual.HealthyThreshold = tg.HealthyThresholdCount + actual.Port = tg.HealthCheckPort + } + + return actual, nil +} diff --git a/upup/pkg/fi/cloudup/awstasks/networkloadbalancerattachment_fitask.go b/upup/pkg/fi/cloudup/awstasks/networkloadbalancerattachment_fitask.go new file mode 100644 index 0000000000000..e1abc8f0bccec --- /dev/null +++ b/upup/pkg/fi/cloudup/awstasks/networkloadbalancerattachment_fitask.go @@ -0,0 +1,75 @@ +/* +Copyright 2019 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by ""fitask" -type=NetworkLoadBalancerAttachment"; DO NOT EDIT + +package awstasks + +import ( + "encoding/json" + + "k8s.io/kops/upup/pkg/fi" +) + +// NetworkLoadBalancerAttachment + +// JSON marshaling boilerplate +type realNetworkLoadBalancerAttachment NetworkLoadBalancerAttachment + +// UnmarshalJSON implements conversion to JSON, supporting an alternate specification of the object as a string +func (o *NetworkLoadBalancerAttachment) UnmarshalJSON(data []byte) error { + var jsonName string + if err := json.Unmarshal(data, &jsonName); err == nil { + o.Name = &jsonName + return nil + } + + var r realNetworkLoadBalancerAttachment + if err := json.Unmarshal(data, &r); err != nil { + return err + } + *o = NetworkLoadBalancerAttachment(r) + return nil +} + +var _ fi.HasLifecycle = &NetworkLoadBalancerAttachment{} + +// GetLifecycle returns the Lifecycle of the object, implementing fi.HasLifecycle +func (o *NetworkLoadBalancerAttachment) GetLifecycle() *fi.Lifecycle { + return o.Lifecycle +} + +// SetLifecycle sets the Lifecycle of the object, implementing fi.SetLifecycle +func (o *NetworkLoadBalancerAttachment) SetLifecycle(lifecycle fi.Lifecycle) { + o.Lifecycle = &lifecycle +} + +var _ fi.HasName = &NetworkLoadBalancerAttachment{} + +// GetName returns the Name of the object, implementing fi.HasName +func (o *NetworkLoadBalancerAttachment) GetName() *string { + return o.Name +} + +// SetName sets the Name of the object, implementing fi.SetName +func (o *NetworkLoadBalancerAttachment) SetName(name string) { + o.Name = &name +} + +// String is the stringer function for the task, producing readable output using fi.TaskAsString +func (o *NetworkLoadBalancerAttachment) String() string { + return fi.TaskAsString(o) +} diff --git a/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go b/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go index 801f85834d08c..6c79142e78ee6 100644 --- a/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go +++ b/upup/pkg/fi/cloudup/awstasks/securitygrouprule.go @@ -47,6 +47,7 @@ type SecurityGroupRule struct { SourceGroup *SecurityGroup Egress *bool + VPC *VPC } func (e *SecurityGroupRule) Find(c *fi.Context) (*SecurityGroupRule, error) { @@ -104,6 +105,7 @@ func (e *SecurityGroupRule) Find(c *fi.Context) (*SecurityGroupRule, error) { ToPort: foundRule.ToPort, Protocol: foundRule.IpProtocol, Egress: e.Egress, + VPC: e.VPC, } if aws.StringValue(actual.Protocol) == "-1" { @@ -250,9 +252,13 @@ func (_ *SecurityGroupRule) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Secu }, } } else { + CIDR := e.CIDR + if e.VPC != nil { //ALLOW security group to use vpc cidr for network load balancer. + CIDR = e.VPC.CIDR + } // Default to 0.0.0.0/0 ? ipPermission.IpRanges = []*ec2.IpRange{ - {CidrIp: e.CIDR}, + {CidrIp: CIDR}, } } diff --git a/upup/pkg/fi/cloudup/awsup/aws_apitarget.go b/upup/pkg/fi/cloudup/awsup/aws_apitarget.go index 02082ea220c40..22536f9b55644 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_apitarget.go +++ b/upup/pkg/fi/cloudup/awsup/aws_apitarget.go @@ -53,6 +53,57 @@ func (t *AWSAPITarget) AddAWSTags(id string, expected map[string]string) error { func (t *AWSAPITarget) DeleteTags(id string, tags map[string]string) error { return t.Cloud.DeleteTags(id, tags) } +func (t *AWSAPITarget) AddELBV2Tags(ResourceArn string, expected map[string]string) error { + actual, err := t.Cloud.GetELBV2Tags(ResourceArn) + if err != nil { + return fmt.Errorf("unexpected error fetching tags for resource: %v", err) + } + + missing := map[string]string{} + for k, v := range expected { + actualValue, found := actual[k] + if found && actualValue == v { + continue + } + missing[k] = v + } + + if len(missing) != 0 { + klog.V(4).Infof("adding tags to %q: %v", ResourceArn, missing) + err := t.Cloud.CreateELBV2Tags(ResourceArn, missing) + if err != nil { + return fmt.Errorf("error adding tags to ELBV2 %q: %v", ResourceArn, err) + } + } + + return nil +} + +func (t *AWSAPITarget) RemoveELBV2Tags(ResourceArn string, expected map[string]string) error { + actual, err := t.Cloud.GetELBV2Tags(ResourceArn) + if err != nil { + return fmt.Errorf("unexpected error fetching tags for resource: %v", err) + } + + extra := map[string]string{} + for k, v := range actual { + expectedValue, found := expected[k] + if found && expectedValue == v { + continue + } + extra[k] = v + } + + if len(extra) != 0 { + klog.V(4).Infof("removing tags from %q: %v", ResourceArn, extra) + err := t.Cloud.RemoveELBV2Tags(ResourceArn, extra) + if err != nil { + return fmt.Errorf("error removing tags from ELBV2 %q: %v", ResourceArn, err) + } + } + + return nil +} func (t *AWSAPITarget) AddELBTags(loadBalancerName string, expected map[string]string) error { actual, err := t.Cloud.GetELBTags(loadBalancerName) diff --git a/upup/pkg/fi/cloudup/awsup/aws_cloud.go b/upup/pkg/fi/cloudup/awsup/aws_cloud.go index ff18398bd17e7..50a38eff48333 100644 --- a/upup/pkg/fi/cloudup/awsup/aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/aws_cloud.go @@ -128,11 +128,14 @@ type AWSCloud interface { AddAWSTags(id string, expected map[string]string) error GetELBTags(loadBalancerName string) (map[string]string, error) + GetELBV2Tags(ResourceArn string) (map[string]string, error) // CreateELBTags will add tags to the specified loadBalancer, retrying up to MaxCreateTagsAttempts times if it hits an eventual-consistency type error CreateELBTags(loadBalancerName string, tags map[string]string) error + CreateELBV2Tags(ResourceArn string, tags map[string]string) error // RemoveELBTags will remove tags from the specified loadBalancer, retrying up to MaxCreateTagsAttempts times if it hits an eventual-consistency type error RemoveELBTags(loadBalancerName string, tags map[string]string) error + RemoveELBV2Tags(ResourceArn string, tags map[string]string) error // DeleteTags will delete tags from the specified resource, retrying up to MaxCreateTagsAttempts times if it hits an eventual-consistency type error DeleteTags(id string, tags map[string]string) error @@ -1194,6 +1197,32 @@ func removeELBTags(c AWSCloud, loadBalancerName string, tags map[string]string) return nil } +func (c *awsCloudImplementation) RemoveELBV2Tags(ResourceArn string, tags map[string]string) error { + return removeELBTags(c, ResourceArn, tags) +} + +func removeELBV2Tags(c AWSCloud, ResourceArn string, tags map[string]string) error { + if len(tags) == 0 { + return nil + } + + elbTagKeysOnly := []*string{} + for k := range tags { + elbTagKeysOnly = append(elbTagKeysOnly, aws.String(k)) + } + + request := &elbv2.RemoveTagsInput{ + TagKeys: elbTagKeysOnly, + ResourceArns: []*string{&ResourceArn}, + } + + _, err := c.ELBV2().RemoveTags(request) + if err != nil { + return fmt.Errorf("error creating tags on %v: %v", ResourceArn, err) + } + + return nil +} func (c *awsCloudImplementation) GetELBV2Tags(ResourceArn string) (map[string]string, error) { return getELBV2Tags(c, ResourceArn) diff --git a/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go b/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go index 220d55a4ff7f3..adfbd57560c2b 100644 --- a/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go +++ b/upup/pkg/fi/cloudup/awsup/mock_aws_cloud.go @@ -178,6 +178,10 @@ func (c *MockAWSCloud) CreateELBV2Tags(ResourceArn string, tags map[string]strin return createELBV2Tags(c, ResourceArn, tags) } +func (c *MockAWSCloud) RemoveELBV2Tags(ResourceArn string, tags map[string]string) error { + return removeELBV2Tags(c, ResourceArn, tags) +} + func (c *MockAWSCloud) DescribeInstance(instanceID string) (*ec2.Instance, error) { return nil, fmt.Errorf("MockAWSCloud DescribeInstance not implemented") } diff --git a/upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go b/upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go index f0b763c3187c8..7750a722bf4c4 100644 --- a/upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go +++ b/upup/pkg/fi/cloudup/spotinsttasks/elastigroup.go @@ -334,12 +334,29 @@ func (e *Elastigroup) Find(c *fi.Context) (*Elastigroup, error) { if e.LoadBalancer != nil && actual.LoadBalancer != nil && fi.StringValue(actual.LoadBalancer.Name) != fi.StringValue(e.LoadBalancer.Name) { - elb, err := awstasks.FindLoadBalancerByNameTag(cloud, fi.StringValue(e.LoadBalancer.Name)) - if err != nil { - return nil, err + + if true { + panic("this code not ready for NLB") } - if fi.StringValue(elb.LoadBalancerName) == fi.StringValue(lbs[0].Name) { - actual.LoadBalancer = e.LoadBalancer + useNLB := true + if useNLB { + elb, err := awstasks.FindNetworkLoadBalancerByNameTag(cloud, fi.StringValue(e.LoadBalancer.Name)) + if err != nil { + return nil, err + } + if fi.StringValue(elb.LoadBalancerName) == fi.StringValue(lbs[0].Name) { + actual.LoadBalancer = e.LoadBalancer + } + } + useELB := true + if useELB { + elb, err := awstasks.FindLoadBalancerByNameTag(cloud, fi.StringValue(e.LoadBalancer.Name)) + if err != nil { + return nil, err + } + if fi.StringValue(elb.LoadBalancerName) == fi.StringValue(lbs[0].Name) { + actual.LoadBalancer = e.LoadBalancer + } } } } @@ -612,6 +629,11 @@ func (_ *Elastigroup) create(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e // Load balancer. { if e.LoadBalancer != nil { + + if true { + panic("this code is not ready for NLB") + } + elb, err := awstasks.FindLoadBalancerByNameTag(cloud, fi.StringValue(e.LoadBalancer.Name)) if err != nil { return err @@ -1101,6 +1123,11 @@ func (_ *Elastigroup) update(cloud awsup.AWSCloud, a, e, changes *Elastigroup) e // Load balancer. { if changes.LoadBalancer != nil { + + if true { + panic("this code is not ready for NLB") + } + elb, err := awstasks.FindLoadBalancerByNameTag(cloud, fi.StringValue(e.LoadBalancer.Name)) if err != nil { return err diff --git a/upup/pkg/fi/default_methods.go b/upup/pkg/fi/default_methods.go index 9779645fb9e5a..0537febb15f7e 100644 --- a/upup/pkg/fi/default_methods.go +++ b/upup/pkg/fi/default_methods.go @@ -26,6 +26,7 @@ import ( // DefaultDeltaRunMethod implements the standard change-based run procedure: // find the existing item; compare properties; call render with (actual, expected, changes) func DefaultDeltaRunMethod(e Task, c *Context) error { + var a Task var err error