diff --git a/api/v1beta2/network_types.go b/api/v1beta2/network_types.go index d487183025..ea5d627a61 100644 --- a/api/v1beta2/network_types.go +++ b/api/v1beta2/network_types.go @@ -27,6 +27,16 @@ const ( DefaultAPIServerPort = 6443 // DefaultAPIServerPortString defines the API server port as a string for convenience. DefaultAPIServerPortString = "6443" + // DefaultAPIServerHealthCheckPath the API server health check path. + DefaultAPIServerHealthCheckPath = "/readyz" + // DefaultAPIServerHealthCheckIntervalSec the API server health check interval in seconds. + DefaultAPIServerHealthCheckIntervalSec = 10 + // DefaultAPIServerHealthCheckTimeoutSec the API server health check timeout in seconds. + DefaultAPIServerHealthCheckTimeoutSec = 5 + // DefaultAPIServerHealthThresholdCount the API server health check threshold count. + DefaultAPIServerHealthThresholdCount = 5 + // DefaultAPIServerUnhealthThresholdCount the API server unhealthy check threshold count. + DefaultAPIServerUnhealthThresholdCount = 3 ) // NetworkStatus encapsulates AWS networking resources. diff --git a/pkg/cloud/services/elb/loadbalancer.go b/pkg/cloud/services/elb/loadbalancer.go index 592ef4370c..80ad95456c 100644 --- a/pkg/cloud/services/elb/loadbalancer.go +++ b/pkg/cloud/services/elb/loadbalancer.go @@ -172,6 +172,23 @@ func (s *Service) getAPIServerLBSpec(elbName string, lbSpec *infrav1.AWSLoadBala scheme = *lbSpec.Scheme } + // The default API health check is TCP, allowing customization to HTTP or HTTPS when HealthCheckProtocol is set. + apiHealthCheckProtocol := infrav1.ELBProtocolTCP + if lbSpec != nil && lbSpec.HealthCheckProtocol != nil { + s.scope.Trace("Found API health check protocol override in the Load Balancer spec, applying it to the API Target Group", "api-server-elb", lbSpec.HealthCheckProtocol) + apiHealthCheckProtocol = *lbSpec.HealthCheckProtocol + } + apiHealthCheck := &infrav1.TargetGroupHealthCheck{ + Protocol: aws.String(apiHealthCheckProtocol.String()), + Port: aws.String(infrav1.DefaultAPIServerPortString), + Path: nil, + IntervalSeconds: aws.Int64(infrav1.DefaultAPIServerHealthCheckIntervalSec), + TimeoutSeconds: aws.Int64(infrav1.DefaultAPIServerHealthCheckTimeoutSec), + ThresholdCount: aws.Int64(infrav1.DefaultAPIServerHealthThresholdCount), + } + if apiHealthCheckProtocol == infrav1.ELBProtocolHTTP || apiHealthCheckProtocol == infrav1.ELBProtocolHTTPS { + apiHealthCheck.Path = aws.String(infrav1.DefaultAPIServerHealthCheckPath) + } res := &infrav1.LoadBalancer{ Name: elbName, Scheme: scheme, @@ -181,14 +198,11 @@ func (s *Service) getAPIServerLBSpec(elbName string, lbSpec *infrav1.AWSLoadBala Protocol: infrav1.ELBProtocolTCP, Port: infrav1.DefaultAPIServerPort, TargetGroup: infrav1.TargetGroupSpec{ - Name: fmt.Sprintf("apiserver-target-%d", time.Now().Unix()), - Port: infrav1.DefaultAPIServerPort, - Protocol: infrav1.ELBProtocolTCP, - VpcID: s.scope.VPC().ID, - HealthCheck: &infrav1.TargetGroupHealthCheck{ - Protocol: aws.String(string(infrav1.ELBProtocolTCP)), - Port: aws.String(infrav1.DefaultAPIServerPortString), - }, + Name: fmt.Sprintf("apiserver-target-%d", time.Now().Unix()), + Port: infrav1.DefaultAPIServerPort, + Protocol: infrav1.ELBProtocolTCP, + VpcID: s.scope.VPC().ID, + HealthCheck: apiHealthCheck, }, }, }, @@ -321,6 +335,19 @@ func (s *Service) createLB(spec *infrav1.LoadBalancer, lbSpec *infrav1.AWSLoadBa targetGroupInput.HealthCheckEnabled = aws.Bool(true) targetGroupInput.HealthCheckProtocol = ln.TargetGroup.HealthCheck.Protocol targetGroupInput.HealthCheckPort = ln.TargetGroup.HealthCheck.Port + targetGroupInput.UnhealthyThresholdCount = aws.Int64(infrav1.DefaultAPIServerUnhealthThresholdCount) + if ln.TargetGroup.HealthCheck.Path != nil { + targetGroupInput.HealthCheckPath = ln.TargetGroup.HealthCheck.Path + } + if ln.TargetGroup.HealthCheck.IntervalSeconds != nil { + targetGroupInput.HealthCheckIntervalSeconds = ln.TargetGroup.HealthCheck.IntervalSeconds + } + if ln.TargetGroup.HealthCheck.TimeoutSeconds != nil { + targetGroupInput.HealthCheckTimeoutSeconds = ln.TargetGroup.HealthCheck.TimeoutSeconds + } + if ln.TargetGroup.HealthCheck.ThresholdCount != nil { + targetGroupInput.HealthyThresholdCount = ln.TargetGroup.HealthCheck.ThresholdCount + } } s.scope.Debug("creating target group", "group", targetGroupInput, "listener", ln) group, err := s.ELBV2Client.CreateTargetGroup(targetGroupInput) @@ -1007,10 +1034,10 @@ func (s *Service) getAPIServerClassicELBSpec(elbName string) (*infrav1.LoadBalan }, HealthCheck: &infrav1.ClassicELBHealthCheck{ Target: s.getHealthCheckTarget(), - Interval: 10 * time.Second, - Timeout: 5 * time.Second, - HealthyThreshold: 5, - UnhealthyThreshold: 3, + Interval: infrav1.DefaultAPIServerHealthCheckIntervalSec * time.Second, + Timeout: infrav1.DefaultAPIServerHealthCheckTimeoutSec * time.Second, + HealthyThreshold: infrav1.DefaultAPIServerHealthThresholdCount, + UnhealthyThreshold: infrav1.DefaultAPIServerUnhealthThresholdCount, }, SecurityGroupIDs: securityGroupIDs, ClassicElbAttributes: infrav1.ClassicELBAttributes{ @@ -1506,7 +1533,7 @@ func (s *Service) getHealthCheckTarget() string { if controlPlaneELB != nil && controlPlaneELB.HealthCheckProtocol != nil { protocol = controlPlaneELB.HealthCheckProtocol if protocol.String() == infrav1.ELBProtocolHTTP.String() || protocol.String() == infrav1.ELBProtocolHTTPS.String() { - return fmt.Sprintf("%v:%d/readyz", protocol, infrav1.DefaultAPIServerPort) + return fmt.Sprintf("%v:%d%s", protocol, infrav1.DefaultAPIServerPort, infrav1.DefaultAPIServerHealthCheckPath) } } return fmt.Sprintf("%v:%d", protocol, infrav1.DefaultAPIServerPort) diff --git a/pkg/cloud/services/elb/loadbalancer_test.go b/pkg/cloud/services/elb/loadbalancer_test.go index 593cbc3625..267e7dc3f8 100644 --- a/pkg/cloud/services/elb/loadbalancer_test.go +++ b/pkg/cloud/services/elb/loadbalancer_test.go @@ -43,6 +43,17 @@ import ( "sigs.k8s.io/cluster-api/util/conditions" ) +var stubInfraV1TargetGroupSpecAPI = infrav1.TargetGroupSpec{ + Name: "name", + Port: infrav1.DefaultAPIServerPort, + Protocol: "TCP", + HealthCheck: &infrav1.TargetGroupHealthCheck{ + IntervalSeconds: aws.Int64(10), + TimeoutSeconds: aws.Int64(5), + ThresholdCount: aws.Int64(5), + }, +} + func TestELBName(t *testing.T) { tests := []struct { name string @@ -842,7 +853,7 @@ func TestRegisterInstanceWithAPIServerNLB(t *testing.T) { TargetGroups: []*elbv2.TargetGroup{ { HealthCheckEnabled: aws.Bool(true), - HealthCheckPort: aws.String("infrav1.DefaultAPIServerPort"), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), HealthCheckProtocol: aws.String("TCP"), LoadBalancerArns: aws.StringSlice([]string{elbArn}), Port: aws.Int64(infrav1.DefaultAPIServerPort), @@ -945,7 +956,7 @@ func TestRegisterInstanceWithAPIServerNLB(t *testing.T) { TargetGroups: []*elbv2.TargetGroup{ { HealthCheckEnabled: aws.Bool(true), - HealthCheckPort: aws.String("infrav1.DefaultAPIServerPort"), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), HealthCheckProtocol: aws.String("TCP"), LoadBalancerArns: aws.StringSlice([]string{elbArn}), Port: aws.Int64(infrav1.DefaultAPIServerPort), @@ -1189,13 +1200,14 @@ func TestCreateNLB(t *testing.T) { }, }, nil) m.CreateTargetGroup(gomock.Eq(&elbv2.CreateTargetGroupInput{ - HealthCheckEnabled: aws.Bool(true), - HealthCheckPort: aws.String("infrav1.DefaultAPIServerPort"), - HealthCheckProtocol: aws.String("tcp"), - Name: aws.String("name"), - Port: aws.Int64(infrav1.DefaultAPIServerPort), - Protocol: aws.String("TCP"), - VpcId: aws.String(vpcID), + HealthCheckEnabled: aws.Bool(true), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), + HealthCheckProtocol: aws.String("tcp"), + UnhealthyThresholdCount: aws.Int64(infrav1.DefaultAPIServerUnhealthThresholdCount), + Name: aws.String("name"), + Port: aws.Int64(infrav1.DefaultAPIServerPort), + Protocol: aws.String("TCP"), + VpcId: aws.String(vpcID), Tags: []*elbv2.Tag{ { Key: aws.String("test"), @@ -1291,14 +1303,15 @@ func TestCreateNLB(t *testing.T) { }, }, nil) m.CreateTargetGroup(gomock.Eq(&elbv2.CreateTargetGroupInput{ - HealthCheckEnabled: aws.Bool(true), - HealthCheckPort: aws.String("infrav1.DefaultAPIServerPort"), - HealthCheckProtocol: aws.String("tcp"), - Name: aws.String("name"), - Port: aws.Int64(infrav1.DefaultAPIServerPort), - Protocol: aws.String("TCP"), - VpcId: aws.String(vpcID), - IpAddressType: aws.String("ipv6"), + HealthCheckEnabled: aws.Bool(true), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), + HealthCheckProtocol: aws.String("tcp"), + UnhealthyThresholdCount: aws.Int64(infrav1.DefaultAPIServerUnhealthThresholdCount), + Name: aws.String("name"), + Port: aws.Int64(infrav1.DefaultAPIServerPort), + Protocol: aws.String("TCP"), + VpcId: aws.String(vpcID), + IpAddressType: aws.String("ipv6"), Tags: []*elbv2.Tag{ { Key: aws.String("test"), @@ -1529,13 +1542,14 @@ func TestCreateNLB(t *testing.T) { }, }, nil) m.CreateTargetGroup(gomock.Eq(&elbv2.CreateTargetGroupInput{ - HealthCheckEnabled: aws.Bool(true), - HealthCheckPort: aws.String("infrav1.DefaultAPIServerPort"), - HealthCheckProtocol: aws.String("tcp"), - Name: aws.String("name"), - Port: aws.Int64(infrav1.DefaultAPIServerPort), - Protocol: aws.String("TCP"), - VpcId: aws.String(vpcID), + HealthCheckEnabled: aws.Bool(true), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), + HealthCheckProtocol: aws.String("tcp"), + UnhealthyThresholdCount: aws.Int64(infrav1.DefaultAPIServerUnhealthThresholdCount), + Name: aws.String("name"), + Port: aws.Int64(infrav1.DefaultAPIServerPort), + Protocol: aws.String("TCP"), + VpcId: aws.String(vpcID), Tags: []*elbv2.Tag{ { Key: aws.String("test"), @@ -1619,13 +1633,14 @@ func TestCreateNLB(t *testing.T) { }, }, nil) m.CreateTargetGroup(gomock.Eq(&elbv2.CreateTargetGroupInput{ - HealthCheckEnabled: aws.Bool(true), - HealthCheckPort: aws.String("infrav1.DefaultAPIServerPort"), - HealthCheckProtocol: aws.String("tcp"), - Name: aws.String("name"), - Port: aws.Int64(infrav1.DefaultAPIServerPort), - Protocol: aws.String("TCP"), - VpcId: aws.String(vpcID), + HealthCheckEnabled: aws.Bool(true), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), + HealthCheckProtocol: aws.String("tcp"), + UnhealthyThresholdCount: aws.Int64(infrav1.DefaultAPIServerUnhealthThresholdCount), + Name: aws.String("name"), + Port: aws.Int64(infrav1.DefaultAPIServerPort), + Protocol: aws.String("TCP"), + VpcId: aws.String(vpcID), Tags: []*elbv2.Tag{ { Key: aws.String("test"), @@ -1684,6 +1699,260 @@ func TestCreateNLB(t *testing.T) { } }, }, + { + name: "NLB with HTTP health check", + awsCluster: func(acl infrav1.AWSCluster) infrav1.AWSCluster { + acl.Spec.ControlPlaneLoadBalancer.Scheme = &infrav1.ELBSchemeInternetFacing + acl.Spec.ControlPlaneLoadBalancer.LoadBalancerType = infrav1.LoadBalancerTypeNLB + acl.Spec.ControlPlaneLoadBalancer.HealthCheckProtocol = &infrav1.ELBProtocolHTTP + return acl + }, + spec: func(spec infrav1.LoadBalancer) infrav1.LoadBalancer { + tg := stubInfraV1TargetGroupSpecAPI + tg.VpcID = vpcID + tg.HealthCheck.Protocol = aws.String("HTTP") + tg.HealthCheck.Port = aws.String(infrav1.DefaultAPIServerPortString) + tg.HealthCheck.Path = aws.String("/readyz") + spec.ELBListeners = []infrav1.Listener{ + { + Protocol: "TCP", + Port: infrav1.DefaultAPIServerPort, + TargetGroup: tg, + }, + } + return spec + }, + elbV2APIMocks: func(m *mocks.MockELBV2APIMockRecorder) { + m.CreateLoadBalancer(gomock.Eq(&elbv2.CreateLoadBalancerInput{ + Name: aws.String(elbName), + Scheme: aws.String("internet-facing"), + SecurityGroups: aws.StringSlice([]string{}), + Type: aws.String("network"), + Subnets: aws.StringSlice([]string{clusterSubnetID}), + Tags: []*elbv2.Tag{ + { + Key: aws.String("test"), + Value: aws.String("tag"), + }, + }, + })).Return(&elbv2.CreateLoadBalancerOutput{ + LoadBalancers: []*elbv2.LoadBalancer{ + { + LoadBalancerArn: aws.String(elbArn), + LoadBalancerName: aws.String(elbName), + Scheme: aws.String(string(infrav1.ELBSchemeInternetFacing)), + DNSName: aws.String(dns), + }, + }, + }, nil) + m.CreateTargetGroup(gomock.Eq(&elbv2.CreateTargetGroupInput{ + Name: aws.String("name"), + Port: aws.Int64(infrav1.DefaultAPIServerPort), + Protocol: aws.String("TCP"), + VpcId: aws.String(vpcID), + HealthCheckEnabled: aws.Bool(true), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), + HealthCheckProtocol: aws.String("HTTP"), + HealthCheckPath: aws.String("/readyz"), + HealthCheckIntervalSeconds: aws.Int64(10), + HealthCheckTimeoutSeconds: aws.Int64(5), + HealthyThresholdCount: aws.Int64(5), + UnhealthyThresholdCount: aws.Int64(3), + Tags: []*elbv2.Tag{ + { + Key: aws.String("test"), + Value: aws.String("tag"), + }, + }, + })).Return(&elbv2.CreateTargetGroupOutput{ + TargetGroups: []*elbv2.TargetGroup{ + { + TargetGroupArn: aws.String("target-group::arn"), + TargetGroupName: aws.String("name"), + VpcId: aws.String(vpcID), + HealthCheckEnabled: aws.Bool(true), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), + HealthCheckProtocol: aws.String("http"), + HealthCheckPath: aws.String("/readyz"), + HealthCheckIntervalSeconds: aws.Int64(10), + HealthCheckTimeoutSeconds: aws.Int64(5), + HealthyThresholdCount: aws.Int64(5), + UnhealthyThresholdCount: aws.Int64(3), + }, + }, + }, nil) + m.CreateListener(gomock.Eq(&elbv2.CreateListenerInput{ + DefaultActions: []*elbv2.Action{ + { + TargetGroupArn: aws.String("target-group::arn"), + Type: aws.String(elbv2.ActionTypeEnumForward), + }, + }, + LoadBalancerArn: aws.String(elbArn), + Port: aws.Int64(infrav1.DefaultAPIServerPort), + Protocol: aws.String("TCP"), + Tags: []*elbv2.Tag{ + { + Key: aws.String("test"), + Value: aws.String("tag"), + }, + }, + })).Return(&elbv2.CreateListenerOutput{ + Listeners: []*elbv2.Listener{ + { + ListenerArn: aws.String("listener::arn"), + }, + }, + }, nil) + m.ModifyTargetGroupAttributes(gomock.Eq(&elbv2.ModifyTargetGroupAttributesInput{ + TargetGroupArn: aws.String("target-group::arn"), + Attributes: []*elbv2.TargetGroupAttribute{ + { + Key: aws.String(infrav1.TargetGroupAttributeEnablePreserveClientIP), + Value: aws.String("false"), + }, + }, + })).Return(nil, nil) + }, + check: func(t *testing.T, lb *infrav1.LoadBalancer, err error) { + t.Helper() + if err != nil { + t.Fatalf("did not expect error: %v", err) + } + got := *lb.ELBListeners[0].TargetGroup.HealthCheck.Protocol + want := "HTTP" + if got != want { + t.Fatalf("Health Check protocol for the API Target group did not equal expected value: %s; was: '%s'", want, got) + } + }, + }, + { + name: "NLB with HTTPS health check", + awsCluster: func(acl infrav1.AWSCluster) infrav1.AWSCluster { + acl.Spec.ControlPlaneLoadBalancer.Scheme = &infrav1.ELBSchemeInternetFacing + acl.Spec.ControlPlaneLoadBalancer.LoadBalancerType = infrav1.LoadBalancerTypeNLB + acl.Spec.ControlPlaneLoadBalancer.HealthCheckProtocol = &infrav1.ELBProtocolHTTPS + return acl + }, + spec: func(spec infrav1.LoadBalancer) infrav1.LoadBalancer { + tg := stubInfraV1TargetGroupSpecAPI + tg.VpcID = vpcID + tg.HealthCheck.Protocol = aws.String("HTTPS") + tg.HealthCheck.Port = aws.String(infrav1.DefaultAPIServerPortString) + tg.HealthCheck.Path = aws.String("/readyz") + spec.ELBListeners = []infrav1.Listener{ + { + Protocol: "TCP", + Port: infrav1.DefaultAPIServerPort, + TargetGroup: tg, + }, + } + return spec + }, + elbV2APIMocks: func(m *mocks.MockELBV2APIMockRecorder) { + m.CreateLoadBalancer(gomock.Eq(&elbv2.CreateLoadBalancerInput{ + Name: aws.String(elbName), + Scheme: aws.String("internet-facing"), + SecurityGroups: aws.StringSlice([]string{}), + Type: aws.String("network"), + Subnets: aws.StringSlice([]string{clusterSubnetID}), + Tags: []*elbv2.Tag{ + { + Key: aws.String("test"), + Value: aws.String("tag"), + }, + }, + })).Return(&elbv2.CreateLoadBalancerOutput{ + LoadBalancers: []*elbv2.LoadBalancer{ + { + LoadBalancerArn: aws.String(elbArn), + LoadBalancerName: aws.String(elbName), + Scheme: aws.String(string(infrav1.ELBSchemeInternetFacing)), + DNSName: aws.String(dns), + }, + }, + }, nil) + m.CreateTargetGroup(gomock.Eq(&elbv2.CreateTargetGroupInput{ + Name: aws.String("name"), + Port: aws.Int64(infrav1.DefaultAPIServerPort), + Protocol: aws.String("TCP"), + VpcId: aws.String(vpcID), + HealthCheckEnabled: aws.Bool(true), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), + HealthCheckProtocol: aws.String("HTTPS"), + HealthCheckPath: aws.String("/readyz"), + HealthCheckIntervalSeconds: aws.Int64(10), + HealthCheckTimeoutSeconds: aws.Int64(5), + HealthyThresholdCount: aws.Int64(5), + UnhealthyThresholdCount: aws.Int64(3), + Tags: []*elbv2.Tag{ + { + Key: aws.String("test"), + Value: aws.String("tag"), + }, + }, + })).Return(&elbv2.CreateTargetGroupOutput{ + TargetGroups: []*elbv2.TargetGroup{ + { + TargetGroupArn: aws.String("target-group::arn"), + TargetGroupName: aws.String("name"), + VpcId: aws.String(vpcID), + HealthCheckEnabled: aws.Bool(true), + HealthCheckPort: aws.String(infrav1.DefaultAPIServerPortString), + HealthCheckProtocol: aws.String("HTTPS"), + HealthCheckPath: aws.String("/readyz"), + HealthCheckIntervalSeconds: aws.Int64(10), + HealthCheckTimeoutSeconds: aws.Int64(5), + HealthyThresholdCount: aws.Int64(5), + UnhealthyThresholdCount: aws.Int64(3), + }, + }, + }, nil) + m.CreateListener(gomock.Eq(&elbv2.CreateListenerInput{ + DefaultActions: []*elbv2.Action{ + { + TargetGroupArn: aws.String("target-group::arn"), + Type: aws.String(elbv2.ActionTypeEnumForward), + }, + }, + LoadBalancerArn: aws.String(elbArn), + Port: aws.Int64(infrav1.DefaultAPIServerPort), + Protocol: aws.String("TCP"), + Tags: []*elbv2.Tag{ + { + Key: aws.String("test"), + Value: aws.String("tag"), + }, + }, + })).Return(&elbv2.CreateListenerOutput{ + Listeners: []*elbv2.Listener{ + { + ListenerArn: aws.String("listener::arn"), + }, + }, + }, nil) + m.ModifyTargetGroupAttributes(gomock.Eq(&elbv2.ModifyTargetGroupAttributesInput{ + TargetGroupArn: aws.String("target-group::arn"), + Attributes: []*elbv2.TargetGroupAttribute{ + { + Key: aws.String(infrav1.TargetGroupAttributeEnablePreserveClientIP), + Value: aws.String("false"), + }, + }, + })).Return(nil, nil) + }, + check: func(t *testing.T, lb *infrav1.LoadBalancer, err error) { + t.Helper() + if err != nil { + t.Fatalf("did not expect error: %v", err) + } + got := *lb.ELBListeners[0].TargetGroup.HealthCheck.Protocol + want := "HTTPS" + if got != want { + t.Fatalf("Health Check protocol for the API Target group did not equal expected value: %s; was: '%s'", want, got) + } + }, + }, } for _, tc := range tests { @@ -1751,7 +2020,7 @@ func TestCreateNLB(t *testing.T) { VpcID: vpcID, HealthCheck: &infrav1.TargetGroupHealthCheck{ Protocol: aws.String("tcp"), - Port: aws.String("infrav1.DefaultAPIServerPort"), + Port: aws.String(infrav1.DefaultAPIServerPortString), }, }, },