diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/BUILD b/staging/src/k8s.io/legacy-cloud-providers/aws/BUILD index 0a4b5b691072..4f394bd8ed0e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/BUILD +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/BUILD @@ -86,13 +86,16 @@ go_test( "//staging/src/k8s.io/api/core/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", + "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library", "//staging/src/k8s.io/client-go/informers:go_default_library", "//staging/src/k8s.io/client-go/kubernetes/fake:go_default_library", "//staging/src/k8s.io/cloud-provider/volume:go_default_library", "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/aws/awserr:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library", "//vendor/github.com/aws/aws-sdk-go/service/elb:go_default_library", + "//vendor/github.com/aws/aws-sdk-go/service/elbv2:go_default_library", "//vendor/github.com/stretchr/testify/assert:go_default_library", "//vendor/github.com/stretchr/testify/mock:go_default_library", "//vendor/github.com/stretchr/testify/require:go_default_library", diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go index 2af25a946b5d..9207442a2d9d 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_loadbalancer.go @@ -603,7 +603,9 @@ func (c *Cloud) ensureTargetGroup(targetGroup *elbv2.TargetGroup, serviceName ty } actualIDs := []string{} for _, healthDescription := range healthResponse.TargetHealthDescriptions { - if healthDescription.TargetHealth.Reason != nil { + if aws.StringValue(healthDescription.TargetHealth.State) == elbv2.TargetHealthStateEnumHealthy { + actualIDs = append(actualIDs, *healthDescription.Target.Id) + } else if healthDescription.TargetHealth.Reason != nil { switch aws.StringValue(healthDescription.TargetHealth.Reason) { case elbv2.TargetHealthReasonEnumTargetDeregistrationInProgress: // We don't need to count this instance in service if it is diff --git a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go index 0cfe6645dcfc..57a3ebc0cd2e 100644 --- a/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go +++ b/staging/src/k8s.io/legacy-cloud-providers/aws/aws_test.go @@ -22,21 +22,25 @@ import ( "context" "fmt" "io" + "math/rand" "reflect" "sort" "strings" "testing" "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/awserr" "github.com/aws/aws-sdk-go/service/ec2" "github.com/aws/aws-sdk-go/service/elb" + "github.com/aws/aws-sdk-go/service/elbv2" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" @@ -1710,7 +1714,6 @@ func TestAddLoadBalancerTags(t *testing.T) { } func TestEnsureLoadBalancerHealthCheck(t *testing.T) { - tests := []struct { name string annotations map[string]string @@ -1976,6 +1979,579 @@ func informerNotSynced() bool { return false } +type MockedFakeELBV2 struct { + LoadBalancers []*elbv2.LoadBalancer + TargetGroups []*elbv2.TargetGroup + Listeners []*elbv2.Listener + + // keys on all of these maps are ARNs + LoadBalancerAttributes map[string]map[string]string + Tags map[string][]elbv2.Tag + RegisteredInstances map[string][]string // value is list of instance IDs +} + +func (m *MockedFakeELBV2) AddTags(request *elbv2.AddTagsInput) (*elbv2.AddTagsOutput, error) { + for _, arn := range request.ResourceArns { + for _, tag := range request.Tags { + m.Tags[aws.StringValue(arn)] = append(m.Tags[aws.StringValue(arn)], *tag) + } + } + + return &elbv2.AddTagsOutput{}, nil +} + +func (m *MockedFakeELBV2) CreateLoadBalancer(request *elbv2.CreateLoadBalancerInput) (*elbv2.CreateLoadBalancerOutput, error) { + accountID := 123456789 + arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:loadbalancer/net/%x/%x", + accountID, + rand.Uint64(), + rand.Uint32()) + + newLB := &elbv2.LoadBalancer{ + LoadBalancerArn: aws.String(arn), + LoadBalancerName: request.Name, + Type: aws.String(elbv2.LoadBalancerTypeEnumNetwork), + VpcId: aws.String("vpc-abc123def456abc78"), + } + m.LoadBalancers = append(m.LoadBalancers, newLB) + + return &elbv2.CreateLoadBalancerOutput{ + LoadBalancers: []*elbv2.LoadBalancer{newLB}, + }, nil +} + +func (m *MockedFakeELBV2) DescribeLoadBalancers(request *elbv2.DescribeLoadBalancersInput) (*elbv2.DescribeLoadBalancersOutput, error) { + findMeNames := make(map[string]bool) + for _, name := range request.Names { + findMeNames[aws.StringValue(name)] = true + } + + findMeARNs := make(map[string]bool) + for _, arn := range request.LoadBalancerArns { + findMeARNs[aws.StringValue(arn)] = true + } + + result := []*elbv2.LoadBalancer{} + + for _, lb := range m.LoadBalancers { + if _, present := findMeNames[aws.StringValue(lb.LoadBalancerName)]; present { + result = append(result, lb) + delete(findMeNames, aws.StringValue(lb.LoadBalancerName)) + } else if _, present := findMeARNs[aws.StringValue(lb.LoadBalancerArn)]; present { + result = append(result, lb) + delete(findMeARNs, aws.StringValue(lb.LoadBalancerArn)) + } + } + + if len(findMeNames) > 0 || len(findMeARNs) > 0 { + return nil, awserr.New(elbv2.ErrCodeLoadBalancerNotFoundException, "not found", nil) + } + + return &elbv2.DescribeLoadBalancersOutput{ + LoadBalancers: result, + }, nil +} + +func (m *MockedFakeELBV2) DeleteLoadBalancer(*elbv2.DeleteLoadBalancerInput) (*elbv2.DeleteLoadBalancerOutput, error) { + panic("Not implemented") +} + +func (m *MockedFakeELBV2) ModifyLoadBalancerAttributes(request *elbv2.ModifyLoadBalancerAttributesInput) (*elbv2.ModifyLoadBalancerAttributesOutput, error) { + attrMap, present := m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)] + + if !present { + attrMap = make(map[string]string) + m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)] = attrMap + } + + for _, attr := range request.Attributes { + attrMap[aws.StringValue(attr.Key)] = aws.StringValue(attr.Value) + } + + return &elbv2.ModifyLoadBalancerAttributesOutput{ + Attributes: request.Attributes, + }, nil +} + +func (m *MockedFakeELBV2) DescribeLoadBalancerAttributes(request *elbv2.DescribeLoadBalancerAttributesInput) (*elbv2.DescribeLoadBalancerAttributesOutput, error) { + attrs := []*elbv2.LoadBalancerAttribute{} + + if lbAttrs, present := m.LoadBalancerAttributes[aws.StringValue(request.LoadBalancerArn)]; present { + for key, value := range lbAttrs { + attrs = append(attrs, &elbv2.LoadBalancerAttribute{ + Key: aws.String(key), + Value: aws.String(value), + }) + } + } + + return &elbv2.DescribeLoadBalancerAttributesOutput{ + Attributes: attrs, + }, nil +} + +func (m *MockedFakeELBV2) CreateTargetGroup(request *elbv2.CreateTargetGroupInput) (*elbv2.CreateTargetGroupOutput, error) { + accountID := 123456789 + arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:targetgroup/%x/%x", + accountID, + rand.Uint64(), + rand.Uint32()) + + newTG := &elbv2.TargetGroup{ + TargetGroupArn: aws.String(arn), + TargetGroupName: request.Name, + Port: request.Port, + Protocol: request.Protocol, + } + + m.TargetGroups = append(m.TargetGroups, newTG) + + return &elbv2.CreateTargetGroupOutput{ + TargetGroups: []*elbv2.TargetGroup{newTG}, + }, nil +} + +func (m *MockedFakeELBV2) DescribeTargetGroups(request *elbv2.DescribeTargetGroupsInput) (*elbv2.DescribeTargetGroupsOutput, error) { + var targetGroups []*elbv2.TargetGroup + + if request.LoadBalancerArn != nil { + targetGroups = []*elbv2.TargetGroup{} + + for _, tg := range m.TargetGroups { + for _, lbArn := range tg.LoadBalancerArns { + if aws.StringValue(lbArn) == aws.StringValue(request.LoadBalancerArn) { + targetGroups = append(targetGroups, tg) + break + } + } + } + } else if len(request.Names) != 0 { + targetGroups = []*elbv2.TargetGroup{} + + for _, tg := range m.TargetGroups { + for _, name := range request.Names { + if aws.StringValue(tg.TargetGroupName) == aws.StringValue(name) { + targetGroups = append(targetGroups, tg) + break + } + } + } + } else if len(request.TargetGroupArns) != 0 { + targetGroups = []*elbv2.TargetGroup{} + + for _, tg := range m.TargetGroups { + for _, arn := range request.TargetGroupArns { + if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(arn) { + targetGroups = append(targetGroups, tg) + break + } + } + } + } else { + targetGroups = m.TargetGroups + } + + return &elbv2.DescribeTargetGroupsOutput{ + TargetGroups: targetGroups, + }, nil +} + +func (m *MockedFakeELBV2) ModifyTargetGroup(request *elbv2.ModifyTargetGroupInput) (*elbv2.ModifyTargetGroupOutput, error) { + var matchingTargetGroup *elbv2.TargetGroup + dirtyGroups := []*elbv2.TargetGroup{} + + for _, tg := range m.TargetGroups { + if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(request.TargetGroupArn) { + matchingTargetGroup = tg + break + } + } + + if matchingTargetGroup != nil { + dirtyGroups = append(dirtyGroups, matchingTargetGroup) + + if request.HealthCheckEnabled != nil { + matchingTargetGroup.HealthCheckEnabled = request.HealthCheckEnabled + } + if request.HealthCheckIntervalSeconds != nil { + matchingTargetGroup.HealthCheckIntervalSeconds = request.HealthCheckIntervalSeconds + } + if request.HealthCheckPath != nil { + matchingTargetGroup.HealthCheckPath = request.HealthCheckPath + } + if request.HealthCheckPort != nil { + matchingTargetGroup.HealthCheckPort = request.HealthCheckPort + } + if request.HealthCheckProtocol != nil { + matchingTargetGroup.HealthCheckProtocol = request.HealthCheckProtocol + } + if request.HealthCheckTimeoutSeconds != nil { + matchingTargetGroup.HealthCheckTimeoutSeconds = request.HealthCheckTimeoutSeconds + } + if request.HealthyThresholdCount != nil { + matchingTargetGroup.HealthyThresholdCount = request.HealthyThresholdCount + } + if request.Matcher != nil { + matchingTargetGroup.Matcher = request.Matcher + } + if request.UnhealthyThresholdCount != nil { + matchingTargetGroup.UnhealthyThresholdCount = request.UnhealthyThresholdCount + } + } + + return &elbv2.ModifyTargetGroupOutput{ + TargetGroups: dirtyGroups, + }, nil +} + +func (m *MockedFakeELBV2) DeleteTargetGroup(request *elbv2.DeleteTargetGroupInput) (*elbv2.DeleteTargetGroupOutput, error) { + newTargetGroups := []*elbv2.TargetGroup{} + + for _, tg := range m.TargetGroups { + if aws.StringValue(tg.TargetGroupArn) != aws.StringValue(request.TargetGroupArn) { + newTargetGroups = append(newTargetGroups, tg) + } + } + + m.TargetGroups = newTargetGroups + + delete(m.RegisteredInstances, aws.StringValue(request.TargetGroupArn)) + + return &elbv2.DeleteTargetGroupOutput{}, nil +} + +func (m *MockedFakeELBV2) DescribeTargetHealth(request *elbv2.DescribeTargetHealthInput) (*elbv2.DescribeTargetHealthOutput, error) { + healthDescriptions := []*elbv2.TargetHealthDescription{} + + var matchingTargetGroup *elbv2.TargetGroup + + for _, tg := range m.TargetGroups { + if aws.StringValue(tg.TargetGroupArn) == aws.StringValue(request.TargetGroupArn) { + matchingTargetGroup = tg + break + } + } + + if registeredTargets, present := m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)]; present { + for _, target := range registeredTargets { + healthDescriptions = append(healthDescriptions, &elbv2.TargetHealthDescription{ + HealthCheckPort: matchingTargetGroup.HealthCheckPort, + Target: &elbv2.TargetDescription{ + Id: aws.String(target), + Port: matchingTargetGroup.Port, + }, + TargetHealth: &elbv2.TargetHealth{ + State: aws.String("healthy"), + }, + }) + } + } + + return &elbv2.DescribeTargetHealthOutput{ + TargetHealthDescriptions: healthDescriptions, + }, nil +} + +func (m *MockedFakeELBV2) DescribeTargetGroupAttributes(*elbv2.DescribeTargetGroupAttributesInput) (*elbv2.DescribeTargetGroupAttributesOutput, error) { + panic("Not implemented") +} + +func (m *MockedFakeELBV2) ModifyTargetGroupAttributes(*elbv2.ModifyTargetGroupAttributesInput) (*elbv2.ModifyTargetGroupAttributesOutput, error) { + panic("Not implemented") +} + +func (m *MockedFakeELBV2) RegisterTargets(request *elbv2.RegisterTargetsInput) (*elbv2.RegisterTargetsOutput, error) { + arn := aws.StringValue(request.TargetGroupArn) + alreadyExists := make(map[string]bool) + for _, targetID := range m.RegisteredInstances[arn] { + alreadyExists[targetID] = true + } + for _, target := range request.Targets { + if !alreadyExists[aws.StringValue(target.Id)] { + m.RegisteredInstances[arn] = append(m.RegisteredInstances[arn], aws.StringValue(target.Id)) + } + } + return &elbv2.RegisterTargetsOutput{}, nil +} + +func (m *MockedFakeELBV2) DeregisterTargets(request *elbv2.DeregisterTargetsInput) (*elbv2.DeregisterTargetsOutput, error) { + removeMe := make(map[string]bool) + + for _, target := range request.Targets { + removeMe[aws.StringValue(target.Id)] = true + } + newRegisteredInstancesForArn := []string{} + for _, targetID := range m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)] { + if !removeMe[targetID] { + newRegisteredInstancesForArn = append(newRegisteredInstancesForArn, targetID) + } + } + m.RegisteredInstances[aws.StringValue(request.TargetGroupArn)] = newRegisteredInstancesForArn + + return &elbv2.DeregisterTargetsOutput{}, nil +} + +func (m *MockedFakeELBV2) CreateListener(request *elbv2.CreateListenerInput) (*elbv2.CreateListenerOutput, error) { + accountID := 123456789 + arn := fmt.Sprintf("arn:aws:elasticloadbalancing:us-east-1:%d:listener/net/%x/%x/%x", + accountID, + rand.Uint64(), + rand.Uint32(), + rand.Uint32()) + + newListener := &elbv2.Listener{ + ListenerArn: aws.String(arn), + Port: request.Port, + Protocol: request.Protocol, + DefaultActions: request.DefaultActions, + LoadBalancerArn: request.LoadBalancerArn, + } + + m.Listeners = append(m.Listeners, newListener) + + for _, tg := range m.TargetGroups { + for _, action := range request.DefaultActions { + if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) { + tg.LoadBalancerArns = append(tg.LoadBalancerArns, request.LoadBalancerArn) + break + } + } + } + + return &elbv2.CreateListenerOutput{ + Listeners: []*elbv2.Listener{newListener}, + }, nil +} + +func (m *MockedFakeELBV2) DescribeListeners(request *elbv2.DescribeListenersInput) (*elbv2.DescribeListenersOutput, error) { + if len(request.ListenerArns) == 0 && request.LoadBalancerArn == nil { + return &elbv2.DescribeListenersOutput{ + Listeners: m.Listeners, + }, nil + } else if len(request.ListenerArns) == 0 { + listeners := []*elbv2.Listener{} + + for _, lb := range m.Listeners { + if aws.StringValue(lb.LoadBalancerArn) == aws.StringValue(request.LoadBalancerArn) { + listeners = append(listeners, lb) + } + } + + return &elbv2.DescribeListenersOutput{ + Listeners: listeners, + }, nil + } + panic("Not implemented") +} + +func (m *MockedFakeELBV2) DeleteListener(*elbv2.DeleteListenerInput) (*elbv2.DeleteListenerOutput, error) { + panic("Not implemented") +} + +func (m *MockedFakeELBV2) ModifyListener(request *elbv2.ModifyListenerInput) (*elbv2.ModifyListenerOutput, error) { + modifiedListeners := []*elbv2.Listener{} + + for _, listener := range m.Listeners { + if aws.StringValue(listener.ListenerArn) == aws.StringValue(request.ListenerArn) { + if request.DefaultActions != nil { + // for each old action, find the corresponding target group, and remove the listener's LB ARN from its list + for _, action := range listener.DefaultActions { + var targetGroupForAction *elbv2.TargetGroup + + for _, tg := range m.TargetGroups { + if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) { + targetGroupForAction = tg + break + } + } + + if targetGroupForAction != nil { + newLoadBalancerARNs := []*string{} + for _, lbArn := range targetGroupForAction.LoadBalancerArns { + if aws.StringValue(lbArn) != aws.StringValue(listener.LoadBalancerArn) { + newLoadBalancerARNs = append(newLoadBalancerARNs, lbArn) + } + } + + targetGroupForAction.LoadBalancerArns = newLoadBalancerARNs + } + } + + listener.DefaultActions = request.DefaultActions + + // for each new action, add the listener's LB ARN to that action's target groups' lists + for _, action := range request.DefaultActions { + var targetGroupForAction *elbv2.TargetGroup + + for _, tg := range m.TargetGroups { + if aws.StringValue(action.TargetGroupArn) == aws.StringValue(tg.TargetGroupArn) { + targetGroupForAction = tg + break + } + } + + if targetGroupForAction != nil { + targetGroupForAction.LoadBalancerArns = append(targetGroupForAction.LoadBalancerArns, listener.LoadBalancerArn) + } + } + } + if request.Port != nil { + listener.Port = request.Port + } + if request.Protocol != nil { + listener.Protocol = request.Protocol + } + + modifiedListeners = append(modifiedListeners, listener) + } + } + + return &elbv2.ModifyListenerOutput{ + Listeners: modifiedListeners, + }, nil +} + +func (m *MockedFakeELBV2) WaitUntilLoadBalancersDeleted(*elbv2.DescribeLoadBalancersInput) error { + panic("Not implemented") +} + +func (m *MockedFakeEC2) maybeExpectDescribeSecurityGroups(clusterID, groupName string) { + tags := []*ec2.Tag{ + {Key: aws.String(TagNameKubernetesClusterLegacy), Value: aws.String(clusterID)}, + {Key: aws.String(fmt.Sprintf("%s%s", TagNameKubernetesClusterPrefix, clusterID)), Value: aws.String(ResourceLifecycleOwned)}, + } + + m.On("DescribeSecurityGroups", &ec2.DescribeSecurityGroupsInput{Filters: []*ec2.Filter{ + newEc2Filter("group-name", groupName), + newEc2Filter("vpc-id", ""), + }}).Maybe().Return([]*ec2.SecurityGroup{{Tags: tags}}) + + m.On("DescribeSecurityGroups", &ec2.DescribeSecurityGroupsInput{}).Maybe().Return([]*ec2.SecurityGroup{{Tags: tags}}) +} + +func TestNLBNodeRegistration(t *testing.T) { + awsServices := newMockedFakeAWSServices(TestClusterID) + awsServices.elbv2 = &MockedFakeELBV2{Tags: make(map[string][]elbv2.Tag), RegisteredInstances: make(map[string][]string), LoadBalancerAttributes: make(map[string]map[string]string)} + c, _ := newAWSCloud(CloudConfig{}, awsServices) + + awsServices.ec2.(*MockedFakeEC2).Subnets = []*ec2.Subnet{ + { + AvailabilityZone: aws.String("us-west-2a"), + SubnetId: aws.String("subnet-abc123de"), + Tags: []*ec2.Tag{ + { + Key: aws.String(c.tagging.clusterTagKey()), + Value: aws.String("owned"), + }, + }, + }, + } + + awsServices.ec2.(*MockedFakeEC2).RouteTables = []*ec2.RouteTable{ + { + Associations: []*ec2.RouteTableAssociation{ + { + Main: aws.Bool(true), + RouteTableAssociationId: aws.String("rtbassoc-abc123def456abc78"), + RouteTableId: aws.String("rtb-abc123def456abc78"), + SubnetId: aws.String("subnet-abc123de"), + }, + }, + RouteTableId: aws.String("rtb-abc123def456abc78"), + Routes: []*ec2.Route{ + { + DestinationCidrBlock: aws.String("0.0.0.0/0"), + GatewayId: aws.String("igw-abc123def456abc78"), + State: aws.String("active"), + }, + }, + }, + } + awsServices.ec2.(*MockedFakeEC2).maybeExpectDescribeSecurityGroups(TestClusterID, "k8s-elb-aid") + + nodes := []*v1.Node{makeNamedNode(awsServices, 0, "a"), makeNamedNode(awsServices, 1, "b"), makeNamedNode(awsServices, 2, "c")} + + fauxService := &v1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "myservice", + UID: "id", + Annotations: map[string]string{ + "service.beta.kubernetes.io/aws-load-balancer-type": "nlb", + }, + }, + Spec: v1.ServiceSpec{ + Ports: []v1.ServicePort{ + { + Name: "http", + Port: 8080, + NodePort: 31173, + TargetPort: intstr.FromInt(31173), + Protocol: v1.ProtocolTCP, + }, + }, + SessionAffinity: v1.ServiceAffinityNone, + }, + } + + _, err := c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes) + if err != nil { + t.Errorf("EnsureLoadBalancer returned an error: %v", err) + } + for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances { + if len(instances) != 3 { + t.Errorf("Expected 3 nodes registered with target group, saw %d", len(instances)) + } + } + + _, err = c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes[:2]) + if err != nil { + t.Errorf("EnsureLoadBalancer returned an error: %v", err) + } + for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances { + if len(instances) != 2 { + t.Errorf("Expected 2 nodes registered with target group, saw %d", len(instances)) + } + } + + _, err = c.EnsureLoadBalancer(context.TODO(), TestClusterName, fauxService, nodes) + if err != nil { + t.Errorf("EnsureLoadBalancer returned an error: %v", err) + } + for _, instances := range awsServices.elbv2.(*MockedFakeELBV2).RegisteredInstances { + if len(instances) != 3 { + t.Errorf("Expected 3 nodes registered with target group, saw %d", len(instances)) + } + } +} + +func makeNamedNode(s *FakeAWSServices, offset int, name string) *v1.Node { + instanceID := fmt.Sprintf("i-%x", int64(0x02bce90670bb0c7cd)+int64(offset)) + instance := &ec2.Instance{} + instance.InstanceId = aws.String(instanceID) + instance.Placement = &ec2.Placement{ + AvailabilityZone: aws.String("us-east-1c"), + } + instance.PrivateDnsName = aws.String(fmt.Sprintf("ip-172-20-0-%d.ec2.internal", 101+offset)) + instance.PrivateIpAddress = aws.String(fmt.Sprintf("192.168.0.%d", 1+offset)) + + var tag ec2.Tag + tag.Key = aws.String(TagNameKubernetesClusterLegacy) + tag.Value = aws.String(TestClusterID) + instance.Tags = []*ec2.Tag{&tag} + + s.instances = append(s.instances, instance) + + testProviderID := "aws:///us-east-1c/" + instanceID + return &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + }, + Spec: v1.NodeSpec{ + ProviderID: testProviderID, + }, + } +} + func newMockedFakeAWSServices(id string) *FakeAWSServices { s := NewFakeAWSServices(id) s.ec2 = &MockedFakeEC2{FakeEC2Impl: s.ec2.(*FakeEC2Impl)}