Use DualStack API NLB for IPv6

cloudmock/aws/mockelbv2/loadbalancers.go

@@ -83,6 +83,7 @@ func (m *MockELBV2) CreateLoadBalancer(request *elbv2.CreateLoadBalancerInput) (
 		LoadBalancerName:      request.Name,
 		Scheme:                request.Scheme,
 		Type:                  request.Type,
+		IpAddressType:         request.IpAddressType,
 		DNSName:               aws.String(fmt.Sprintf("%v.amazonaws.com", aws.StringValue(request.Name))),
 		CanonicalHostedZoneId: aws.String("HZ123456"),
 	}

pkg/apis/kops/validation/validation.go

@@ -418,7 +418,7 @@ func validateSubnets(cluster *kops.ClusterSpec, fieldPath *field.Path) field.Err
 	if kops.CloudProviderID(cluster.CloudProvider) != kops.CloudProviderAWS {
 		for i := range subnets {
 			if subnets[i].IPv6CIDR != "" {
-				allErrs = append(allErrs, field.Forbidden(fieldPath.Child("ipv6CIDR"), "ipv6CIDR can only be specified for AWS"))
+				allErrs = append(allErrs, field.Forbidden(fieldPath.Index(i).Child("ipv6CIDR"), "ipv6CIDR can only be specified for AWS"))
 			}
 		}
 	}

pkg/model/awsmodel/api_loadbalancer.go

@@ -178,9 +178,13 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 			Listeners:        nlbListeners,
 			TargetGroups:     make([]*awstasks.TargetGroup, 0),
 
-			Tags: tags,
-			VPC:  b.LinkToVPC(),
-			Type: fi.String("network"),
+			Tags:          tags,
+			VPC:           b.LinkToVPC(),
+			Type:          fi.String("network"),
+			IpAddressType: fi.String("ipv4"),
+		}
+		if b.UseIPv6ForAPI() {
+			nlb.IpAddressType = fi.String("dualstack")
 		}
 
 		clb = &awstasks.ClassicLoadBalancer{

pkg/model/awsmodel/autoscalinggroup.go

@@ -215,10 +215,14 @@ func (b *AutoscalingGroupModelBuilder) buildLaunchTemplateTask(c *fi.ModelBuilde
 		}
 
 		// @step: add an IPv6 address
-		for _, subnet := range subnets {
-			if subnet.IPv6CIDR != "" {
-				lt.IPv6AddressCount = fi.Int64(1)
-				break
+		for _, clusterSubnet := range b.Cluster.Spec.Subnets {
+			for _, igSubnet := range ig.Spec.Subnets {
+				if clusterSubnet.Name != igSubnet {
+					continue
+				}
+				if clusterSubnet.IPv6CIDR != "" {
+					lt.IPv6AddressCount = fi.Int64(1)
+				}
 			}
 		}
 	}

pkg/model/awsmodel/dns.go

@@ -112,14 +112,24 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error {
 				return err
 			}
 
-			apiDnsName := &awstasks.DNSName{
+			c.AddTask(&awstasks.DNSName{
 				Name:               fi.String(b.Cluster.Spec.MasterPublicName),
+				ResourceName:       fi.String(b.Cluster.Spec.MasterPublicName),
 				Lifecycle:          b.Lifecycle,
 				Zone:               b.LinkToDNSZone(),
 				ResourceType:       fi.String("A"),
 				TargetLoadBalancer: targetLoadBalancer,
+			})
+			if b.UseIPv6ForAPI() {
+				c.AddTask(&awstasks.DNSName{
+					Name:               fi.String(b.Cluster.Spec.MasterPublicName + "-AAAA"),
+					ResourceName:       fi.String(b.Cluster.Spec.MasterPublicName),
+					Lifecycle:          b.Lifecycle,
+					Zone:               b.LinkToDNSZone(),
+					ResourceType:       fi.String("AAAA"),
+					TargetLoadBalancer: targetLoadBalancer,
+				})
 			}
-			c.AddTask(apiDnsName)
 		}
 	}
 
@@ -132,15 +142,25 @@ func (b *DNSModelBuilder) Build(c *fi.ModelBuilderContext) error {
 				return err
 			}
 
-			internalApiDnsName := &awstasks.DNSName{
+			// Using EnsureTask as MasterInternalName and MasterPublicName could be the same
+			c.EnsureTask(&awstasks.DNSName{
 				Name:               fi.String(b.Cluster.Spec.MasterInternalName),
+				ResourceName:       fi.String(b.Cluster.Spec.MasterInternalName),
 				Lifecycle:          b.Lifecycle,
 				Zone:               b.LinkToDNSZone(),
 				ResourceType:       fi.String("A"),
 				TargetLoadBalancer: targetLoadBalancer,
+			})
+			if b.UseIPv6ForAPI() {
+				c.EnsureTask(&awstasks.DNSName{
+					Name:               fi.String(b.Cluster.Spec.MasterInternalName + "-AAAA"),
+					ResourceName:       fi.String(b.Cluster.Spec.MasterInternalName),
+					Lifecycle:          b.Lifecycle,
+					Zone:               b.LinkToDNSZone(),
+					ResourceType:       fi.String("AAAA"),
+					TargetLoadBalancer: targetLoadBalancer,
+				})
 			}
-			// Using EnsureTask as MasterInternalName and MasterPublicName could be the same
-			c.EnsureTask(internalApiDnsName)
 		}
 	}
 

pkg/model/context.go

@@ -371,6 +371,25 @@ func (b *KopsModelContext) IsIPv6Only() bool {
 	return b.Cluster.Spec.IsIPv6Only()
 }
 
+func (b *KopsModelContext) UseIPv6ForAPI() bool {
+	for _, ig := range b.InstanceGroups {
+		if ig.Spec.Role != kops.InstanceGroupRoleMaster && ig.Spec.Role != kops.InstanceGroupRoleAPIServer {
+			break
+		}
+		for _, igSubnetName := range ig.Spec.Subnets {
+			for _, clusterSubnet := range b.Cluster.Spec.Subnets {
+				if igSubnetName != clusterSubnet.Name {
+					continue
+				}
+				if clusterSubnet.IPv6CIDR != "" {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
 // WellKnownServiceIP returns a service ip with the service cidr
 func (b *KopsModelContext) WellKnownServiceIP(id int) (net.IP, error) {
 	return components.WellKnownServiceIP(&b.Cluster.Spec, id)

pkg/resources/aws/aws.go

@@ -1801,7 +1801,7 @@ func ListRoute53Records(cloud fi.Cloud, clusterName string) ([]*resources.Resour
 		}
 		err := c.Route53().ListResourceRecordSetsPages(request, func(p *route53.ListResourceRecordSetsOutput, lastPage bool) bool {
 			for _, rrs := range p.ResourceRecordSets {
-				if aws.StringValue(rrs.Type) != "A" {
+				if aws.StringValue(rrs.Type) != "A" && aws.StringValue(rrs.Type) != "AAAA" {
 					continue
 				}
 
@@ -1827,7 +1827,7 @@ func ListRoute53Records(cloud fi.Cloud, clusterName string) ([]*resources.Resour
 
 				resourceTracker := &resources.Resource{
 					Name:     aws.StringValue(rrs.Name),
-					ID:       hostedZoneID + "/" + aws.StringValue(rrs.Name),
+					ID:       hostedZoneID + "/" + aws.StringValue(rrs.Type) + "/" + aws.StringValue(rrs.Name),
 					Type:     "route53-record",
 					GroupKey: hostedZoneID,
 					GroupDeleter: func(cloud fi.Cloud, resourceTrackers []*resources.Resource) error {

tests/integration/update_cluster/minimal-ipv6/cloudformation.json

@@ -1546,6 +1546,29 @@
         },
         "HostedZoneId": "/hostedzone/Z1AFAKE1ZON3YO"
       }
+    },
+    "AWSRoute53RecordSetapiminimalipv6examplecomAAAA": {
+      "Type": "AWS::Route53::RecordSet",
+      "Properties": {
+        "Name": "api.minimal-ipv6.example.com",
+        "Type": "AAAA",
+        "AliasTarget": {
+          "DNSName": {
+            "Fn::GetAtt": [
+              "AWSElasticLoadBalancingV2LoadBalancerapiminimalipv6examplecom",
+              "DNSName"
+            ]
+          },
+          "HostedZoneId": {
+            "Fn::GetAtt": [
+              "AWSElasticLoadBalancingV2LoadBalancerapiminimalipv6examplecom",
+              "CanonicalHostedZoneID"
+            ]
+          },
+          "EvaluateTargetHealth": false
+        },
+        "HostedZoneId": "/hostedzone/Z1AFAKE1ZON3YO"
+      }
     }
   }
 }

tests/integration/update_cluster/minimal-ipv6/kubernetes.tf

@@ -511,6 +511,17 @@ resource "aws_route53_record" "api-minimal-ipv6-example-com" {
   zone_id = "/hostedzone/Z1AFAKE1ZON3YO"
 }
 
+resource "aws_route53_record" "api-minimal-ipv6-example-com-AAAA" {
+  alias {
+    evaluate_target_health = false
+    name                   = aws_lb.api-minimal-ipv6-example-com.dns_name
+    zone_id                = aws_lb.api-minimal-ipv6-example-com.zone_id
+  }
+  name    = "api.minimal-ipv6.example.com"
+  type    = "AAAA"
+  zone_id = "/hostedzone/Z1AFAKE1ZON3YO"
+}
+
 resource "aws_route_table" "minimal-ipv6-example-com" {
   tags = {
     "KubernetesCluster"                              = "minimal-ipv6.example.com"

upup/pkg/fi/cloudup/awstasks/dnsname.go

@@ -38,6 +38,7 @@ type DNSName struct {
 
 	ID           *string
 	Zone         *DNSZone
+	ResourceName *string
 	ResourceType *string
 
 	TargetLoadBalancer DNSTarget
@@ -56,11 +57,11 @@ func (e *DNSName) Find(c *fi.Context) (*DNSName, error) {
 	cloud := c.Cloud.(awsup.AWSCloud)
 
 	if e.Zone == nil || e.Zone.ZoneID == nil {
-		klog.V(4).Infof("Zone / ZoneID not found for %s, skipping Find", fi.StringValue(e.Name))
+		klog.V(4).Infof("Zone / ZoneID not found for %s, skipping Find", fi.StringValue(e.ResourceName))
 		return nil, nil
 	}
 
-	findName := fi.StringValue(e.Name)
+	findName := fi.StringValue(e.ResourceName)
 	if findName == "" {
 		return nil, nil
 	}
@@ -111,16 +112,17 @@ func (e *DNSName) Find(c *fi.Context) (*DNSName, error) {
 	}
 
 	actual := &DNSName{}
-	actual.Zone = e.Zone
 	actual.Name = e.Name
+	actual.Zone = e.Zone
+	actual.ResourceName = e.ResourceName
 	actual.ResourceType = e.ResourceType
 	actual.Lifecycle = e.Lifecycle
 
 	if found.AliasTarget != nil {
 		dnsName := aws.StringValue(found.AliasTarget.DNSName)
 		klog.Infof("AliasTarget for %q is %q", aws.StringValue(found.Name), dnsName)
 		if dnsName != "" {
-			if actual.TargetLoadBalancer, err = findDNSTarget(cloud, found.AliasTarget, dnsName, e.Name); err != nil {
+			if actual.TargetLoadBalancer, err = findDNSTarget(cloud, found.AliasTarget, dnsName, e.ResourceName); err != nil {
 				return nil, err
 			}
 		}
@@ -196,7 +198,7 @@ func (e *DNSName) Run(c *fi.Context) error {
 
 func (s *DNSName) CheckChanges(a, e, changes *DNSName) error {
 	if a == nil {
-		if fi.StringValue(e.Name) == "" {
+		if fi.StringValue(e.ResourceName) == "" {
 			return fi.RequiredField("Name")
 		}
 	}
@@ -205,7 +207,7 @@ func (s *DNSName) CheckChanges(a, e, changes *DNSName) error {
 
 func (_ *DNSName) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *DNSName) error {
 	rrs := &route53.ResourceRecordSet{
-		Name: e.Name,
+		Name: e.ResourceName,
 		Type: e.ResourceType,
 	}
 
@@ -229,7 +231,7 @@ func (_ *DNSName) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *DNSName) error
 	request.HostedZoneId = e.Zone.ZoneID
 	request.ChangeBatch = changeBatch
 
-	klog.V(2).Infof("Updating DNS record %q", *e.Name)
+	klog.V(2).Infof("Updating DNS record %q", *e.ResourceName)
 
 	response, err := t.Cloud.Route53().ChangeResourceRecordSets(request)
 	if err != nil {
@@ -253,13 +255,14 @@ type terraformRoute53Record struct {
 
 type terraformAlias struct {
 	Name                 *terraformWriter.Literal `json:"name" cty:"name"`
+	Type                 *terraformWriter.Literal `json:"type" cty:"type"`
 	ZoneID               *terraformWriter.Literal `json:"zone_id" cty:"zone_id"`
 	EvaluateTargetHealth *bool                    `json:"evaluate_target_health" cty:"evaluate_target_health"`
 }
 
 func (_ *DNSName) RenderTerraform(t *terraform.TerraformTarget, a, e, changes *DNSName) error {
 	tf := &terraformRoute53Record{
-		Name:   e.Name,
+		Name:   e.ResourceName,
 		ZoneID: e.Zone.TerraformLink(),
 		Type:   e.ResourceType,
 	}
@@ -297,7 +300,7 @@ type cloudformationAlias struct {
 
 func (_ *DNSName) RenderCloudformation(t *cloudformation.CloudformationTarget, a, e, changes *DNSName) error {
 	cf := &cloudformationRoute53Record{
-		Name:   e.Name,
+		Name:   e.ResourceName,
 		ZoneID: e.Zone.CloudformationLink(),
 		Type:   e.ResourceType,
 	}

upup/pkg/fi/cloudup/awstasks/network_load_balancer.go

@@ -60,6 +60,8 @@ type NetworkLoadBalancer struct {
 
 	CrossZoneLoadBalancing *bool
 
+	IpAddressType *string
+
 	Tags         map[string]string
 	ForAPIServer bool
 
@@ -259,6 +261,7 @@ func (e *NetworkLoadBalancer) Find(c *fi.Context) (*NetworkLoadBalancer, error)
 	actual.Scheme = lb.Scheme
 	actual.VPC = &VPC{ID: lb.VpcId}
 	actual.Type = lb.Type
+	actual.IpAddressType = lb.IpAddressType
 
 	tagMap, err := cloud.DescribeELBV2Tags([]string{*loadBalancerArn})
 	if err != nil {
@@ -502,6 +505,7 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
 		request.Name = e.LoadBalancerName
 		request.Scheme = e.Scheme
 		request.Type = e.Type
+		request.IpAddressType = e.IpAddressType
 		request.Tags = awsup.ELBv2Tags(e.Tags)
 
 		for _, subnetMapping := range e.SubnetMappings {
@@ -555,6 +559,16 @@ func (_ *NetworkLoadBalancer) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *Ne
 
 		loadBalancerArn = fi.StringValue(lb.LoadBalancerArn)
 
+		if changes.IpAddressType != nil {
+			request := &elbv2.SetIpAddressTypeInput{
+				IpAddressType:   e.IpAddressType,
+				LoadBalancerArn: lb.LoadBalancerArn,
+			}
+			if _, err := t.Cloud.ELBV2().SetIpAddressType(request); err != nil {
+				return fmt.Errorf("error setting the IP addresses type: %v", err)
+			}
+		}
+
 		if changes.SubnetMappings != nil {
 			actualSubnets := make(map[string]*string)
 			for _, s := range a.SubnetMappings {