WIP Put control-plane and apiserver nodes in IPv6-only subnets

cloudmock/aws/mockelbv2/targetgroups.go

@@ -95,6 +95,7 @@ func (m *MockELBV2) CreateTargetGroup(request *elbv2.CreateTargetGroupInput) (*e
 
 	tg := elbv2.TargetGroup{
 		TargetGroupName:         request.Name,
+		IpAddressType:           request.IpAddressType,
 		Port:                    request.Port,
 		Protocol:                request.Protocol,
 		VpcId:                   request.VpcId,

docs/networking/ipv6.md

@@ -23,7 +23,7 @@ For example, if the VPC's CIDR is `2001:db8::/56` then the syntax `/64#a` would
 
 Public and utility subnets are expected to be dual-stack. Subnets of type `Private` are expected to be IPv6-only.
 There is a new type of subnet `DualStack` which is like `Private` but is dual-stack.
-The `DualStack` subnets are used by default for the control plane and APIServer nodes.
+Prior to kOps 1.29, `DualStack` subnets are used by default for bastion servers, the control plane, and APIServer nodes.
 
 IPv6-only subnets require Kubernetes 1.22 or later. For this reason, private topology on an IPv6 cluster also
 requires Kubernetes 1.22 or later.

docs/topology.md

@@ -37,7 +37,7 @@ NAT64 range `64:ff9b::/96` is typically routed to a NAT64 device, such as an AWS
 
 A subnet of type `DualStack` is like `Private`, but supports both IPv4 and IPv6.
 
-On AWS, this subnet type is used for nodes, such as control plane nodes and bastions,
+On AWS prior to kOps 1.29, this subnet type is used for nodes, such as control plane nodes and bastions,
 which need to be instance targets of a load balancer.
 
 ## Utility Subnet

pkg/apis/kops/validation/aws.go

@@ -47,6 +47,9 @@ func awsValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
 		if lbSpec.Class == kops.LoadBalancerClassNetwork && lbSpec.UseForInternalAPI && lbSpec.Type == kops.LoadBalancerTypeInternal {
 			allErrs = append(allErrs, field.Forbidden(lbPath.Child("useForInternalAPI"), "useForInternalAPI cannot be used with internal NLB due lack of hairpinning support"))
 		}
+		if lbSpec.Class == kops.LoadBalancerClassClassic && c.Spec.IsIPv6Only() {
+			allErrs = append(allErrs, field.Forbidden(lbPath.Child("class"), "IPv6 clusters do not support classic load balancers"))
+		}
 		if lbSpec.SSLCertificate != "" && lbSpec.Class != kops.LoadBalancerClassNetwork {
 			allErrs = append(allErrs, field.Forbidden(lbPath.Child("sslCertificate"), "sslCertificate requires a network load balancer. See https://github.com/kubernetes/kops/blob/master/permalinks/acm_nlb.md"))
 		}

pkg/model/awsmodel/api_loadbalancer.go

@@ -270,6 +270,11 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			}
 		}
 
+		ipAddressType := "ipv4"
+		if b.Cluster.Spec.IsIPv6Only() {
+			ipAddressType = "ipv6"
+		}
+
 		if b.APILoadBalancerClass() == kops.LoadBalancerClassClassic {
 			c.AddTask(clb)
 		} else if b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork {
@@ -290,6 +295,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 					Lifecycle:          b.Lifecycle,
 					VPC:                b.LinkToVPC(),
 					Tags:               groupTags,
+					IPAddressType:      fi.PtrTo(ipAddressType),
 					Protocol:           fi.PtrTo("TCP"),
 					Port:               fi.PtrTo(int64(443)),
 					Attributes:         groupAttrs,
@@ -316,6 +322,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 					Lifecycle:          b.Lifecycle,
 					VPC:                b.LinkToVPC(),
 					Tags:               groupTags,
+					IPAddressType:      fi.PtrTo(ipAddressType),
 					Protocol:           fi.PtrTo("TCP"),
 					Port:               fi.PtrTo(int64(wellknownports.KopsControllerPort)),
 					Attributes:         groupAttrs,
@@ -341,6 +348,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 					Lifecycle:          b.Lifecycle,
 					VPC:                b.LinkToVPC(),
 					Tags:               tlsGroupTags,
+					IPAddressType:      fi.PtrTo(ipAddressType),
 					Protocol:           fi.PtrTo("TLS"),
 					Port:               fi.PtrTo(int64(443)),
 					Attributes:         groupAttrs,
@@ -517,24 +525,6 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 				SourceGroup:   lbSG,
 				ToPort:        fi.PtrTo(int64(443)),
 			})
-			c.AddTask(&awstasks.SecurityGroupRule{
-				Name:          fi.PtrTo(fmt.Sprintf("icmp-pmtu-elb-to-cp%s", suffix)),
-				Lifecycle:     b.SecurityLifecycle,
-				FromPort:      fi.PtrTo(int64(3)),
-				Protocol:      fi.PtrTo("icmp"),
-				SecurityGroup: masterGroup.Task,
-				SourceGroup:   lbSG,
-				ToPort:        fi.PtrTo(int64(4)),
-			})
-			c.AddTask(&awstasks.SecurityGroupRule{
-				Name:          fi.PtrTo(fmt.Sprintf("icmp-pmtu-cp%s-to-elb", suffix)),
-				Lifecycle:     b.SecurityLifecycle,
-				FromPort:      fi.PtrTo(int64(3)),
-				Protocol:      fi.PtrTo("icmp"),
-				SecurityGroup: lbSG,
-				SourceGroup:   masterGroup.Task,
-				ToPort:        fi.PtrTo(int64(4)),
-			})
 			if b.Cluster.UsesNoneDNS() {
 				c.AddTask(&awstasks.SecurityGroupRule{
 					Name:          fi.PtrTo(fmt.Sprintf("kops-controller-elb-to-cp%s", suffix)),
@@ -546,6 +536,45 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 					SourceGroup:   lbSG,
 				})
 			}
+			if b.Cluster.Spec.IsIPv6Only() {
+				c.AddTask(&awstasks.SecurityGroupRule{
+					Name:          fi.PtrTo(fmt.Sprintf("icmpv6-pmtu-elb-to-cp%s", suffix)),
+					Lifecycle:     b.SecurityLifecycle,
+					FromPort:      fi.PtrTo(int64(-1)),
+					Protocol:      fi.PtrTo("icmpv6"),
+					SecurityGroup: masterGroup.Task,
+					SourceGroup:   lbSG,
+					ToPort:        fi.PtrTo(int64(-1)),
+				})
+				c.AddTask(&awstasks.SecurityGroupRule{
+					Name:          fi.PtrTo(fmt.Sprintf("icmpv6-pmtu-cp%s-to-elb", suffix)),
+					Lifecycle:     b.SecurityLifecycle,
+					FromPort:      fi.PtrTo(int64(-1)),
+					Protocol:      fi.PtrTo("icmpv6"),
+					SecurityGroup: lbSG,
+					SourceGroup:   masterGroup.Task,
+					ToPort:        fi.PtrTo(int64(-1)),
+				})
+			} else {
+				c.AddTask(&awstasks.SecurityGroupRule{
+					Name:          fi.PtrTo(fmt.Sprintf("icmp-pmtu-elb-to-cp%s", suffix)),
+					Lifecycle:     b.SecurityLifecycle,
+					FromPort:      fi.PtrTo(int64(3)),
+					Protocol:      fi.PtrTo("icmp"),
+					SecurityGroup: masterGroup.Task,
+					SourceGroup:   lbSG,
+					ToPort:        fi.PtrTo(int64(4)),
+				})
+				c.AddTask(&awstasks.SecurityGroupRule{
+					Name:          fi.PtrTo(fmt.Sprintf("icmp-pmtu-cp%s-to-elb", suffix)),
+					Lifecycle:     b.SecurityLifecycle,
+					FromPort:      fi.PtrTo(int64(3)),
+					Protocol:      fi.PtrTo("icmp"),
+					SecurityGroup: lbSG,
+					SourceGroup:   masterGroup.Task,
+					ToPort:        fi.PtrTo(int64(4)),
+				})
+			}
 		}
 	}
 

pkg/model/awsmodel/bastion.go

@@ -288,31 +288,60 @@ func (b *BastionModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			}
 			AddDirectionalGroupRule(c, t)
 		}
-		{
-			suffix := bastionGroup.Suffix
-			t := &awstasks.SecurityGroupRule{
-				Name:          fi.PtrTo(fmt.Sprintf("icmp-to-bastion%s", suffix)),
-				Lifecycle:     b.SecurityLifecycle,
-				SecurityGroup: bastionGroup.Task,
-				SourceGroup:   lbSG,
-				Protocol:      fi.PtrTo("icmp"),
-				FromPort:      fi.PtrTo(int64(3)),
-				ToPort:        fi.PtrTo(int64(4)),
+		if useIPv6ForBastion(b) {
+			{
+				suffix := bastionGroup.Suffix
+				t := &awstasks.SecurityGroupRule{
+					Name:          fi.PtrTo(fmt.Sprintf("icmpv6-to-bastion%s", suffix)),
+					Lifecycle:     b.SecurityLifecycle,
+					SecurityGroup: bastionGroup.Task,
+					SourceGroup:   lbSG,
+					Protocol:      fi.PtrTo("icmpv6"),
+					FromPort:      fi.PtrTo(int64(-1)),
+					ToPort:        fi.PtrTo(int64(-1)),
+				}
+				AddDirectionalGroupRule(c, t)
 			}
-			AddDirectionalGroupRule(c, t)
-		}
-		{
-			suffix := bastionGroup.Suffix
-			t := &awstasks.SecurityGroupRule{
-				Name:          fi.PtrTo(fmt.Sprintf("icmp-from-bastion%s", suffix)),
-				Lifecycle:     b.SecurityLifecycle,
-				SecurityGroup: lbSG,
-				SourceGroup:   bastionGroup.Task,
-				Protocol:      fi.PtrTo("icmp"),
-				FromPort:      fi.PtrTo(int64(3)),
-				ToPort:        fi.PtrTo(int64(4)),
+			{
+				suffix := bastionGroup.Suffix
+				t := &awstasks.SecurityGroupRule{
+					Name:          fi.PtrTo(fmt.Sprintf("icmpv6-from-bastion%s", suffix)),
+					Lifecycle:     b.SecurityLifecycle,
+					SecurityGroup: lbSG,
+					SourceGroup:   bastionGroup.Task,
+					Protocol:      fi.PtrTo("icmpv6"),
+					FromPort:      fi.PtrTo(int64(-1)),
+					ToPort:        fi.PtrTo(int64(-1)),
+				}
+				AddDirectionalGroupRule(c, t)
+			}
+		} else {
+			{
+				suffix := bastionGroup.Suffix
+				t := &awstasks.SecurityGroupRule{
+					Name:          fi.PtrTo(fmt.Sprintf("icmp-to-bastion%s", suffix)),
+					Lifecycle:     b.SecurityLifecycle,
+					SecurityGroup: bastionGroup.Task,
+					SourceGroup:   lbSG,
+					Protocol:      fi.PtrTo("icmp"),
+					FromPort:      fi.PtrTo(int64(3)),
+					ToPort:        fi.PtrTo(int64(4)),
+				}
+				AddDirectionalGroupRule(c, t)
+			}
+			{
+				suffix := bastionGroup.Suffix
+				t := &awstasks.SecurityGroupRule{
+					Name:          fi.PtrTo(fmt.Sprintf("icmp-from-bastion%s", suffix)),
+					Lifecycle:     b.SecurityLifecycle,
+					SecurityGroup: lbSG,
+					SourceGroup:   bastionGroup.Task,
+					Protocol:      fi.PtrTo("icmp"),
+					FromPort:      fi.PtrTo(int64(3)),
+					ToPort:        fi.PtrTo(int64(4)),
+				}
+				AddDirectionalGroupRule(c, t)
 			}
-			AddDirectionalGroupRule(c, t)
 		}
 	}
 
@@ -381,6 +410,7 @@ func (b *BastionModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			Lifecycle:          b.Lifecycle,
 			VPC:                b.LinkToVPC(),
 			Tags:               sshGroupTags,
+			IPAddressType:      fi.PtrTo("ipv4"),
 			Protocol:           fi.PtrTo("TCP"),
 			Port:               fi.PtrTo(int64(22)),
 			Attributes:         groupAttrs,
@@ -389,6 +419,9 @@ func (b *BastionModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			UnhealthyThreshold: fi.PtrTo(int64(2)),
 			Shared:             fi.PtrTo(false),
 		}
+		if useIPv6ForBastion(b) {
+			tg.IPAddressType = fi.PtrTo("ipv6")
+		}
 
 		c.AddTask(tg)
 

tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml

@@ -87,7 +87,7 @@ spec:
   minSize: 1
   role: Master
   subnets:
-  - dualstack-us-test-1a
+  - us-test-1a
 
 ---
 

tests/integration/update_cluster/bastionadditional_user-data/kubernetes.tf

@@ -803,9 +803,10 @@ resource "aws_lb_target_group" "bastion-bastionuserdata-e-4grhsv" {
     protocol            = "TCP"
     unhealthy_threshold = 2
   }
-  name     = "bastion-bastionuserdata-e-4grhsv"
-  port     = 22
-  protocol = "TCP"
+  ip_address_type = "ipv4"
+  name            = "bastion-bastionuserdata-e-4grhsv"
+  port            = 22
+  protocol        = "TCP"
   tags = {
     "KubernetesCluster"                                 = "bastionuserdata.example.com"
     "Name"                                              = "bastion-bastionuserdata-e-4grhsv"

tests/integration/update_cluster/complex/kubernetes.tf

@@ -683,9 +683,10 @@ resource "aws_lb_target_group" "tcp-complex-example-com-vpjolq" {
     protocol            = "TCP"
     unhealthy_threshold = 2
   }
-  name     = "tcp-complex-example-com-vpjolq"
-  port     = 443
-  protocol = "TCP"
+  ip_address_type = "ipv4"
+  name            = "tcp-complex-example-com-vpjolq"
+  port            = 443
+  protocol        = "TCP"
   tags = {
     "KubernetesCluster"                         = "complex.example.com"
     "Name"                                      = "tcp-complex-example-com-vpjolq"
@@ -705,9 +706,10 @@ resource "aws_lb_target_group" "tls-complex-example-com-5nursn" {
     protocol            = "TCP"
     unhealthy_threshold = 2
   }
-  name     = "tls-complex-example-com-5nursn"
-  port     = 443
-  protocol = "TLS"
+  ip_address_type = "ipv4"
+  name            = "tls-complex-example-com-5nursn"
+  port            = 443
+  protocol        = "TLS"
   tags = {
     "KubernetesCluster"                         = "complex.example.com"
     "Name"                                      = "tls-complex-example-com-5nursn"

tests/integration/update_cluster/minimal-dns-none/kubernetes.tf

@@ -612,9 +612,10 @@ resource "aws_lb_target_group" "kops-controller-minimal-e-uvauf3" {
     protocol            = "TCP"
     unhealthy_threshold = 2
   }
-  name     = "kops-controller-minimal-e-uvauf3"
-  port     = 3988
-  protocol = "TCP"
+  ip_address_type = "ipv4"
+  name            = "kops-controller-minimal-e-uvauf3"
+  port            = 3988
+  protocol        = "TCP"
   tags = {
     "KubernetesCluster"                         = "minimal.example.com"
     "Name"                                      = "kops-controller-minimal-e-uvauf3"
@@ -632,9 +633,10 @@ resource "aws_lb_target_group" "tcp-minimal-example-com-5905t8" {
     protocol            = "TCP"
     unhealthy_threshold = 2
   }
-  name     = "tcp-minimal-example-com-5905t8"
-  port     = 443
-  protocol = "TCP"
+  ip_address_type = "ipv4"
+  name            = "tcp-minimal-example-com-5905t8"
+  port            = 443
+  protocol        = "TCP"
   tags = {
     "KubernetesCluster"                         = "minimal.example.com"
     "Name"                                      = "tcp-minimal-example-com-5905t8"

tests/integration/update_cluster/minimal-ipv6-calico/kubernetes.tf

@@ -652,9 +652,10 @@ resource "aws_lb_target_group" "tcp-minimal-ipv6-example--bne5ih" {
     protocol            = "TCP"
     unhealthy_threshold = 2
   }
-  name     = "tcp-minimal-ipv6-example--bne5ih"
-  port     = 443
-  protocol = "TCP"
+  ip_address_type = "ipv6"
+  name            = "tcp-minimal-ipv6-example--bne5ih"
+  port            = 443
+  protocol        = "TCP"
   tags = {
     "KubernetesCluster"                              = "minimal-ipv6.example.com"
     "Name"                                           = "tcp-minimal-ipv6-example--bne5ih"
@@ -1212,33 +1213,33 @@ resource "aws_security_group_rule" "icmp-pmtu-api-elb-0-0-0-0--0" {
   type              = "ingress"
 }
 
-resource "aws_security_group_rule" "icmp-pmtu-cp-to-elb" {
-  from_port                = 3
-  protocol                 = "icmp"
+resource "aws_security_group_rule" "icmpv6-pmtu-api-elb-__--0" {
+  from_port         = -1
+  ipv6_cidr_blocks  = ["::/0"]
+  protocol          = "icmpv6"
+  security_group_id = aws_security_group.api-elb-minimal-ipv6-example-com.id
+  to_port           = -1
+  type              = "ingress"
+}
+
+resource "aws_security_group_rule" "icmpv6-pmtu-cp-to-elb" {
+  from_port                = -1
+  protocol                 = "icmpv6"
   security_group_id        = aws_security_group.api-elb-minimal-ipv6-example-com.id
   source_security_group_id = aws_security_group.masters-minimal-ipv6-example-com.id
-  to_port                  = 4
+  to_port                  = -1
   type                     = "ingress"
 }
 
-resource "aws_security_group_rule" "icmp-pmtu-elb-to-cp" {
-  from_port                = 3
-  protocol                 = "icmp"
+resource "aws_security_group_rule" "icmpv6-pmtu-elb-to-cp" {
+  from_port                = -1
+  protocol                 = "icmpv6"
   security_group_id        = aws_security_group.masters-minimal-ipv6-example-com.id
   source_security_group_id = aws_security_group.api-elb-minimal-ipv6-example-com.id
-  to_port                  = 4
+  to_port                  = -1
   type                     = "ingress"
 }
 
-resource "aws_security_group_rule" "icmpv6-pmtu-api-elb-__--0" {
-  from_port         = -1
-  ipv6_cidr_blocks  = ["::/0"]
-  protocol          = "icmpv6"
-  security_group_id = aws_security_group.api-elb-minimal-ipv6-example-com.id
-  to_port           = -1
-  type              = "ingress"
-}
-
 resource "aws_sqs_queue" "minimal-ipv6-example-com-nth" {
   message_retention_seconds = 300
   name                      = "minimal-ipv6-example-com-nth"