Stop using DualStack subnets by default

docs/networking/ipv6.md

@@ -23,7 +23,7 @@ For example, if the VPC's CIDR is `2001:db8::/56` then the syntax `/64#a` would
 
 Public and utility subnets are expected to be dual-stack. Subnets of type `Private` are expected to be IPv6-only.
 There is a new type of subnet `DualStack` which is like `Private` but is dual-stack.
-The `DualStack` subnets are used by default for the control plane and APIServer nodes.
+Prior to kOps 1.29, `DualStack` subnets are used by default for bastion servers, the control plane, and APIServer nodes.
 
 IPv6-only subnets require Kubernetes 1.22 or later. For this reason, private topology on an IPv6 cluster also
 requires Kubernetes 1.22 or later.

docs/topology.md

@@ -37,7 +37,7 @@ NAT64 range `64:ff9b::/96` is typically routed to a NAT64 device, such as an AWS
 
 A subnet of type `DualStack` is like `Private`, but supports both IPv4 and IPv6.
 
-On AWS, this subnet type is used for nodes, such as control plane nodes and bastions,
+On AWS prior to kOps 1.29, this subnet type is used for nodes, such as control plane nodes and bastions,
 which need to be instance targets of a load balancer.
 
 ## Utility Subnet

pkg/apis/kops/validation/aws.go

@@ -47,6 +47,9 @@ func awsValidateCluster(c *kops.Cluster, strict bool) field.ErrorList {
 		if lbSpec.Class == kops.LoadBalancerClassNetwork && lbSpec.UseForInternalAPI && lbSpec.Type == kops.LoadBalancerTypeInternal {
 			allErrs = append(allErrs, field.Forbidden(lbPath.Child("useForInternalAPI"), "useForInternalAPI cannot be used with internal NLB due lack of hairpinning support"))
 		}
+		if lbSpec.Class == kops.LoadBalancerClassClassic && c.Spec.IsIPv6Only() {
+			allErrs = append(allErrs, field.Forbidden(lbPath.Child("class"), "IPv6 clusters do not support classic load balancers"))
+		}
 		if lbSpec.SSLCertificate != "" && lbSpec.Class != kops.LoadBalancerClassNetwork {
 			allErrs = append(allErrs, field.Forbidden(lbPath.Child("sslCertificate"), "sslCertificate requires a network load balancer. See https://github.com/kubernetes/kops/blob/master/permalinks/acm_nlb.md"))
 		}

pkg/model/awsmodel/api_loadbalancer.go

@@ -269,6 +269,11 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			}
 		}
 
+		ipAddressType := "ipv4"
+		if b.Cluster.Spec.IsIPv6Only() {
+			ipAddressType = "ipv6"
+		}
+
 		if b.APILoadBalancerClass() == kops.LoadBalancerClassClassic {
 			c.AddTask(clb)
 		} else if b.APILoadBalancerClass() == kops.LoadBalancerClassNetwork {
@@ -289,6 +294,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 					Lifecycle:          b.Lifecycle,
 					VPC:                b.LinkToVPC(),
 					Tags:               groupTags,
+					IPAddressType:      fi.PtrTo(ipAddressType),
 					Protocol:           fi.PtrTo("TCP"),
 					Port:               fi.PtrTo(int64(443)),
 					Attributes:         groupAttrs,
@@ -315,6 +321,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 					Lifecycle:          b.Lifecycle,
 					VPC:                b.LinkToVPC(),
 					Tags:               groupTags,
+					IPAddressType:      fi.PtrTo(ipAddressType),
 					Protocol:           fi.PtrTo("TCP"),
 					Port:               fi.PtrTo(int64(wellknownports.KopsControllerPort)),
 					Attributes:         groupAttrs,
@@ -340,6 +347,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 					Lifecycle:          b.Lifecycle,
 					VPC:                b.LinkToVPC(),
 					Tags:               tlsGroupTags,
+					IPAddressType:      fi.PtrTo(ipAddressType),
 					Protocol:           fi.PtrTo("TLS"),
 					Port:               fi.PtrTo(int64(443)),
 					Attributes:         groupAttrs,

pkg/model/awsmodel/bastion.go

@@ -299,6 +299,7 @@ func (b *BastionModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			Lifecycle:          b.Lifecycle,
 			VPC:                b.LinkToVPC(),
 			Tags:               sshGroupTags,
+			IPAddressType:      fi.PtrTo("ipv4"),
 			Protocol:           fi.PtrTo("TCP"),
 			Port:               fi.PtrTo(int64(22)),
 			Attributes:         groupAttrs,
@@ -307,6 +308,9 @@ func (b *BastionModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			UnhealthyThreshold: fi.PtrTo(int64(2)),
 			Shared:             fi.PtrTo(false),
 		}
+		if useIPv6ForBastion(b) {
+			tg.IPAddressType = fi.PtrTo("ipv6")
+		}
 
 		c.AddTask(tg)
 

tests/integration/create_cluster/ipv6/expected-v1alpha2.yaml

@@ -87,7 +87,7 @@ spec:
   minSize: 1
   role: Master
   subnets:
-  - dualstack-us-test-1a
+  - us-test-1a
 
 ---
 

upup/pkg/fi/cloudup/awstasks/targetgroup.go

@@ -41,12 +41,13 @@ const (
 
 // +kops:fitask
 type TargetGroup struct {
-	Name      *string
-	Lifecycle fi.Lifecycle
-	VPC       *VPC
-	Tags      map[string]string
-	Port      *int64
-	Protocol  *string
+	Name          *string
+	Lifecycle     fi.Lifecycle
+	VPC           *VPC
+	Tags          map[string]string
+	IPAddressType *string
+	Port          *int64
+	Protocol      *string
 
 	// ARN is the Amazon Resource Name for the Target Group
 	ARN *string
@@ -97,6 +98,7 @@ func (e *TargetGroup) Find(c *fi.CloudupContext) (*TargetGroup, error) {
 
 	actual := &TargetGroup{
 		Name:               tg.TargetGroupName,
+		IPAddressType:      tg.IpAddressType,
 		Port:               tg.Port,
 		Protocol:           tg.Protocol,
 		ARN:                tg.TargetGroupArn,
@@ -168,13 +170,17 @@ func (_ *TargetGroup) RenderAWS(t *awsup.AWSAPITarget, a, e, changes *TargetGrou
 		return nil
 	}
 
+	// TODO: To fix the IPAddressType we need to recreate the TargetGroup.
+	// We can't delete the existing TargetGroup until it's unreferenced by the listener.
+
 	// You register targets for your Network Load Balancer with a target group. By default, the load balancer sends requests
 	// to registered targets using the port and protocol that you specified for the target group. You can override this port
 	// when you register each target with the target group.
 
 	if a == nil {
 		request := &elbv2.CreateTargetGroupInput{
 			Name:                       e.Name,
+			IpAddressType:              e.IPAddressType,
 			Port:                       e.Port,
 			Protocol:                   e.Protocol,
 			VpcId:                      e.VPC.ID,
@@ -239,6 +245,7 @@ func (a OrderTargetGroupsByName) Less(i, j int) bool {
 
 type terraformTargetGroup struct {
 	Name                  string                          `cty:"name"`
+	IPAddressType         string                          `cty:"ip_address_type"`
 	Port                  int64                           `cty:"port"`
 	Protocol              string                          `cty:"protocol"`
 	VPCID                 *terraformWriter.Literal        `cty:"vpc_id"`
@@ -266,11 +273,12 @@ func (_ *TargetGroup) RenderTerraform(t *terraform.TerraformTarget, a, e, change
 	}
 
 	tf := &terraformTargetGroup{
-		Name:     *e.Name,
-		Port:     *e.Port,
-		Protocol: *e.Protocol,
-		VPCID:    e.VPC.TerraformLink(),
-		Tags:     e.Tags,
+		Name:          *e.Name,
+		IPAddressType: *e.IPAddressType,
+		Port:          *e.Port,
+		Protocol:      *e.Protocol,
+		VPCID:         e.VPC.TerraformLink(),
+		Tags:          e.Tags,
 		HealthCheck: terraformTargetGroupHealthCheck{
 			Interval:           *e.Interval,
 			HealthyThreshold:   *e.HealthyThreshold,

upup/pkg/fi/cloudup/new_cluster.go

@@ -493,14 +493,6 @@ func NewCluster(opt *NewClusterOptions, clientset simple.Clientset) (*NewCluster
 			if len(ig.Spec.Subnets) == 0 {
 				return nil, fmt.Errorf("control-plane InstanceGroup %s did not specify any Subnets", g.ObjectMeta.Name)
 			}
-		} else if ig.IsAPIServerOnly() && cluster.Spec.IsIPv6Only() {
-			if len(ig.Spec.Subnets) == 0 {
-				for _, subnet := range cluster.Spec.Networking.Subnets {
-					if subnet.Type != api.SubnetTypePrivate && subnet.Type != api.SubnetTypeUtility {
-						ig.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
-					}
-				}
-			}
 		} else {
 			if len(ig.Spec.Subnets) == 0 {
 				for _, subnet := range cluster.Spec.Networking.Subnets {
@@ -903,11 +895,7 @@ func setupControlPlane(opt *NewClusterOptions, cluster *api.Cluster, zoneToSubne
 			default:
 				// Use only the main subnet for control-plane nodes
 				subnet := subnets[0]
-				if opt.IPv6 && opt.Topology == api.TopologyPrivate {
-					g.Spec.Subnets = append(g.Spec.Subnets, "dualstack-"+subnet.Name)
-				} else {
-					g.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
-				}
+				g.Spec.Subnets = append(g.Spec.Subnets, subnet.Name)
 			}
 
 			if cloudProvider == api.CloudProviderGCE || cloudProvider == api.CloudProviderAzure {

upup/pkg/fi/cloudup/populate_instancegroup_spec.go

@@ -157,14 +157,6 @@ func PopulateInstanceGroupSpec(cluster *kops.Cluster, input *kops.InstanceGroup,
 		if len(ig.Spec.Subnets) == 0 {
 			return nil, fmt.Errorf("control-plane InstanceGroup %s did not specify any Subnets", ig.ObjectMeta.Name)
 		}
-	} else if ig.IsAPIServerOnly() && cluster.Spec.IsIPv6Only() {
-		if len(ig.Spec.Subnets) == 0 {
-			for _, subnet := range cluster.Spec.Networking.Subnets {
-				if subnet.Type != kops.SubnetTypePrivate && subnet.Type != kops.SubnetTypeUtility {
-					ig.Spec.Subnets = append(ig.Spec.Subnets, subnet.Name)
-				}
-			}
-		}
 	} else {
 		if len(ig.Spec.Subnets) == 0 {
 			for _, subnet := range cluster.Spec.Networking.Subnets {