Merge pull request #10907 from hakman/automated-cherry-pick-of-#10872…

…-upstream-release-1.20 Automated cherry pick of #10872: Adding Elastic IP Allocations to NLB API
kubernetes · Feb 23, 2021 · dee18ce · dee18ce
2 parents 496f2d8 + 3eacca4
commit dee18ce
Show file tree

Hide file tree

Showing 17 changed files with 149 additions and 14 deletions.
diff --git a/cloudmock/aws/mockelbv2/loadbalancers.go b/cloudmock/aws/mockelbv2/loadbalancers.go
@@ -97,6 +97,9 @@ func (m *MockELBV2) CreateLoadBalancer(request *elbv2.CreateLoadBalancerInput) (
 		if subnetMapping.PrivateIPv4Address != nil {
 			lbAddrs = append(lbAddrs, &elbv2.LoadBalancerAddress{PrivateIPv4Address: subnetMapping.PrivateIPv4Address})
 		}
+		if subnetMapping.AllocationId != nil {
+			lbAddrs = append(lbAddrs, &elbv2.LoadBalancerAddress{AllocationId: subnetMapping.AllocationId})
+		}
 		zones = append(zones, &elbv2.AvailabilityZone{
 			SubnetId:              subnetMapping.SubnetId,
 			LoadBalancerAddresses: lbAddrs,

diff --git a/docs/cluster_spec.md b/docs/cluster_spec.md
@@ -137,6 +137,19 @@ spec:
 
 The specified IPv4 addresses must be part of the subnets CIDR. They can not be changed after initial deployment.
 
+If the `type` is `Public` and the `class` is `Network`, you can also specify an Elastic IP allocationID to bind a fixed public IP address per subnet. Pleae note only IPv4 addresses have been tested:
+```yaml
+spec:
+  api:
+    loadBalancer:
+      type: Public
+      subnets:
+        - name: utility-subnet-a
+          allocationID: eipalloc-222ghi789
+```
+
+The specified Allocation ID's must already be created manually or external infrastructure as code, eg Terraform. You will need to place the loadBalanacer in the utility subnets for external connectivity.
+
 If you made a mistake or need to change subnets for any other reason, you're currently forced to manually delete the
 underlying ELB/NLB and re-run `kops update`.
 

diff --git a/k8s/crds/kops.k8s.io_clusters.yaml b/k8s/crds/kops.k8s.io_clusters.yaml
@@ -119,6 +119,10 @@ spec:
                           description: LoadBalancerSubnetSpec provides configuration
                             for subnets used for a load balancer
                           properties:
+                            allocationId:
+                              description: AllocationID specifies the Elastic IP Allocation
+                                ID for use by a NLB
+                              type: string
                             name:
                               description: Name specifies the name of the cluster
                                 subnet

diff --git a/pkg/apis/kops/cluster.go b/pkg/apis/kops/cluster.go
@@ -383,6 +383,8 @@ type LoadBalancerSubnetSpec struct {
 	Name string `json:"name,omitempty"`
 	// PrivateIPv4Address specifies the private IPv4 address to use for a NLB
 	PrivateIPv4Address *string `json:"privateIPv4Address,omitempty"`
+	// AllocationID specifies the Elastic IP Allocation ID for use by a NLB
+	AllocationID *string `json:"allocationId,omitempty"`
 }
 
 // LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and its access

diff --git a/pkg/apis/kops/v1alpha2/cluster.go b/pkg/apis/kops/v1alpha2/cluster.go
@@ -385,6 +385,8 @@ type LoadBalancerSubnetSpec struct {
 	Name string `json:"name,omitempty"`
 	// PrivateIPv4Address specifies the private IPv4 address to use for a NLB
 	PrivateIPv4Address *string `json:"privateIPv4Address,omitempty"`
+	// AllocationID specifies the Elastic IP Allocation ID for use by a NLB
+	AllocationID *string `json:"allocationId,omitempty"`
 }
 
 // LoadBalancerAccessSpec provides configuration details related to API LoadBalancer and its access

diff --git a/pkg/apis/kops/v1alpha2/zz_generated.conversion.go b/pkg/apis/kops/v1alpha2/zz_generated.conversion.go
diff --git a/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go b/pkg/apis/kops/v1alpha2/zz_generated.deepcopy.go
diff --git a/pkg/apis/kops/validation/aws.go b/pkg/apis/kops/validation/aws.go
@@ -263,6 +263,16 @@ func awsValidateLoadBalancerSubnets(fieldPath *field.Path, spec kops.ClusterSpec
 				allErrs = append(allErrs, field.Forbidden(fieldPath.Index(i).Child("privateIPv4Address"), "privateIPv4Address only allowed for internal NLBs"))
 			}
 		}
+
+		if subnet.AllocationID != nil {
+			if *subnet.AllocationID == "" {
+				allErrs = append(allErrs, field.Required(fieldPath.Index(i).Child("allocationID"), "allocationID can't be empty"))
+			}
+
+			if lbSpec.Class != kops.LoadBalancerClassNetwork || lbSpec.Type == kops.LoadBalancerTypeInternal {
+				allErrs = append(allErrs, field.Forbidden(fieldPath.Index(i).Child("allocationID"), "allocationID only allowed for Public NLBs"))
+			}
+		}
 	}
 
 	return allErrs

diff --git a/pkg/apis/kops/validation/aws_test.go b/pkg/apis/kops/validation/aws_test.go
@@ -352,16 +352,18 @@ func TestLoadBalancerSubnets(t *testing.T) {
 		lbSubnets      []kops.LoadBalancerSubnetSpec
 		expected       []string
 	}{
-		{ // valid (no privateIPv4Address)
+		{ // valid (no privateIPv4Address, no allocationID)
 			clusterSubnets: []string{"a", "b", "c"},
 			lbSubnets: []kops.LoadBalancerSubnetSpec{
 				{
 					Name:               "a",
 					PrivateIPv4Address: nil,
+					AllocationID:       nil,
 				},
 				{
 					Name:               "b",
 					PrivateIPv4Address: nil,
+					AllocationID:       nil,
 				},
 			},
 		},
@@ -371,10 +373,12 @@ func TestLoadBalancerSubnets(t *testing.T) {
 				{
 					Name:               "a",
 					PrivateIPv4Address: fi.String("10.0.0.10"),
+					AllocationID:       nil,
 				},
 				{
 					Name:               "b",
 					PrivateIPv4Address: nil,
+					AllocationID:       nil,
 				},
 			},
 		},
@@ -384,6 +388,7 @@ func TestLoadBalancerSubnets(t *testing.T) {
 				{
 					Name:               "",
 					PrivateIPv4Address: nil,
+					AllocationID:       nil,
 				},
 			},
 			expected: []string{"Required value::spec.api.loadBalancer.subnets[0].name"},
@@ -394,62 +399,103 @@ func TestLoadBalancerSubnets(t *testing.T) {
 				{
 					Name:               "d",
 					PrivateIPv4Address: nil,
+					AllocationID:       nil,
 				},
 			},
 			expected: []string{"Not found::spec.api.loadBalancer.subnets[0].name"},
 		},
-		{ // empty privateIPv4Address
+		{ // empty privateIPv4Address, no allocationID
 			clusterSubnets: []string{"a", "b", "c"},
 			lbSubnets: []kops.LoadBalancerSubnetSpec{
 				{
 					Name:               "a",
 					PrivateIPv4Address: fi.String(""),
+					AllocationID:       nil,
 				},
 			},
 			expected: []string{"Required value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
 		},
-		{ // invalid privateIPv4Address
+		{ // empty no privateIPv4Address, with allocationID
+			clusterSubnets: []string{"a", "b", "c"},
+			lbSubnets: []kops.LoadBalancerSubnetSpec{
+				{
+					Name:               "a",
+					PrivateIPv4Address: nil,
+					AllocationID:       fi.String(""),
+				},
+			},
+			expected: []string{"Required value::spec.api.loadBalancer.subnets[0].allocationID"},
+		},
+		{ // invalid privateIPv4Address, no allocationID
 			clusterSubnets: []string{"a", "b", "c"},
 			lbSubnets: []kops.LoadBalancerSubnetSpec{
 				{
 					Name:               "a",
 					PrivateIPv4Address: fi.String("invalidip"),
+					AllocationID:       nil,
 				},
 			},
 			expected: []string{"Invalid value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
 		},
-		{ // privateIPv4Address not matching subnet cidr
+		{ // privateIPv4Address not matching subnet cidr, no allocationID
 			clusterSubnets: []string{"a", "b", "c"},
 			lbSubnets: []kops.LoadBalancerSubnetSpec{
 				{
 					Name:               "a",
 					PrivateIPv4Address: fi.String("11.0.0.10"),
+					AllocationID:       nil,
 				},
 			},
 			expected: []string{"Invalid value::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
 		},
-		{ // invalid class
+		{ // invalid class - with privateIPv4Address, no allocationID
 			class:          fi.String(string(kops.LoadBalancerClassClassic)),
 			clusterSubnets: []string{"a", "b", "c"},
 			lbSubnets: []kops.LoadBalancerSubnetSpec{
 				{
 					Name:               "a",
 					PrivateIPv4Address: fi.String("10.0.0.10"),
+					AllocationID:       nil,
 				},
 			},
 			expected: []string{"Forbidden::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
 		},
-		{ // invalid type
+		{ // invalid class - no privateIPv4Address, with allocationID
+			class:          fi.String(string(kops.LoadBalancerClassClassic)),
+			clusterSubnets: []string{"a", "b", "c"},
+			lbSubnets: []kops.LoadBalancerSubnetSpec{
+				{
+					Name:               "a",
+					PrivateIPv4Address: nil,
+					AllocationID:       fi.String("eipalloc-222ghi789"),
+				},
+			},
+			expected: []string{"Forbidden::spec.api.loadBalancer.subnets[0].allocationID"},
+		},
+		{ // invalid type external for private IP
 			lbType:         fi.String(string(kops.LoadBalancerTypePublic)),
 			clusterSubnets: []string{"a", "b", "c"},
 			lbSubnets: []kops.LoadBalancerSubnetSpec{
 				{
 					Name:               "a",
 					PrivateIPv4Address: fi.String("10.0.0.10"),
+					AllocationID:       nil,
 				},
 			},
 			expected: []string{"Forbidden::spec.api.loadBalancer.subnets[0].privateIPv4Address"},
 		},
+		{ // invalid type Internal for public IP
+			lbType:         fi.String(string(kops.LoadBalancerTypeInternal)),
+			clusterSubnets: []string{"a", "b", "c"},
+			lbSubnets: []kops.LoadBalancerSubnetSpec{
+				{
+					Name:               "a",
+					PrivateIPv4Address: nil,
+					AllocationID:       fi.String("eipalloc-222ghi789"),
+				},
+			},
+			expected: []string{"Forbidden::spec.api.loadBalancer.subnets[0].allocationID"},
+		},
 	}
 
 	for _, test := range tests {

diff --git a/pkg/apis/kops/zz_generated.deepcopy.go b/pkg/apis/kops/zz_generated.deepcopy.go
diff --git a/pkg/model/awsmodel/api_loadbalancer.go b/pkg/model/awsmodel/api_loadbalancer.go
@@ -79,6 +79,9 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {
 					if subnet.PrivateIPv4Address != nil {
 						nlbSubnetMapping.PrivateIPv4Address = subnet.PrivateIPv4Address
 					}
+					if subnet.AllocationID != nil {
+						nlbSubnetMapping.AllocationID = subnet.AllocationID
+					}
 					nlbSubnetMappings = append(nlbSubnetMappings, nlbSubnetMapping)
 					break
 				}

diff --git a/tests/integration/update_cluster/complex/cloudformation.json b/tests/integration/update_cluster/complex/cloudformation.json
@@ -1259,7 +1259,8 @@
           {
             "SubnetId": {
               "Ref": "AWSEC2Subnetustest1acomplexexamplecom"
-            }
+            },
+            "AllocationId": "eipalloc-012345a678b9cdefa"
           }
         ],
         "Type": "network",

diff --git a/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-legacy-v1alpha2.yaml
@@ -14,6 +14,9 @@ spec:
       class: Network
       sslCertificate: arn:aws:acm:us-test-1:000000000000:certificate/123456789012-1234-1234-1234-12345678
       sslPolicy: ELBSecurityPolicy-2016-08
+      subnets:
+        - name: us-test-1a
+          allocationId: eipalloc-012345a678b9cdefa
   kubernetesApiAccess:
   - 1.1.1.0/24
   - 2001:0:8500::/40

diff --git a/tests/integration/update_cluster/complex/in-v1alpha2.yaml b/tests/integration/update_cluster/complex/in-v1alpha2.yaml
@@ -13,7 +13,10 @@ spec:
       crossZoneLoadBalancing: true
       class: Network
       sslCertificate: arn:aws:acm:us-test-1:000000000000:certificate/123456789012-1234-1234-1234-12345678
-      sslPolicy: ELBSecurityPolicy-2016-08 
+      sslPolicy: ELBSecurityPolicy-2016-08
+      subnets:
+        - name: us-test-1a
+          allocationId: eipalloc-012345a678b9cdefa
   kubernetesApiAccess:
   - 1.1.1.0/24
   - 2001:0:8500::/40

diff --git a/tests/integration/update_cluster/complex/kubernetes.tf b/tests/integration/update_cluster/complex/kubernetes.tf
@@ -533,7 +533,8 @@ resource "aws_lb" "api-complex-example-com" {
   load_balancer_type               = "network"
   name                             = "api-complex-example-com-vd3t5n"
   subnet_mapping {
-    subnet_id = aws_subnet.us-test-1a-complex-example-com.id
+    allocation_id = "eipalloc-012345a678b9cdefa"
+    subnet_id     = aws_subnet.us-test-1a-complex-example-com.id
   }
   tags = {
     "KubernetesCluster"                         = "complex.example.com"