pkg/aws/eni: new subnet-ids parameter

Documentation/concepts/networking/ipam/eni.rst

@@ -207,6 +207,15 @@ allocation:
 
   If unspecified, the security group ids of ``eth0`` will be used.
 
+``spec.eni.subnet-ids``
+  The subnet IDs used to select the AWS subnets for IP allocation. This is an
+  additional requirement on top of requiring to match the availability zone and
+  VPC of the instance. This parameter is mutually exclusive and has priority over
+  ``spec.eni.subnet-tags``.
+
+  If unspecified, it will let the operator pick any available subnet in the AZ 
+  with the most IP addresses available.
+
 ``spec.eni.subnet-tags``
   The tags used to select the AWS subnets for IP allocation. This is an
   additional requirement on top of requiring to match the availability zone and

pkg/aws/eni/instances.go

@@ -105,6 +105,30 @@ func (m *InstancesManager) GetSubnets(ctx context.Context) ipamTypes.SubnetMap {
 	return subnetsCopy
 }
 
+// FindSubnetByIDs returns the subnet with the most addresses matching VPC ID,
+// availability zone within a provided list of subnet ids
+//
+// The returned subnet is immutable so it can be safely accessed
+func (m *InstancesManager) FindSubnetByIDs(vpcID, availabilityZone string, subnetIDs []string) (bestSubnet *ipamTypes.Subnet) {
+	m.mutex.RLock()
+	defer m.mutex.RUnlock()
+
+	for _, s := range m.subnets {
+		if s.VirtualNetworkID == vpcID && s.AvailabilityZone == availabilityZone {
+			for _, subnetID := range subnetIDs {
+				if s.ID == subnetID {
+					if bestSubnet == nil || bestSubnet.AvailableAddresses < s.AvailableAddresses {
+						bestSubnet = s
+					}
+					continue
+				}
+			}
+		}
+	}
+
+	return
+}
+
 // FindSubnetByTags returns the subnet with the most addresses matching VPC ID,
 // availability zone and all required tags
 //

pkg/aws/eni/instances_test.go

@@ -237,6 +237,45 @@ func (e *ENISuite) TestGetSubnet(c *check.C) {
 	c.Assert(subnet3.ID, check.Equals, "subnet-3")
 }
 
+func (e *ENISuite) TestFindSubnetByIDs(c *check.C) {
+	api := ec2mock.NewAPI(subnets2, vpcs, securityGroups)
+	c.Assert(api, check.Not(check.IsNil))
+
+	mngr := NewInstancesManager(api)
+	c.Assert(mngr, check.Not(check.IsNil))
+
+	iteration1(api, mngr)
+	iteration2(api, mngr)
+
+	// exact match subnet-1
+	s := mngr.FindSubnetByIDs("vpc-1", "us-west-1", []string{"subnet-1"})
+	c.Assert(s.ID, check.Equals, "subnet-1")
+
+	// exact match subnet-2
+	s = mngr.FindSubnetByIDs("vpc-2", "us-east-1", []string{"subnet-2"})
+	c.Assert(s.ID, check.Equals, "subnet-2")
+
+	// exact match subnet-3
+	s = mngr.FindSubnetByIDs("vpc-1", "us-west-1", []string{"subnet-3"})
+	c.Assert(s.ID, check.Equals, "subnet-3")
+
+	// empty list shall return nil
+	c.Assert(mngr.FindSubnetByIDs("vpc-1", "us-west-1", []string{}), check.IsNil)
+
+	// all subnet match, subnet-1 has more addresses
+	s = mngr.FindSubnetByIDs("vpc-1", "us-west-1", []string{"subnet-1", "subnet-3"})
+	c.Assert(s.ID, check.Equals, "subnet-1")
+
+	// invalid vpc, no match
+	c.Assert(mngr.FindSubnetByIDs("vpc-unknown", "us-west-1", []string{"subnet-1"}), check.IsNil)
+
+	// invalid AZ, no match
+	c.Assert(mngr.FindSubnetByIDs("vpc-1", "us-west-unknown", []string{"subnet-1"}), check.IsNil)
+
+	// invalid ids, no match
+	c.Assert(mngr.FindSubnetByIDs("vpc-1", "us-west-1", []string{"subnet-unknown"}), check.IsNil)
+}
+
 func (e *ENISuite) TestFindSubnetByTags(c *check.C) {
 	api := ec2mock.NewAPI(subnets, vpcs, securityGroups)
 	c.Assert(api, check.Not(check.IsNil))

pkg/aws/eni/node.go

@@ -362,14 +362,21 @@ func (n *Node) CreateInterface(ctx context.Context, allocation *ipam.AllocationA
 	resource := *n.k8sObj
 	n.mutex.RUnlock()
 
-	bestSubnet := n.manager.FindSubnetByTags(resource.Spec.ENI.VpcID, resource.Spec.ENI.AvailabilityZone, resource.Spec.ENI.SubnetTags)
+	var bestSubnet *ipamTypes.Subnet
+	if len(resource.Spec.ENI.SubnetIDs) > 0 {
+		bestSubnet = n.manager.FindSubnetByIDs(resource.Spec.ENI.VpcID, resource.Spec.ENI.AvailabilityZone, resource.Spec.ENI.SubnetIDs)
+	} else {
+		bestSubnet = n.manager.FindSubnetByTags(resource.Spec.ENI.VpcID, resource.Spec.ENI.AvailabilityZone, resource.Spec.ENI.SubnetTags)
+	}
+
 	if bestSubnet == nil {
 		return 0,
 			errUnableToFindSubnet,
 			fmt.Errorf(
-				"No matching subnet available for interface creation (VPC=%s AZ=%s SubnetTags=%s",
+				"No matching subnet available for interface creation (VPC=%s AZ=%s SubnetIDs=%v SubnetTags=%s)",
 				resource.Spec.ENI.VpcID,
 				resource.Spec.ENI.AvailabilityZone,
+				resource.Spec.ENI.SubnetIDs,
 				resource.Spec.ENI.SubnetTags,
 			)
 	}

pkg/aws/eni/types/types.go

@@ -97,6 +97,12 @@ type ENISpec struct {
 	// +kubebuilder:validation:Optional
 	SecurityGroupTags map[string]string `json:"security-group-tags,omitempty"`
 
+	// SubnetIDs is the list of subnet ids to use when evaluating what AWS
+	// subnets to use for ENI and IP allocation.
+	//
+	// +kubebuilder:validation:Optional
+	SubnetIDs []string `json:"subnet-ids,omitempty"`
+
 	// SubnetTags is the list of tags to use when evaluating what AWS
 	// subnets to use for ENI and IP allocation.
 	//

pkg/k8s/apis/cilium.io/client/crds/v2/ciliumnodes.yaml

@@ -179,6 +179,12 @@ spec:
                     items:
                       type: string
                     type: array
+                  subnet-ids:
+                    description: SubnetIDs is the list of subnet ids to use when evaluating
+                      what AWS subnets to use for ENI and IP allocation.
+                    items:
+                      type: string
+                    type: array
                   subnet-tags:
                     additionalProperties:
                       type: string

pkg/nodediscovery/nodediscovery.go

@@ -492,6 +492,10 @@ func (n *NodeDiscovery) mutateNodeResource(nodeResource *ciliumv2.CiliumNode) er
 				nodeResource.Spec.ENI.SecurityGroups = c.ENI.SecurityGroups
 			}
 
+			if len(c.ENI.SubnetIDs) > 0 {
+				nodeResource.Spec.ENI.SubnetIDs = c.ENI.SubnetIDs
+			}
+
 			if len(c.ENI.SubnetTags) > 0 {
 				nodeResource.Spec.ENI.SubnetTags = c.ENI.SubnetTags
 			}

plugins/cilium-cni/types/types_test.go

@@ -102,6 +102,9 @@ func (t *CNITypesSuite) TestReadCNIConfENIWithPlugins(c *check.C) {
         "security-groups":[
           "sg-xxx"
         ],
+        "subnet-ids":[
+          "subnet-xxx"
+        ],
         "subnet-tags":{
           "foo":"true"
         }
@@ -120,6 +123,7 @@ func (t *CNITypesSuite) TestReadCNIConfENIWithPlugins(c *check.C) {
 			PreAllocate:         5,
 			FirstInterfaceIndex: &firstInterfaceIndex,
 			SecurityGroups:      []string{"sg-xxx"},
+			SubnetIDs:           []string{"subnet-xxx"},
 			SubnetTags: map[string]string{
 				"foo": "true",
 			},
@@ -138,6 +142,10 @@ func (t *CNITypesSuite) TestReadCNIConfENI(c *check.C) {
     "pre-allocate": 16,
     "first-interface-index": 2,
     "security-groups": [ "sg1", "sg2" ],
+    "subnet-ids":[
+      "subnet-1",
+      "subnet-2"
+    ],
     "subnet-tags": {
       "key1": "val1",
       "key2": "val2"
@@ -158,6 +166,7 @@ func (t *CNITypesSuite) TestReadCNIConfENI(c *check.C) {
 			PreAllocate:         16,
 			FirstInterfaceIndex: &firstInterfaceIndex,
 			SecurityGroups:      []string{"sg1", "sg2"},
+			SubnetIDs:           []string{"subnet-1", "subnet-2"},
 			SubnetTags: map[string]string{
 				"key1": "val1",
 				"key2": "val2",
@@ -183,6 +192,9 @@ func (t *CNITypesSuite) TestReadCNIConfENIv2WithPlugins(c *check.C) {
         "security-groups":[
           "sg-xxx"
         ],
+        "subnet-ids":[
+          "subnet-xxx"
+        ],
         "subnet-tags":{
           "foo":"true"
         }
@@ -203,6 +215,7 @@ func (t *CNITypesSuite) TestReadCNIConfENIv2WithPlugins(c *check.C) {
 		ENI: eniTypes.ENISpec{
 			FirstInterfaceIndex: &firstInterfaceIndex,
 			SecurityGroups:      []string{"sg-xxx"},
+			SubnetIDs:           []string{"subnet-xxx"},
 			SubnetTags: map[string]string{
 				"foo": "true",
 			},