Support DISABLE_NETWORK_RESOURCE_PROVISIONING (#1586)

* Support DISABLE_NETWORK_RESOURCE_PROVISIONING

* Skip tagENI in reconciler

* Prevent cleanup leaked ENI

* fix up readme

README.md

@@ -495,6 +495,16 @@ This environment variable works when `ENABLE_PREFIX_DELEGATION` is set to `true`
 
 ---
 
+#### `DISABLE_NETWORK_RESOURCE_PROVISIONING` (v1.9.1+)
+
+Type: Boolean as a String
+
+Default: `false`
+
+Setting `DISABLE_NETWORK_RESOURCE_PROVISIONING` to `true` will make IPAMD to depend only on IMDS to get attached ENIs and IPs/prefixes.
+
+---
+
 ### ENI tags related to Allocation
 
 This plugin interacts with the following tags on ENIs:

charts/aws-vpc-cni/values.yaml

@@ -50,6 +50,7 @@ env:
   ENABLE_PREFIX_DELEGATION: "false"
   WARM_ENI_TARGET: "1"
   WARM_PREFIX_TARGET: "1"
+  DISABLE_NETWORK_RESOURCE_PROVISIONING: "false"
 
 # this flag enables you to use the match label that was present in the original daemonset deployed by EKS
 # You can then annotate and label the original aws-node resources and 'adopt' them into a helm release

config/master/aws-k8s-cni-cn.yaml

@@ -145,6 +145,8 @@
           "value": "false"
         - "name": "DISABLE_METRICS"
           "value": "false"
+        - "name": "DISABLE_NETWORK_RESOURCE_PROVISIONING"
+          "value": "false"
         - "name": "ENABLE_POD_ENI"
           "value": "false"
         - "name": "ENABLE_PREFIX_DELEGATION"

config/master/aws-k8s-cni-us-gov-east-1.yaml

@@ -145,6 +145,8 @@
           "value": "false"
         - "name": "DISABLE_METRICS"
           "value": "false"
+        - "name": "DISABLE_NETWORK_RESOURCE_PROVISIONING"
+          "value": "false"
         - "name": "ENABLE_POD_ENI"
           "value": "false"
         - "name": "ENABLE_PREFIX_DELEGATION"

config/master/aws-k8s-cni-us-gov-west-1.yaml

@@ -145,6 +145,8 @@
           "value": "false"
         - "name": "DISABLE_METRICS"
           "value": "false"
+        - "name": "DISABLE_NETWORK_RESOURCE_PROVISIONING"
+          "value": "false"
         - "name": "ENABLE_POD_ENI"
           "value": "false"
         - "name": "ENABLE_PREFIX_DELEGATION"

config/master/aws-k8s-cni.yaml

@@ -145,6 +145,8 @@
           "value": "false"
         - "name": "DISABLE_METRICS"
           "value": "false"
+        - "name": "DISABLE_NETWORK_RESOURCE_PROVISIONING"
+          "value": "false"
         - "name": "ENABLE_POD_ENI"
           "value": "false"
         - "name": "ENABLE_PREFIX_DELEGATION"

config/master/manifests.jsonnet

@@ -176,6 +176,7 @@ local awsnode = {
                 DISABLE_METRICS: "false",
                 ENABLE_POD_ENI: "false",
                 ENABLE_PREFIX_DELEGATION: "false",
+                DISABLE_NETWORK_RESOURCE_PROVISIONING: "false",
                 MY_NODE_NAME: {
                   valueFrom: {
                     fieldRef: {fieldPath: "spec.nodeName"},

pkg/awsutils/awsutils.go

@@ -342,7 +342,7 @@ func (i instrumentedIMDS) GetMetadataWithContext(ctx context.Context, p string)
 }
 
 // New creates an EC2InstanceMetadataCache
-func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) {
+func New(useCustomNetworking, disableENIProvisioning bool) (*EC2InstanceMetadataCache, error) {
 	//ctx is passed to initWithEC2Metadata func to cancel spawned go-routines when tests are run
 	ctx := context.Background()
 
@@ -379,7 +379,9 @@ func New(useCustomNetworking bool) (*EC2InstanceMetadataCache, error) {
 	}
 
 	// Clean up leaked ENIs in the background
-	go wait.Forever(cache.cleanUpLeakedENIs, time.Hour)
+	if !disableENIProvisioning {
+		go wait.Forever(cache.cleanUpLeakedENIs, time.Hour)
+	}
 
 	return cache, nil
 }

pkg/ipamd/ipamd.go

@@ -303,8 +303,9 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex
 	c.networkClient = networkutils.New()
 	c.useCustomNetworking = UseCustomNetworkCfg()
 	c.enableIpv4PrefixDelegation = useIpv4PrefixDelegation()
+	c.disableENIProvisioning = disablingENIProvisioning()
 
-	client, err := awsutils.New(c.useCustomNetworking)
+	client, err := awsutils.New(c.useCustomNetworking, c.disableENIProvisioning)
 	if err != nil {
 		return nil, errors.Wrap(err, "ipamd: can not initialize with AWS SDK interface")
 	}
@@ -317,7 +318,6 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex
 	c.minimumIPTarget = getMinimumIPTarget()
 	c.warmPrefixTarget = getWarmPrefixTarget()
 
-	c.disableENIProvisioning = disablingENIProvisioning()
 	c.enablePodENI = enablePodENI()
 
 	hypervisorType, err := c.awsClient.GetInstanceHypervisorFamily()
@@ -341,15 +341,17 @@ func New(rawK8SClient client.Client, cachedK8SClient client.Client) (*IPAMContex
 
 	mac := c.awsClient.GetPrimaryENImac()
 	// retrieve security groups
+	if !c.disableENIProvisioning {
+		err = c.awsClient.RefreshSGIDs(mac)
+		if err != nil {
+			return nil, err
+		}
 
-	err = c.awsClient.RefreshSGIDs(mac)
-	if err != nil {
-		return nil, err
+		// Refresh security groups and VPC CIDR blocks in the background
+		// Ignoring errors since we will retry in 30s
+		go wait.Forever(func() { _ = c.awsClient.RefreshSGIDs(mac) }, 30*time.Second)
 	}
 
-	// Refresh security groups and VPC CIDR blocks in the background
-	// Ignoring errors since we will retry in 30s
-	go wait.Forever(func() { _ = c.awsClient.RefreshSGIDs(mac) }, 30*time.Second)
 	return c, nil
 }
 
@@ -401,7 +403,7 @@ func (c *IPAMContext) nodeInit() error {
 
 		isTrunkENI := eni.ENIID == metadataResult.TrunkENI
 		isEFAENI := metadataResult.EFAENIs[eni.ENIID]
-		if !isTrunkENI {
+		if !isTrunkENI && !c.disableENIProvisioning {
 			if err := c.awsClient.TagENI(eni.ENIID, metadataResult.TagMap[eni.ENIID]); err != nil {
 				return errors.Wrapf(err, "ipamd init: failed to tag managed ENI %v", eni.ENIID)
 			}
@@ -489,12 +491,14 @@ func (c *IPAMContext) nodeInit() error {
 		c.askForTrunkENIIfNeeded(ctx)
 	}
 
-	// For a new node, attach Cidrs (secondary ips/prefixes)
-	increasedPool, err := c.tryAssignCidrs()
-	if err == nil && increasedPool {
-		c.updateLastNodeIPPoolAction()
-	} else if err != nil {
-		return err
+	if !c.disableENIProvisioning {
+		// For a new node, attach Cidrs (secondary ips/prefixes)
+		increasedPool, err := c.tryAssignCidrs()
+		if err == nil && increasedPool {
+			c.updateLastNodeIPPoolAction()
+		} else if err != nil {
+			return err
+		}
 	}
 	return nil
 }
@@ -1177,7 +1181,7 @@ func (c *IPAMContext) nodeIPPoolReconcile(ctx context.Context, interval time.Dur
 
 		isTrunkENI := attachedENI.ENIID == trunkENI
 		isEFAENI := efaENIs[attachedENI.ENIID]
-		if !isTrunkENI {
+		if !isTrunkENI && !c.disableENIProvisioning {
 			if err := c.awsClient.TagENI(attachedENI.ENIID, eniTagMap[attachedENI.ENIID]); err != nil {
 				log.Errorf("IP pool reconcile: failed to tag managed ENI %v: %v", attachedENI.ENIID, err)
 				ipamdErrInc("eniReconcileAdd")