Merge e851ce4 into edfc480

docs/config_examples/StaticRoute/README.md

@@ -7,22 +7,35 @@ Support for CIS to configure static routes in BIG-IP with node subnets assigned
 ```
    args:
      --static-routing-mode=true
-     --orchestration-cni=<ovn-k8s/flannel/antrea/cilium-k8s>
+     --orchestration-cni=<ovn-k8s/flannel/antrea/cilium-k8s/calico>
 ```
 * With CNI ovn-k8s, if node has multiple interfaces --static-route-node-cidr can be configured to specify node network from which nodeip has to be selected. Without this config, CIS always picks primary interface address from annotation k8s.ovn.org/node-primary-ifaddr on node manifest as nodeip for static route creation on BIGIP.Use cis-deployment-ovn-k8s-mnic.yaml to deploy with this configuration.
 ```
     args:
       --static-route-node-cidr=10.4.0.0/14
 ```
+### Calico CNI
+* For Calico CNI you need to add the following permissions to the CIS service account to read blockaffinities.
+
+```yaml
+- apiGroups:
+  - crd.projectcalico.org
+  resources:
+  - blockaffinities
+  verbs:
+  - get
+  - list
+  - watch
+```
 
 ## Parameters for StaticRoutingMode
 
-| Parameter              | Type    | Required | Default | Description                                                                                                                          | Allowed Values                      | Agent | Minimum Supported CIS Version |
-|------------------------|---------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------|-------------------------------------|-------|-------------------------------|
-| static-routing-mode    | Boolean | Optional | false   | Adds Static Routes on the BIGIP so that traffic can be directly route to the pods. (Without tunnels)                                 | true, false                         | AS3   | 2.13.0                        |
-| orchestration-cni      | String  | Optional | flannel | Kubernetes Cluster CNI Name                                                                                                          | cilium-k8s, flannel,ovn-k8s, antrea | AS3   | 2.13.0                        |
-| shared-static-routes   | Boolean | Optional | false   | When set to true, static routes are created on the /Common partition, which can be valid only when static-routing-mode is enabled.   | true, false                         | AS3   | 2.14.0                        |
-| static-route-node-cidr | String  | Optional | NA      | To specify node network cidr to be used for static routing when node has multiple interfaces.This is supported only with CNI ovn-k8s | Any valid CIDR eg: 10.4.0.0/14      | AS3   | 2.15.0                        |
+| Parameter              | Type    | Required | Default | Description                                                                                                                          | Allowed Values                              | Agent | Minimum Supported CIS Version |
+|------------------------|---------|----------|---------|--------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------|-------|-------------------------------|
+| static-routing-mode    | Boolean | Optional | false   | Adds Static Routes on the BIGIP so that traffic can be directly route to the pods. (Without tunnels)                                 | true, false                                 | AS3   | 2.13.0                        |
+| orchestration-cni      | String  | Optional | flannel | Kubernetes Cluster CNI Name                                                                                                          | cilium-k8s, flannel,ovn-k8s, antrea, calico | AS3   | 2.13.0                        |
+| shared-static-routes   | Boolean | Optional | false   | When set to true, static routes are created on the /Common partition, which can be valid only when static-routing-mode is enabled.   | true, false                                 | AS3   | 2.14.0                        |
+| static-route-node-cidr | String  | Optional | NA      | To specify node network cidr to be used for static routing when node has multiple interfaces.This is supported only with CNI ovn-k8s | Any valid CIDR eg: 10.4.0.0/14              | AS3   | 2.15.0                        |
 
 
 ## cis-deployment-ovn-k8s.yaml
@@ -54,7 +67,9 @@ In case static routes are not added, along with looking at CIS logs you can also
 | cilium-k8s              | CiliumK8sNodeSubnetAnnotation12 = "io.cilium.network.ipv4-pod-cidr" or CiliumK8sNodeSubnetAnnotation13 = "network.cilium.io/ipv4-pod-cidr", node ip from field node.Status.Addresses                                                                    | io.cilium.network.ipv4-pod-cidr or network.cilium.io/ipv4-pod-cidr annotation is used based on cilium version to read podcidr allocated. Nodeip is parsed from node manifest using field node.Status.Addresses | 
 | antrea/flannel(default) | podcidr from node.Spec.PodCIDR, nodeIP from node.Status.Addresses                                                                                                                                                                                       | podcidr is parsed from node manifest using field node.Spec.PodCIDR and Nodeip is parsed using field node.Status.Addresses                                                                                      |
 
+#### Troubleshooting for CALICO CNI
 
+In case static routes are configured with calico CNI, you can check the logs of CIS to see if the blockaffinities are being read properly. If not, you can check the permissions of the CIS service account to read blockaffinities. You can also check and verify that the blockaffinities are being created properly in the calico CNI.
 
 
 

pkg/controller/constants.go

@@ -119,6 +119,11 @@ const (
 	CiliumK8sNodeSubnetAnnotation12 = "io.cilium.network.ipv4-pod-cidr"
 	CiliumK8sNodeSubnetAnnotation13 = "network.cilium.io/ipv4-pod-cidr"
 
+	//Calico CNI
+	CALICO                      = "calico"
+	CALICO_API_BLOCK_AFFINITIES = "/apis/crd.projectcalico.org/v1/blockaffinities"
+	CALICONodeIPAnnotation      = "projectcalico.org/IPv4Address"
+
 	//CNI plugin
 	FLANNEL      = "flannel"
 	ANTREA       = "antrea"

pkg/controller/node_poll_handler.go

@@ -1,16 +1,19 @@
 package controller
 
 import (
+	"context"
 	"encoding/json"
 	"fmt"
 	"github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/networkmanager"
 	bigIPPrometheus "github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/prometheus"
 	log "github.com/F5Networks/k8s-bigip-ctlr/v3/pkg/vlogger"
 	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"net"
 	"reflect"
 	"sort"
 	"strings"
+	"time"
 )
 
 func (ctlr *Controller) SetupNodeProcessing(clusterName string) error {
@@ -195,6 +198,7 @@ func (ctlr *Controller) processStaticRouteUpdate() {
 			addrType = v1.NodeExternalIP
 		}
 		log.Debugf("Processing Node Updates for static routes")
+		nodePodCIDRMap := ctlr.GetNodePodCIDRMap()
 		// reset the route store to handle the deleted nodes
 		staticRouteMap := make(map[networkmanager.StaticRouteConfig]networkmanager.L3Forward)
 		for _, obj := range nodes {
@@ -296,6 +300,28 @@ func (ctlr *Controller) processStaticRouteUpdate() {
 					}
 
 				}
+			} else if ctlr.OrchestrationCNI == CALICO {
+				if nodePodCIDRMap != nil && len(nodePodCIDRMap) > 0 {
+					if len(nodePodCIDRMap) != len(nodes) {
+						//Wait for some time to get the nodePodCIDRMap in case a new node is added, it takes some time to create the block affinity for node
+						time.Sleep(1 * time.Second)
+						nodePodCIDRMap = ctlr.GetNodePodCIDRMap()
+					}
+					if nodeIPValue, ok := node.Annotations[CALICONodeIPAnnotation]; ok {
+						if cidr, ok := nodePodCIDRMap[node.Name]; ok {
+							l3Forward.Config.Gateway = strings.Split(nodeIPValue, "/")[0]
+							l3Forward.Config.L3ForwardType = networkmanager.L3RouteGateway
+							l3Forward.Name = fmt.Sprintf("%v/%v/%v", ctlr.ControllerIdentifier, node.Name, l3Forward.Config.Gateway)
+							l3Forward.Config.Destination = cidr
+						} else {
+							log.Warningf("Pod Network not found for node %v, static route not added", node.Name)
+							continue
+						}
+					} else {
+						log.Warningf("Host addresses annotation %v not found on node %v ,static route not added", CALICONodeIPAnnotation, node.Name)
+						continue
+					}
+				}
 			} else {
 				//For k8s CNI like flannel, antrea etc we can get subnet from node spec
 				podCIDR := node.Spec.PodCIDR
@@ -404,3 +430,31 @@ func parseHostCIDRS(ann string, nodenetwork *net.IPNet) (string, error) {
 	err := fmt.Errorf("Cannot get nodeip from %s within nodenetwork %v", OvnK8sNodeIPAnnotation3, nodenetwork)
 	return "", err
 }
+
+func (ctlr *Controller) GetNodePodCIDRMap() map[string]string {
+	var nodePodCIDRMap map[string]string
+	if ctlr.OrchestrationCNI == CALICO {
+		// Retrieve Calico Block Affinity
+		blockAffinitiesRaw, err := ctlr.clientsets.kubeClient.Discovery().RESTClient().Get().AbsPath(CALICO_API_BLOCK_AFFINITIES).DoRaw(context.TODO())
+		if err != nil {
+			log.Warningf("Calico blockaffinity resource not found on the cluster, getting error %v", err)
+			return nodePodCIDRMap
+		}
+		// Define a map to store the unmarshalled data
+		var blockAffinities unstructured.UnstructuredList
+
+		// Unmarshal the JSON data into the unstructured list
+		err = json.Unmarshal(blockAffinitiesRaw, &blockAffinities)
+		if err != nil {
+			log.Errorf("Unable to unmarshall block affinity resource %v, getting error %v", string(blockAffinitiesRaw), err)
+			return nodePodCIDRMap
+		}
+		nodePodCIDRMap = make(map[string]string)
+		for _, blockAffinity := range blockAffinities.Items {
+			// Access the spec field from the unstructured object
+			specData := blockAffinity.Object["spec"].(map[string]interface{})
+			nodePodCIDRMap[specData["node"].(string)] = specData["cidr"].(string)
+		}
+	}
+	return nodePodCIDRMap
+}