[cni/cilium] Add support for additional config options

In this commit, we enable users define their setup with following
additional fields:

- DisableEndpointCRD
- EnableEndpointHealthChecking
- IdentityAllocationMode
- IdentityChangeGracePeriod
- BPFLBAlgorithm
- BPFLBMaglevTableSize
- BPFNATGlobalMax
- BPFNeighGlobalMax
- BPFPolicyMapMax

Added also validation tests to prevent conflicting value combinations to
reach actual cluster state.

Signed-off-by: dntosas <ntosas@gmail.com>

k8s/crds/kops.k8s.io_clusters.yaml

@@ -3427,6 +3427,31 @@ spec:
                         description: 'BPFCTGlobalTCPMax is the maximum number of entries
                           in the TCP CT table. Default: 524288'
                         type: integer
+                      bpfLBAlgorithm:
+                        description: 'BPFLBAlgorithm is the load balancing algorithm
+                          ("random", "maglev"). Default: random'
+                        type: string
+                      bpfLBMaglevTableSize:
+                        description: 'BPFLBMaglevTableSize is the per service backend
+                          table size when going with Maglev (parameter M). Default:
+                          16381'
+                        type: string
+                      bpfLBMapMax:
+                        description: 'BPFLBMapMax is the maximum number of entries
+                          in bpf lb service, backend and affinity maps. Default: 65536'
+                        type: integer
+                      bpfNATGlobalMax:
+                        description: 'BPFNATGlobalMax is the the maximum number of
+                          entries in the BPF NAT table. Default: 524288'
+                        type: integer
+                      bpfNeighGlobalMax:
+                        description: 'BPFNeighGlobalMax is the the maximum number
+                          of entries in the BPF Neighbor table. Default: 524288'
+                        type: integer
+                      bpfPolicyMapMax:
+                        description: 'BPFPolicyMapMax is the maximum number of entries
+                          in endpoint policy map. Default: 16384'
+                        type: integer
                       bpfRoot:
                         description: BPFRoot is not implemented and may be removed
                           in the future. Setting this has no effect.
@@ -3484,6 +3509,10 @@ spec:
                         description: DisableConntrack is not implemented and may be
                           removed in the future. Setting this has no effect.
                         type: boolean
+                      disableEndpointCRD:
+                        description: 'DisableEndpointCRD disables usage of CiliumEndpoint
+                          CRD. Default: false'
+                        type: boolean
                       disableIpv4:
                         description: 'DisableIpv4 is deprecated: Use EnableIpv4 instead.
                           Setting this flag has no effect.'
@@ -3500,6 +3529,10 @@ spec:
                         description: 'EnableEncryption enables Cilium Encryption.
                           Default: false'
                         type: boolean
+                      enableEndpointHealthChecking:
+                        description: 'EnableEndpointHealthChecking enables connectivity
+                          health checking between virtual endpoints. Default: true'
+                        type: boolean
                       enableHostReachableServices:
                         description: 'EnableHostReachableServices configures Cilium
                           to enable services to be reached from the host namespace
@@ -3553,8 +3586,8 @@ spec:
                           agent.
                         properties:
                           enabled:
-                            description: Enabled specifies whether Hubble is enabled
-                              on the agent.
+                            description: Enabled decides if Hubble is enabled on the
+                              agent or not
                             type: boolean
                           metrics:
                             description: Metrics is a list of metrics to collect.
@@ -3563,14 +3596,23 @@ spec:
                               type: string
                             type: array
                         type: object
+                      identityAllocationMode:
+                        description: 'IdentityAllocationMode specifies in which backend
+                          identities are stored ("crd", "kvstore"). Default: crd'
+                        type: string
+                      identityChangeGracePeriod:
+                        description: 'IdentityChangeGracePeriod specifies the duration
+                          to wait before using a changed identity. Default: 5s'
+                        type: string
                       ipam:
                         description: Ipam specifies the IP address allocation mode
                           to use. Possible values are "crd" and "eni". "eni" will
                           use AWS native networking for pods. Eni requires masquerade
                           to be set to false. "crd" will use CRDs for controlling
                           IP address management. "hostscope" will use hostscope IPAM
                           mode. "kubernetes" will use addersing based on node pod
-                          CIDR. Empty value will use host-scope address management.
+                          CIDR. Empty value will use hostscope for cilum <= 1.7 and
+                          "kubernetes" otherwise.
                         type: string
                       ipv4ClusterCidrMaskSize:
                         description: Ipv4ClusterCIDRMaskSize is not implemented and
@@ -3760,7 +3802,7 @@ spec:
                   classic:
                     description: ClassicNetworkingSpec is the specification of classic
                       networking mode, integrated into kubernetes. Support been removed
-                      since kubernetes 1.4.
+                      since Kubernetes 1.4.
                     type: object
                   cni:
                     description: CNINetworkingSpec is the specification for networking
@@ -3811,7 +3853,7 @@ spec:
                     type: object
                   lyftvpc:
                     description: LyftVPCNetworkingSpec declares that we want to use
-                      the cni-ipvlan-vpc-k8s CNI networking
+                      the cni-ipvlan-vpc-k8s CNI networking.
                     properties:
                       subnetTags:
                         additionalProperties:

pkg/apis/kops/networking.go

@@ -303,6 +303,9 @@ type CiliumNetworkingSpec struct {
 	// DisableConntrack is not implemented and may be removed in the future.
 	// Setting this has no effect.
 	DisableConntrack bool `json:"disableConntrack,omitempty"`
+	// DisableEndpointCRD disables usage of CiliumEndpoint CRD.
+	// Default: false
+	DisableEndpointCRD bool `json:"disableEndpointCRD,omitempty"`
 	// DisableIpv4 is deprecated: Use EnableIpv4 instead.
 	// Setting this flag has no effect.
 	DisableIpv4 bool `json:"disableIpv4,omitempty"`
@@ -315,6 +318,9 @@ type CiliumNetworkingSpec struct {
 	// "never": Cilium allows all traffic regardless of policies in place.
 	// If unspecified, "default" policy mode will be used.
 	EnablePolicy string `json:"enablePolicy,omitempty"`
+	// EnableEndpointHealthChecking enables connectivity health checking between virtual endpoints.
+	// Default: true
+	EnableEndpointHealthChecking *bool `json:"enableEndpointHealthChecking,omitempty"`
 	// EnableTracing is not implemented and may be removed in the future.
 	// Setting this has no effect.
 	EnableTracing bool `json:"enableTracing,omitempty"`
@@ -326,6 +332,12 @@ type CiliumNetworkingSpec struct {
 	// EnvoyLog is not implemented and may be removed in the future.
 	// Setting this has no effect.
 	EnvoyLog string `json:"envoyLog,omitempty"`
+	// IdentityAllocationMode specifies in which backend identities are stored ("crd", "kvstore").
+	// Default: crd
+	IdentityAllocationMode string `json:"identityAllocationMode,omitempty"`
+	// IdentityChangeGracePeriod specifies the duration to wait before using a changed identity.
+	// Default: 5s
+	IdentityChangeGracePeriod string `json:"identityChangeGracePeriod,omitempty"`
 	// Ipv4ClusterCIDRMaskSize is not implemented and may be removed in the future.
 	// Setting this has no effect.
 	Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"`
@@ -436,6 +448,24 @@ type CiliumNetworkingSpec struct {
 	// BPFCTGlobalAnyMax is the maximum number of entries in the non-TCP CT table.
 	// Default: 262144
 	BPFCTGlobalAnyMax int `json:"bpfCTGlobalAnyMax,omitempty"`
+	// BPFLBAlgorithm is the load balancing algorithm ("random", "maglev").
+	// Default: random
+	BPFLBAlgorithm string `json:"bpfLBAlgorithm,omitempty"`
+	// BPFLBMaglevTableSize is the per service backend table size when going with Maglev (parameter M).
+	// Default: 16381
+	BPFLBMaglevTableSize string `json:"bpfLBMaglevTableSize,omitempty"`
+	// BPFNATGlobalMax is the the maximum number of entries in the BPF NAT table.
+	// Default: 524288
+	BPFNATGlobalMax int `json:"bpfNATGlobalMax,omitempty"`
+	// BPFNeighGlobalMax is the the maximum number of entries in the BPF Neighbor table.
+	// Default: 524288
+	BPFNeighGlobalMax int `json:"bpfNeighGlobalMax,omitempty"`
+	// BPFPolicyMapMax is the maximum number of entries in endpoint policy map.
+	// Default: 16384
+	BPFPolicyMapMax int `json:"bpfPolicyMapMax,omitempty"`
+	// BPFLBMapMax is the maximum number of entries in bpf lb service, backend and affinity maps.
+	// Default: 65536
+	BPFLBMapMax int `json:"bpfLBMapMax,omitempty"`
 	// PreallocateBPFMaps reduces the per-packet latency at the expense of up-front memory allocation.
 	// Default: true
 	PreallocateBPFMaps bool `json:"preallocateBPFMaps,omitempty"`

pkg/apis/kops/v1alpha2/networking.go

@@ -38,7 +38,7 @@ type NetworkingSpec struct {
 }
 
 // ClassicNetworkingSpec is the specification of classic networking mode, integrated into kubernetes.
-// Support been removed since kubernetes 1.4.
+// Support been removed since Kubernetes 1.4.
 type ClassicNetworkingSpec struct {
 }
 
@@ -256,6 +256,8 @@ type AmazonVPCNetworkingSpec struct {
 	Env []EnvVar `json:"env,omitempty"`
 }
 
+const CiliumIpamEni = "eni"
+
 // CiliumNetworkingSpec declares that we want Cilium networking
 type CiliumNetworkingSpec struct {
 	// Version is the version of the Cilium agent and the Cilium Operator.
@@ -301,6 +303,9 @@ type CiliumNetworkingSpec struct {
 	// DisableConntrack is not implemented and may be removed in the future.
 	// Setting this has no effect.
 	DisableConntrack bool `json:"disableConntrack,omitempty"`
+	// DisableEndpointCRD disables usage of CiliumEndpoint CRD.
+	// Default: false
+	DisableEndpointCRD bool `json:"disableEndpointCRD,omitempty"`
 	// DisableIpv4 is deprecated: Use EnableIpv4 instead.
 	// Setting this flag has no effect.
 	DisableIpv4 bool `json:"disableIpv4,omitempty"`
@@ -313,6 +318,9 @@ type CiliumNetworkingSpec struct {
 	// "never": Cilium allows all traffic regardless of policies in place.
 	// If unspecified, "default" policy mode will be used.
 	EnablePolicy string `json:"enablePolicy,omitempty"`
+	// EnableEndpointHealthChecking enables connectivity health checking between virtual endpoints.
+	// Default: true
+	EnableEndpointHealthChecking *bool `json:"enableEndpointHealthChecking,omitempty"`
 	// EnableTracing is not implemented and may be removed in the future.
 	// Setting this has no effect.
 	EnableTracing bool `json:"enableTracing,omitempty"`
@@ -324,6 +332,12 @@ type CiliumNetworkingSpec struct {
 	// EnvoyLog is not implemented and may be removed in the future.
 	// Setting this has no effect.
 	EnvoyLog string `json:"envoyLog,omitempty"`
+	// IdentityAllocationMode specifies in which backend identities are stored ("crd", "kvstore").
+	// Default: crd
+	IdentityAllocationMode string `json:"identityAllocationMode,omitempty"`
+	// IdentityChangeGracePeriod specifies the duration to wait before using a changed identity.
+	// Default: 5s
+	IdentityChangeGracePeriod string `json:"identityChangeGracePeriod,omitempty"`
 	// Ipv4ClusterCIDRMaskSize is not implemented and may be removed in the future.
 	// Setting this has no effect.
 	Ipv4ClusterCIDRMaskSize int `json:"ipv4ClusterCidrMaskSize,omitempty"`
@@ -434,6 +448,24 @@ type CiliumNetworkingSpec struct {
 	// BPFCTGlobalAnyMax is the maximum number of entries in the non-TCP CT table.
 	// Default: 262144
 	BPFCTGlobalAnyMax int `json:"bpfCTGlobalAnyMax,omitempty"`
+	// BPFLBAlgorithm is the load balancing algorithm ("random", "maglev").
+	// Default: random
+	BPFLBAlgorithm string `json:"bpfLBAlgorithm,omitempty"`
+	// BPFLBMaglevTableSize is the per service backend table size when going with Maglev (parameter M).
+	// Default: 16381
+	BPFLBMaglevTableSize string `json:"bpfLBMaglevTableSize,omitempty"`
+	// BPFNATGlobalMax is the the maximum number of entries in the BPF NAT table.
+	// Default: 524288
+	BPFNATGlobalMax int `json:"bpfNATGlobalMax,omitempty"`
+	// BPFNeighGlobalMax is the the maximum number of entries in the BPF Neighbor table.
+	// Default: 524288
+	BPFNeighGlobalMax int `json:"bpfNeighGlobalMax,omitempty"`
+	// BPFPolicyMapMax is the maximum number of entries in endpoint policy map.
+	// Default: 16384
+	BPFPolicyMapMax int `json:"bpfPolicyMapMax,omitempty"`
+	// BPFLBMapMax is the maximum number of entries in bpf lb service, backend and affinity maps.
+	// Default: 65536
+	BPFLBMapMax int `json:"bpfLBMapMax,omitempty"`
 	// PreallocateBPFMaps reduces the per-packet latency at the expense of up-front memory allocation.
 	// Default: true
 	PreallocateBPFMaps bool `json:"preallocateBPFMaps,omitempty"`
@@ -463,7 +495,7 @@ type CiliumNetworkingSpec struct {
 	// "crd" will use CRDs for controlling IP address management.
 	// "hostscope" will use hostscope IPAM mode.
 	// "kubernetes" will use addersing based on node pod CIDR.
-	// Empty value will use host-scope address management.
+	// Empty value will use hostscope for cilum <= 1.7 and "kubernetes" otherwise.
 	Ipam string `json:"ipam,omitempty"`
 	// IPTablesRulesNoinstall disables installing the base IPTables rules used for masquerading and kube-proxy.
 	// Default: false
@@ -509,15 +541,15 @@ type CiliumNetworkingSpec struct {
 
 // HubbleSpec configures the Hubble service on the Cilium agent.
 type HubbleSpec struct {
-	// Enabled specifies whether Hubble is enabled on the agent.
+	// Enabled decides if Hubble is enabled on the agent or not
 	Enabled *bool `json:"enabled,omitempty"`
 
 	// Metrics is a list of metrics to collect. If empty or null, metrics are disabled.
 	// See https://docs.cilium.io/en/stable/configuration/metrics/#hubble-exported-metrics
 	Metrics []string `json:"metrics,omitempty"`
 }
 
-// LyftVPCNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking
+// LyftVPCNetworkingSpec declares that we want to use the cni-ipvlan-vpc-k8s CNI networking.
 type LyftVPCNetworkingSpec struct {
 	SubnetTags map[string]string `json:"subnetTags,omitempty"`
 }

pkg/apis/kops/validation/validation.go

@@ -837,6 +837,18 @@ func validateNetworkingCilium(cluster *kops.Cluster, v *kops.CiliumNetworkingSpe
 		allErrs = append(allErrs, IsValidValue(fldPath.Child("containerRuntimeLabels"), &v.ContainerRuntimeLabels, []string{"none", "containerd", "crio", "docker", "auto"})...)
 	}
 
+	if v.IdentityAllocationMode != "" {
+		allErrs = append(allErrs, IsValidValue(fldPath.Child("identityAllocationMode"), &v.IdentityAllocationMode, []string{"crd", "kvstore"})...)
+
+		if v.IdentityAllocationMode == "kvstore" && !v.EtcdManaged {
+			allErrs = append(allErrs, field.Forbidden(fldPath.Child("identityAllocationMode"), "Cilium requires managed etcd to allocate identities on kvstore mode"))
+		}
+	}
+
+	if v.BPFLBAlgorithm != "" {
+		allErrs = append(allErrs, IsValidValue(fldPath.Child("bpfLBAlgorithm"), &v.BPFLBAlgorithm, []string{"random", "maglev"})...)
+	}
+
 	if v.Ipam != "" {
 		// "azure" not supported by kops
 		allErrs = append(allErrs, IsValidValue(fldPath.Child("ipam"), &v.Ipam, []string{"hostscope", "kubernetes", "crd", "eni"})...)

pkg/apis/kops/validation/validation_test.go

@@ -831,6 +831,12 @@ func Test_Validate_Cilium(t *testing.T) {
 			},
 			ExpectedErrors: []string{"Forbidden::cilium.ipam"},
 		},
+		{
+			Cilium: kops.CiliumNetworkingSpec{
+				IdentityAllocationMode: "kvstore",
+			},
+			ExpectedErrors: []string{"Forbidden::cilium.identityAllocationMode"},
+		},
 		{
 			Cilium: kops.CiliumNetworkingSpec{
 				Version: "v1.0.0",