Enable CiliumEndpointSlice feature

Documentation/spelling_wordlist.txt

@@ -335,6 +335,7 @@ egressMasqueradeInterfaces
 egressing
 elfutils
 ena
+enableCiliumEndpointSlice
 enableCnpStatusUpdates
 enableCriticalPriorityClass
 enableHealthCheck

Makefile

@@ -308,6 +308,7 @@ manifests: ## Generate K8s manifests e.g. CRD, RBAC etc.
 	mv ${TMPDIR}/cilium.io_ciliumexternalworkloads.yaml ./pkg/k8s/apis/cilium.io/client/crds/v2/ciliumexternalworkloads.yaml
 	mv ${TMPDIR}/cilium.io_ciliumlocalredirectpolicies.yaml ./pkg/k8s/apis/cilium.io/client/crds/v2/ciliumlocalredirectpolicies.yaml
 	mv ${TMPDIR}/cilium.io_ciliumegressnatpolicies.yaml ./pkg/k8s/apis/cilium.io/client/crds/v2alpha1/ciliumegressnatpolicies.yaml
+	mv ${TMPDIR}/cilium.io_ciliumendpointslices.yaml ./pkg/k8s/apis/cilium.io/client/crds/v2alpha1/ciliumendpointslices.yaml
 	rm -rf $(TMPDIR)
 
 generate-api: api/v1/openapi.yaml ## Generate cilium-agent client, model and server code from openapi spec.

api/v1/openapi.yaml

@@ -3070,6 +3070,7 @@ definitions:
       Policy specs.
 
       +deepequal-gen=true
+      +k8s:deepcopy-gen=true
     type: array
     items:
       "$ref": "#/definitions/Port"

daemon/cmd/daemon_main.go

@@ -1031,6 +1031,9 @@ func initializeFlags() {
 	flags.MarkHidden(option.BypassIPAvailabilityUponRestore)
 	option.BindEnv(option.BypassIPAvailabilityUponRestore)
 
+	flags.Bool(option.EnableCiliumEndpointSlice, false, "If set to true, CiliumEndpointSlice feature is enabled and cilium agent watch for CiliumEndpointSlice instead of CiliumEndpoint to update the IPCache.")
+	option.BindEnv(option.EnableCiliumEndpointSlice)
+
 	viper.BindPFlags(flags)
 }
 

install/kubernetes/cilium/README.md

@@ -126,6 +126,7 @@ contributors across the globe, there is almost always someone available to help.
 | debug.enabled | bool | `false` | Enable debug logging |
 | disableEndpointCRD | string | `"false"` | Disable the usage of CiliumEndpoint CRD. |
 | egressGateway | object | `{"enabled":false}` | Enables egress gateway (beta) to redirect and SNAT the traffic that leaves the cluster. |
+| enableCiliumEndpointSlice | bool | `false` | Enable CiliumEndpointSlice feature. |
 | enableCnpStatusUpdates | bool | `false` | Whether to enable CNP status updates. |
 | enableCriticalPriorityClass | bool | `true` | Explicitly enable or disable priority class. .Capabilities.KubeVersion is unsettable in `helm template` calls, it depends on k8s libraries version that Helm was compiled against. This option allows to explicitly disable setting the priority class, which is useful for rendering charts for gke clusters in advance. |
 | enableIPv4Masquerade | bool | `true` | Enables masquerading of IPv4 traffic leaving the node from endpoints. |

install/kubernetes/cilium/templates/cilium-agent/clusterrole.yaml

@@ -105,6 +105,7 @@ rules:
   - ciliumlocalredirectpolicies/status
   - ciliumlocalredirectpolicies/finalizers
   - ciliumegressnatpolicies
+  - ciliumendpointslices
   verbs:
   - '*'
 {{- end }}

install/kubernetes/cilium/templates/cilium-configmap.yaml

@@ -771,6 +771,10 @@ data:
   vlan-bpf-bypass: {{ .Values.bpf.vlanBypass | join " " | quote }}
 {{- end }}
 
+{{- if .Values.enableCiliumEndpointSlice }}
+  enable-cilium-endpoint-slice: "true"
+{{- end }}
+
 {{- if .Values.extraConfig }}
   {{ toYaml .Values.extraConfig | nindent 2 }}
 {{- end }}

install/kubernetes/cilium/templates/cilium-operator/clusterrole.yaml

@@ -66,6 +66,7 @@ rules:
   - ciliumnodes/status
   - ciliumnodes/finalizers
   - ciliumidentities
+  - ciliumendpointslices
   - ciliumidentities/status
   - ciliumidentities/finalizers
   - ciliumlocalredirectpolicies

install/kubernetes/cilium/templates/cilium-preflight/clusterrole.yaml

@@ -105,6 +105,7 @@ rules:
   - ciliumlocalredirectpolicies/status
   - ciliumlocalredirectpolicies/finalizers
   - ciliumegressnatpolicies
+  - ciliumendpointslices
   verbs:
   - '*'
 {{- end }}

install/kubernetes/cilium/values.yaml

@@ -414,6 +414,9 @@ enableK8sEventHandover: false
 
 # enableK8sEndpointSlice: false
 
+# -- Enable CiliumEndpointSlice feature.
+enableCiliumEndpointSlice: false
+
 # -- Enables the fallback compatibility solution for when the xt_socket kernel
 # module is missing and it is needed for the datapath L7 redirection to work
 # properly. See documentation for details on when this can be disabled:

operator/flags.go

@@ -305,5 +305,16 @@ func init() {
 	flags.Bool(option.SkipCRDCreation, false, "When true, Kubernetes Custom Resource Definitions will not be created")
 	option.BindEnv(option.SkipCRDCreation)
 
+	flags.Bool(option.EnableCiliumEndpointSlice, false, "If set to true, the CiliumEndpointSlice feature is enabled. If any CiliumEndpoints resources are created, updated, or deleted in the cluster, all those changes are broadcast as CiliumEndpointSlice updates to all of the Cilium agents.")
+	option.BindEnv(option.EnableCiliumEndpointSlice)
+
+	flags.Int(operatorOption.CESMaxCEPsInCES, operatorOption.CESMaxCEPsInCESDefault, "Maximum number of CiliumEndpoints allowed in a CES")
+	flags.MarkHidden(operatorOption.CESMaxCEPsInCES)
+	option.BindEnv(operatorOption.CESMaxCEPsInCES)
+
+	flags.String(operatorOption.CESSlicingMode, operatorOption.CESSlicingModeDefault, "Slicing mode define how ceps are grouped into a CES")
+	flags.MarkHidden(operatorOption.CESSlicingMode)
+	option.BindEnv(operatorOption.CESSlicingMode)
+
 	viper.BindPFlags(flags)
 }

operator/main.go

@@ -22,6 +22,7 @@ import (
 	"github.com/cilium/cilium/operator/cmd"
 	operatorMetrics "github.com/cilium/cilium/operator/metrics"
 	operatorOption "github.com/cilium/cilium/operator/option"
+	ces "github.com/cilium/cilium/operator/pkg/ciliumendpointslice"
 	operatorWatchers "github.com/cilium/cilium/operator/watchers"
 	"github.com/cilium/cilium/pkg/components"
 	"github.com/cilium/cilium/pkg/ipam/allocator"
@@ -349,6 +350,22 @@ func onOperatorStartLeading(ctx context.Context) {
 
 	ciliumK8sClient = k8s.CiliumClient()
 
+	// If CiliumEndpointSlice feature is enabled, create CESController, start CEP watcher and run controller.
+	if !option.Config.DisableCiliumEndpointCRD && option.Config.EnableCiliumEndpointSlice {
+		log.Info("Create and run CES controller, start CEP watcher")
+		// Initialize  the CES controller
+		cesController := ces.NewCESController(k8s.CiliumClient(),
+			operatorOption.Config.CESMaxCEPsInCES,
+			operatorOption.Config.CESSlicingMode,
+			option.Config.K8sClientQPSLimit,
+			option.Config.K8sClientBurst)
+		stopCh := make(chan struct{})
+		// Start CEP watcher
+		operatorWatchers.CiliumEndpointsSliceInit(k8s.CiliumClient().CiliumV2(), cesController)
+		// Start the CES controller, after current CEPs are synced locally in cache.
+		go cesController.Run(operatorWatchers.CiliumEndpointStore, stopCh)
+	}
+
 	// Restart kube-dns as soon as possible since it helps etcd-operator to be
 	// properly setup. If kube-dns is not managed by Cilium it can prevent
 	// etcd from reaching out kube-dns in EKS.

operator/metrics/metrics.go

@@ -82,6 +82,24 @@ var (
 
 	// IdentityGCRuns records how many times identity GC has run
 	IdentityGCRuns *prometheus.GaugeVec
+
+	// CiliumEndpointSliceDensity indicates the number of CEPs batched in a CES and it used to
+	// collect the number of CEPs in CES at various buckets. For example,
+	// number of CESs in the CEP range <0, 10>
+	// number of CESs in the CEP range <11, 20>
+	// number of CESs in the CEP range <21, 30> and so on
+	CiliumEndpointSliceDensity prometheus.Histogram
+
+	// CiliumEndpointsChangeCount indicates the total number of CEPs changed for every CES request sent to k8s-apiserver.
+	// This metric is used to collect number of CEP changes happening at various buckets.
+	CiliumEndpointsChangeCount *prometheus.HistogramVec
+
+	// CiliumEndpointSliceSyncErrors used to track the total number of errors occurred during syncing CES with k8s-apiserver.
+	CiliumEndpointSliceSyncErrors prometheus.Counter
+
+	// CiliumEndpointSliceQueueDelay measures the time spent by CES's in the workqueue. This measures time difference between
+	// CES insert in the workqueue and removal from workqueue.
+	CiliumEndpointSliceQueueDelay prometheus.Histogram
 )
 
 const (
@@ -91,6 +109,9 @@ const (
 	// LabelOutcome indicates whether the outcome of the operation was successful or not
 	LabelOutcome = "outcome"
 
+	// LabelOpcode indicates the kind of CES metric, could be CEP insert or remove
+	LabelOpcode = "opcode"
+
 	// Label values
 
 	// LabelValueOutcomeSuccess is used as a successful outcome of an operation
@@ -104,6 +125,12 @@ const (
 
 	// LabelValueOutcomeDeleted is used as outcome of deleted identity entries
 	LabelValueOutcomeDeleted = "deleted"
+
+	// LabelValueCEPInsert is used to indicate the number of CEPs inserted in a CES
+	LabelValueCEPInsert = "cepinserted"
+
+	// LabelValueCEPRemove is used to indicate the number of CEPs removed from a CES
+	LabelValueCEPRemove = "cepremoved"
 )
 
 func registerMetrics() []prometheus.Collector {
@@ -127,6 +154,34 @@ func registerMetrics() []prometheus.Collector {
 	}, []string{LabelOutcome})
 	collectors = append(collectors, IdentityGCRuns)
 
+	CiliumEndpointSliceDensity = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: Namespace,
+		Name:      "number_of_ceps_per_ces",
+		Help:      "The number of CEPs batched in a CES",
+	})
+	collectors = append(collectors, CiliumEndpointSliceDensity)
+
+	CiliumEndpointsChangeCount = prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Namespace: Namespace,
+		Name:      "number_of_cep_changes_per_ces",
+		Help:      "The number of changed CEPs in each CES update",
+	}, []string{LabelOpcode})
+	collectors = append(collectors, CiliumEndpointsChangeCount)
+
+	CiliumEndpointSliceSyncErrors = prometheus.NewCounter(prometheus.CounterOpts{
+		Namespace: Namespace,
+		Name:      "ces_sync_errors_total",
+		Help:      "Number of CES sync errors",
+	})
+	collectors = append(collectors, CiliumEndpointSliceSyncErrors)
+
+	CiliumEndpointSliceQueueDelay = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: Namespace,
+		Name:      "ces_queueing_delay_seconds",
+		Help:      "CiliumEndpointSlice queueing delay in seconds",
+	})
+	collectors = append(collectors, CiliumEndpointSliceQueueDelay)
+
 	Registry.MustRegister(collectors...)
 
 	return collectors

operator/option/config.go

@@ -20,6 +20,12 @@ const (
 
 	// PrometheusServeAddr is the default server address for operator metrics
 	PrometheusServeAddr = ":6942"
+
+	// CESMaxCEPsInCESDefault is the maximum number of cilium endpoints allowed in a CES
+	CESMaxCEPsInCESDefault = 100
+
+	// CESSlicingModeDefault is default method for grouping CEP in a CES.
+	CESSlicingModeDefault = "cesSliceModeIdentity"
 )
 
 const (
@@ -187,6 +193,15 @@ const (
 	// Enabling this option reduces waste of IP addresses but may increase
 	// the number of API calls to AlibabaCloud ECS service.
 	AlibabaCloudReleaseExcessIPs = "alibaba-cloud-release-excess-ips"
+
+	// CiliumEndpointSlice options
+
+	// CESMaxCEPsInCES is the maximum number of cilium endpoints allowed in single
+	// a CiliumEndpointSlice resource.
+	CESMaxCEPsInCES = "ces-max-ciliumendpoints-per-ces"
+
+	// CESSlicingMode instructs how CEPs are grouped in a CES.
+	CESSlicingMode = "ces-slice-mode"
 )
 
 // OperatorConfig is the configuration used by the operator.
@@ -351,6 +366,16 @@ type OperatorConfig struct {
 	// Enabling this option reduces waste of IP addresses but may increase
 	// the number of API calls to AlibabaCloud ECS service.
 	AlibabaCloudReleaseExcessIPs bool
+
+	// CiliumEndpointSlice options
+
+	// CESMaxCEPsInCES is the maximum number of CiliumEndpoints allowed in single
+	// a CiliumEndpointSlice resource.
+	// The default value of maximum CiliumEndpoints allowed in a CiliumEndpointSlice resource is 100.
+	CESMaxCEPsInCES int
+
+	// CESSlicingMode instructs how CEPs are grouped in a CES.
+	CESSlicingMode string
 }
 
 // Populate sets all options with the values from viper.
@@ -407,6 +432,10 @@ func (c *OperatorConfig) Populate() {
 	c.AlibabaCloudVPCID = viper.GetString(AlibabaCloudVPCID)
 	c.AlibabaCloudReleaseExcessIPs = viper.GetBool(AlibabaCloudReleaseExcessIPs)
 
+	// CiliumEndpointSlice options
+	c.CESMaxCEPsInCES = viper.GetInt(CESMaxCEPsInCES)
+	c.CESSlicingMode = viper.GetString(CESSlicingMode)
+
 	// Option maps and slices
 
 	if m := viper.GetStringSlice(IPAMSubnetsIDs); len(m) != 0 {