Operator manages identities - full implementation

**Feature: Operator Manages Cilium Identities**

CFP: cilium#27752

**Description**

A new feature hidden behind a flag. Disabled by default.

Besides the new flag that enables the feature, there are no other user visible changes.

cilium-operator is going to manage (create and delete) Cilium Identity API objects instead of cilium-agent.

cilium-operator calculates the desired state for Cilium Identities based on watched events for Cilium Identities, Pods, Namespaces and Cilium Endpoint Slices (if enabled).

cilium-operator creates Cilium Identities on pod updates for a unique label set based on pod and namespace labels.

cilium-operator deletes Cilium Identities when their labels are not used by any pods.

cilium-agent no longer writes to Cilium Identities.

cilium-agent only watches Cilium Identities.

In case when there is no Cilium Identity in the watcher store for a newly created or updated pod, a temporary security identity (temp id) will be created and used locally by the agent, until it's replaced by a global identity (Cilium Identity).

```release-note
Feature: Operator Manages Cilium Identities
A new feature hidden behind a flag. Disabled by default.
cilium-operator is going to manage (create and delete) Cilium Identity API objects instead of cilium-agent.
```

kind/feature

Signed-off-by: Dorde Lapcevic <dordel@google.com>

Documentation/installation/requirements-aks.rst

@@ -21,4 +21,7 @@ Encapsulation Cluster Pool Kubernetes CRD
 
 * The AKS cluster must be created with ``--network-plugin none``. See the
   `Bring your own CNI <https://docs.microsoft.com/en-us/azure/aks/use-byo-cni?tabs=azure-cli>`_
-  documentation for more details about BYOCNI prerequisites / implications.
+  documentation for more details about BYOCNI prerequisites / implications.
+
+* Make sure that you set a cluster pool IPAM pod CIDR that does not overlap with the default service
+  CIDR of AKS. For example, you can use ``--helm-set ipam.operator.clusterPoolIPv4PodCIDRList=192.168.0.0/16``.

daemon/cmd/daemon_main.go

@@ -1031,6 +1031,9 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
 	flags.Bool(option.EnableCiliumEndpointSlice, false, "Enable the CiliumEndpointSlice watcher in place of the CiliumEndpoint watcher (beta)")
 	option.BindEnv(vp, option.EnableCiliumEndpointSlice)
 
+	flags.Bool(option.OperatorManagesGlobalIdentities, false, "Denotes whether cilium-operator is responsible for creating global security identities in the form of Cilium Identity custom resource")
+	option.BindEnv(vp, option.OperatorManagesGlobalIdentities)
+
 	flags.Bool(option.EnableK8sTerminatingEndpoint, true, "Enable auto-detect of terminating endpoint condition")
 	option.BindEnv(vp, option.EnableK8sTerminatingEndpoint)
 

daemon/cmd/policy.go

@@ -29,9 +29,12 @@ import (
 	"github.com/cilium/cilium/pkg/hive/cell"
 	"github.com/cilium/cilium/pkg/identity"
 	"github.com/cilium/cilium/pkg/identity/cache"
+	"github.com/cilium/cilium/pkg/identity/nonglobal"
 	"github.com/cilium/cilium/pkg/ipcache"
 	ipcacheTypes "github.com/cilium/cilium/pkg/ipcache/types"
 	"github.com/cilium/cilium/pkg/k8s"
+	cilium_api_v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
+	"github.com/cilium/cilium/pkg/k8s/resource"
 	"github.com/cilium/cilium/pkg/labels"
 	"github.com/cilium/cilium/pkg/logging/logfields"
 	"github.com/cilium/cilium/pkg/metrics"
@@ -74,6 +77,7 @@ type policyParams struct {
 	SecretManager   certificatemanager.SecretManager
 	CacheStatus     k8s.CacheStatus
 	ClusterInfo     cmtypes.ClusterInfo
+	CiliumIdentities resource.Resource[*cilium_api_v2.CiliumIdentity]
 }
 
 type policyOut struct {
@@ -99,8 +103,21 @@ func newPolicyTrifecta(params policyParams) (policyOut, error) {
 		num := identity.InitWellKnownIdentities(option.Config, params.ClusterInfo)
 		metrics.Identity.WithLabelValues(identity.WellKnownIdentityType).Add(float64(num))
 	}
+	ctx, cancel := context.WithCancel(context.Background())
+
 	iao := &identityAllocatorOwner{}
-	idAlloc := cache.NewCachingIdentityAllocator(iao)
+	var idAlloc CachingIdentityAllocator
+	if option.Config.OperatorManagesGlobalIdentities {
+		if option.Config.IdentityAllocationMode == option.IdentityAllocationModeCRD {
+			return policyOut{}, fmt.Errorf("operator managing global identities is only supported with CRD identity allocation mode")
+		}
+		if option.Config.DisableCiliumEndpointCRD {
+			return policyOut{}, fmt.Errorf("operator managing global identities is only supported when Cilium Endpoint CRD is enabled")
+		}
+		idAlloc = nonglobal.NewLocalOnlyCachingIDAllocator(ctx, iao, params.CiliumIdentities, params.EndpointManager.GetEndpoints)
+	} else {
+		idAlloc = cache.NewCachingIdentityAllocator(iao)
+	}
 
 	iao.policy = policy.NewStoppedPolicyRepository(
 		idAlloc,
@@ -116,8 +133,6 @@ func newPolicyTrifecta(params policyParams) (policyOut, error) {
 	}
 	iao.policyUpdater = policyUpdater
 
-	ctx, cancel := context.WithCancel(context.Background())
-
 	ipc := ipcache.NewIPCache(&ipcache.Configuration{
 		Context:           ctx,
 		IdentityAllocator: idAlloc,

daemon/k8s/resources.go

@@ -44,6 +44,7 @@ var (
 			k8s.CiliumEndpointSliceResource,
 			k8s.CiliumEnvoyConfigResource,
 			k8s.CiliumClusterwideEnvoyConfigResource,
+			k8s.CiliumIdentityResource,
 		),
 	)
 
@@ -108,6 +109,7 @@ type Resources struct {
 	CiliumCIDRGroups                 resource.Resource[*cilium_api_v2alpha1.CiliumCIDRGroup]
 	CiliumSlimEndpoint               resource.Resource[*types.CiliumEndpoint]
 	CiliumEndpointSlice              resource.Resource[*cilium_api_v2alpha1.CiliumEndpointSlice]
+	CiliumIdentityIdentity           resource.Resource[*cilium_api_v2.CiliumIdentity]
 	CiliumNode                       resource.Resource[*cilium_api_v2.CiliumNode]
 }
 

operator/cmd/flags.go

@@ -256,6 +256,9 @@ func InitGlobalFlags(cmd *cobra.Command, vp *viper.Viper) {
 	flags.Bool(option.EnableCiliumEndpointSlice, false, "If set to true, the CiliumEndpointSlice feature is enabled. If any CiliumEndpoints resources are created, updated, or deleted in the cluster, all those changes are broadcast as CiliumEndpointSlice updates to all of the Cilium agents.")
 	option.BindEnv(vp, option.EnableCiliumEndpointSlice)
 
+	flags.Bool(option.OperatorManagesGlobalIdentities, false, "Denotes whether cilium-operator is responsible for creating global security identities in the form of Cilium Identity custom resource")
+	option.BindEnv(vp, option.OperatorManagesGlobalIdentities)
+
 	flags.String(operatorOption.CiliumK8sNamespace, "", fmt.Sprintf("Name of the Kubernetes namespace in which Cilium is deployed in. Defaults to the same namespace defined in %s", option.K8sNamespaceName))
 	option.BindEnv(vp, operatorOption.CiliumK8sNamespace)
 

operator/cmd/root.go

@@ -35,6 +35,7 @@ import (
 	"github.com/cilium/cilium/operator/pkg/bgpv2"
 	"github.com/cilium/cilium/operator/pkg/ciliumendpointslice"
 	"github.com/cilium/cilium/operator/pkg/ciliumenvoyconfig"
+	"github.com/cilium/cilium/operator/pkg/ciliumidentity"
 	controllerruntime "github.com/cilium/cilium/operator/pkg/controller-runtime"
 	gatewayapi "github.com/cilium/cilium/operator/pkg/gateway-api"
 	"github.com/cilium/cilium/operator/pkg/ingress"
@@ -145,6 +146,15 @@ var (
 			}
 		}),
 
+		cell.Provide(func(
+			daemonCfg *option.DaemonConfig,
+		) ciliumidentity.SharedConfig {
+			return ciliumidentity.SharedConfig{
+				EnableCiliumEndpointSlice: daemonCfg.EnableCiliumEndpointSlice,
+				EnableOperatorManageCIDs: daemonCfg.OperatorManagesGlobalIdentities,
+			}
+		}),
+
 		cell.Provide(func(
 			daemonCfg *option.DaemonConfig,
 		) ciliumendpointslice.SharedConfig {
@@ -196,6 +206,11 @@ var (
 			// refactored into a proper cell.
 			identitygc.Cell,
 
+			// CiliumIdentity controller manages Cilium Identity API objects. It
+			// creates, deletes and updates Cilium Identities (CIDs) based on CID,
+			// Pod, Namespace and CES events.
+			ciliumidentity.Cell,
+
 			// CiliumEndpointSlice controller depends on the CiliumEndpoint and
 			// CiliumEndpointSlice resources. It reconciles the state of CESs in the
 			// cluster based on the CEPs and CESs events.

operator/identitygc/cell.go

@@ -80,4 +80,10 @@ func (def Config) Flags(flags *pflag.FlagSet) {
 type SharedConfig struct {
 	// IdentityAllocationMode specifies what mode to use for identity allocation
 	IdentityAllocationMode string
+
+	// EnableOperatorManageCIDs enables operator to manage Cilium Identities by
+	// running a Cilium Identity controller. If enabled, Identity GC cell is
+	// then disabled because Cilium Identity controller takes care of Cilium
+	// Identity garbage collection.
+	EnableOperatorManageCIDs bool
 }

operator/identitygc/gc.go

@@ -88,7 +88,7 @@ type GC struct {
 }
 
 func registerGC(p params) {
-	if !p.Clientset.IsEnabled() {
+	if !p.Clientset.IsEnabled() || p.SharedCfg.EnableOperatorManageCIDs {
 		return
 	}
 

operator/k8s/resource_ctors.go

@@ -16,6 +16,8 @@ import (
 	cilium_api_v2alpha1 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2alpha1"
 	"github.com/cilium/cilium/pkg/k8s/client"
 	"github.com/cilium/cilium/pkg/k8s/resource"
+	slim_corev1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/api/core/v1"
+	slim_metav1 "github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/meta/v1"
 	"github.com/cilium/cilium/pkg/k8s/utils"
 )
 
@@ -94,3 +96,42 @@ func CiliumBGPNodeConfigOverrideResource(lc cell.Lifecycle, cs client.Clientset,
 	)
 	return resource.New[*cilium_api_v2alpha1.CiliumBGPNodeConfigOverride](lc, lw, resource.WithMetric("CiliumBGPNodeConfigOverride")), nil
 }
+
+func PodResource(lc cell.Lifecycle, cs client.Clientset, opts ...func(*metav1.ListOptions)) (resource.Resource[*slim_corev1.Pod], error) {
+	if !cs.IsEnabled() {
+		return nil, nil
+	}
+	lw := utils.ListerWatcherWithModifiers(
+		utils.ListerWatcherFromTyped[*slim_corev1.PodList](cs.Slim().CoreV1().Pods("")),
+		opts...,
+	)
+
+	indexers := cache.Indexers{
+		cache.NamespaceIndex: cache.MetaNamespaceIndexFunc,
+	}
+
+	return resource.New[*slim_corev1.Pod](lc, lw,
+		resource.WithMetric("Pod"),
+		resource.WithIndexers(indexers),
+		resource.WithTransform[*slim_corev1.Pod, *slim_corev1.Pod](TransformToPod),
+		),
+		nil
+}
+
+func TransformToPod(pod *slim_corev1.Pod) (*slim_corev1.Pod, error) {
+	p := &slim_corev1.Pod{
+		TypeMeta: pod.TypeMeta,
+		ObjectMeta: slim_metav1.ObjectMeta{
+			Name:            pod.Name,
+			Namespace:       pod.Namespace,
+			ResourceVersion: pod.ResourceVersion,
+		},
+		Spec: slim_corev1.PodSpec{
+			NodeName: pod.Spec.NodeName,
+		},
+		Status: slim_corev1.PodStatus{
+			Phase: pod.Status.Phase,
+		},
+	}
+	return p, nil
+}

operator/k8s/resources.go

@@ -31,18 +31,19 @@ var (
 			k8s.ServiceResource,
 			k8s.EndpointsResource,
 			k8s.LBIPPoolsResource,
-			k8s.CiliumIdentityResource,
 			k8s.CiliumPodIPPoolResource,
 			k8s.CiliumBGPPeeringPolicyResource,
 			CiliumBGPClusterConfigResource,
 			k8s.CiliumBGPAdvertisementResource,
 			k8s.CiliumBGPPeerConfigResource,
 			k8s.CiliumBGPNodeConfigResource,
 			CiliumBGPNodeConfigOverrideResource,
+			k8s.CiliumIdentityResource,
 			CiliumEndpointResource,
 			CiliumEndpointSliceResource,
 			CiliumNodeResource,
-			k8s.PodResource,
+			k8s.NamespaceResource,
+			PodResource,
 		),
 	)
 )
@@ -54,10 +55,11 @@ type Resources struct {
 	Services             resource.Resource[*slim_corev1.Service]
 	Endpoints            resource.Resource[*k8s.Endpoints]
 	LBIPPools            resource.Resource[*cilium_api_v2alpha1.CiliumLoadBalancerIPPool]
-	Identities           resource.Resource[*cilium_api_v2.CiliumIdentity]
 	CiliumPodIPPools     resource.Resource[*cilium_api_v2alpha1.CiliumPodIPPool]
+	CiliumIdentities     resource.Resource[*cilium_api_v2.CiliumIdentity]
 	CiliumEndpoints      resource.Resource[*cilium_api_v2.CiliumEndpoint]
 	CiliumEndpointSlices resource.Resource[*cilium_api_v2alpha1.CiliumEndpointSlice]
 	CiliumNodes          resource.Resource[*cilium_api_v2.CiliumNode]
+	Namespaces           resource.Resource[*slim_corev1.Namespace]
 	Pods                 resource.Resource[*slim_corev1.Pod]
 }