Identity GC skips IDs that are used in CESs

When Cilium Endpoint Slices are enabled, Cilium Identities shouldn’t be deleted while they are still used by CES.

This is a fix in the operator to check if CESs from the CES store are using Identities before deleting them.

```release-note
Fix for Identities that can be deleted before CESs are reconciled
```

Signed-off-by: Dorde Lapcevic <dordel@google.com>

operator/identitygc/crd_gc.go

@@ -11,12 +11,14 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/cilium/cilium/operator/metrics"
+	"github.com/cilium/cilium/operator/pkg/ciliumendpointslice"
 	"github.com/cilium/cilium/operator/watchers"
 	"github.com/cilium/cilium/pkg/controller"
 	v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
 	"github.com/cilium/cilium/pkg/k8s/identitybackend"
 	"github.com/cilium/cilium/pkg/k8s/resource"
 	"github.com/cilium/cilium/pkg/logging/logfields"
+	"github.com/cilium/cilium/pkg/option"
 )
 
 func (igc *GC) startCRDModeGC(ctx context.Context) error {
@@ -75,14 +77,27 @@ func (igc *GC) gc(ctx context.Context) error {
 		igc.logger.WithError(err).Error("unable to get Cilium identities from local store")
 		return err
 	}
+
+	var idsInCESs map[string]bool
+	cesEnabled := option.Config.EnableCiliumEndpointSlice
+	if cesEnabled {
+		idsInCESs = ciliumendpointslice.UsedIdentitiesInCESs()
+	}
+
 	identities := identitiesStore.List()
 	totalEntries := len(identities)
 	deletedEntries := 0
 
 	timeNow := time.Now()
 	for _, identity := range identities {
-		// The identity is definitely alive if there's a CE using it.
-		if watchers.HasCEWithIdentity(identity.Name) {
+		foundInCES := false
+		if cesEnabled {
+			_, foundInCES = idsInCESs[identity.Name]
+		}
+		// The identity is definitely alive if there's a CE or CES using it.
+		alive := foundInCES || watchers.HasCEWithIdentity(identity.Name)
+
+		if alive {
 			igc.heartbeatStore.markAlive(identity.Name, timeNow)
 			continue
 		}

operator/pkg/ciliumendpointslice/endpointslice.go

@@ -5,6 +5,7 @@ package ciliumendpointslice
 
 import (
 	"context"
+	"strconv"
 	"sync"
 	"time"
 
@@ -55,6 +56,10 @@ const (
 	DefaultCESSyncTime = 500 * time.Millisecond
 )
 
+var (
+	ceSliceStore cache.Store
+)
+
 type CiliumEndpointSliceController struct {
 	// Cilium kubernetes clients to access V2 and V2alpha1 resources
 	clientV2   csv2.CiliumV2Interface
@@ -132,6 +137,7 @@ func NewCESController(
 		manager = newCESManagerIdentity(rlQueue, maxCEPsInCES)
 	}
 	cesStore := ciliumEndpointSliceInit(clientset.CiliumV2alpha1(), ctx, wg)
+	ceSliceStore = cesStore
 
 	// List all existing CESs from the api-server and cache it locally.
 	// This sync should happen before starting CEP watcher, because CEP watcher
@@ -375,3 +381,32 @@ func ciliumEndpointSliceInit(client csv2a1.CiliumV2alpha1Interface, ctx context.
 	cache.WaitForCacheSync(ctx.Done(), cesController.HasSynced)
 	return cesStore
 }
+
+// UsedIdentitiesInCESs returns all Identities that are used in CESs.
+func UsedIdentitiesInCESs() map[string]bool {
+	return usedIdentitiesInCESs(ceSliceStore)
+}
+
+// usedIdentitiesInCESs returns all Identities that are used in CESs in the
+// specified store.
+func usedIdentitiesInCESs(cesStore cache.Store) map[string]bool {
+	usedIdentities := make(map[string]bool)
+	if cesStore == nil {
+		return usedIdentities
+	}
+
+	cesObjList := cesStore.List()
+	for _, cesObj := range cesObjList {
+		ces, ok := cesObj.(*v2alpha1.CiliumEndpointSlice)
+		if !ok {
+			continue
+		}
+
+		for _, cep := range ces.Endpoints {
+			id := strconv.FormatInt(cep.IdentityID, 10)
+			usedIdentities[id] = true
+		}
+	}
+
+	return usedIdentities
+}

operator/pkg/ciliumendpointslice/endpointslice_test.go

@@ -10,6 +10,7 @@ import (
 	"sync"
 	"testing"
 
+	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/tools/cache"
@@ -214,3 +215,50 @@ func TestRemoveStaleCEPEntries(t *testing.T) {
 		})
 	}
 }
+
+func createCESWithIDs(cesName string, ids []int64) *capi_v2a1.CiliumEndpointSlice {
+	ces := &capi_v2a1.CiliumEndpointSlice{ObjectMeta: meta_v1.ObjectMeta{Name: cesName}}
+	for _, id := range ids {
+		cep := capi_v2a1.CoreCiliumEndpoint{IdentityID: id}
+		ces.Endpoints = append(ces.Endpoints, cep)
+	}
+	return ces
+}
+
+func assertEqualIDs(t *testing.T, wantIdentities, gotIdentities map[string]bool) {
+	t.Helper()
+	if diff := cmp.Diff(wantIdentities, gotIdentities); diff != "" {
+		t.Errorf("Unexpected Identites in the CES store (-want +got): \n%s", diff)
+	}
+}
+
+func TestUsedIdentitiesInCESs(t *testing.T) {
+	cesStore := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)
+
+	// Empty store.
+	gotIdentities := usedIdentitiesInCESs(cesStore)
+	wantIdentities := make(map[string]bool)
+	assertEqualIDs(t, wantIdentities, gotIdentities)
+
+	// 5 IDs in the store.
+	cesA := createCESWithIDs("cesA", []int64{1, 2, 3, 4, 5})
+	cesStore.Add(cesA)
+	wantIdentities["1"] = true
+	wantIdentities["2"] = true
+	wantIdentities["3"] = true
+	wantIdentities["4"] = true
+	wantIdentities["5"] = true
+	gotIdentities = usedIdentitiesInCESs(cesStore)
+	assertEqualIDs(t, wantIdentities, gotIdentities)
+
+	// 10 IDs in the store.
+	cesB := createCESWithIDs("cesB", []int64{10, 20, 30, 40, 50})
+	cesStore.Add(cesB)
+	wantIdentities["10"] = true
+	wantIdentities["20"] = true
+	wantIdentities["30"] = true
+	wantIdentities["40"] = true
+	wantIdentities["50"] = true
+	gotIdentities = usedIdentitiesInCESs(cesStore)
+	assertEqualIDs(t, wantIdentities, gotIdentities)
+}