operator: Handle conflicts in CES update.

If for some reason operator has outdated version of CES it
will not be able to update such CES and it will never recover
from such state.

This can happen not only due to some othe client updating CES
but also when the update from operator succeeds but for some
reason api-server does't return OK (it can fail after updating
etcd).

Signed-off-by: Alan Kutniewski <kutniewski@google.com>

operator/pkg/ciliumendpointslice/endpointslice.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/sirupsen/logrus"
 	"golang.org/x/time/rate"
+	"k8s.io/apimachinery/pkg/api/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/tools/cache"
@@ -317,6 +318,14 @@ func (c *CiliumEndpointSliceController) handleErr(err error, key interface{}) {
 		metrics.CiliumEndpointSliceSyncErrors.Inc()
 	}
 
+	if errors.IsConflict(err) {
+		// Update metadata of the object from store on conflict
+		obj, exists, err := c.ciliumEndpointSliceStore.GetByKey(key.(string))
+		if err == nil && exists {
+			c.Manager.updateCESInCache(obj.(*v2alpha1.CiliumEndpointSlice), false)
+		}
+	}
+
 	if c.queue.NumRequeues(key) < maxRetries {
 		c.queue.AddRateLimited(key)
 		return

operator/pkg/ciliumendpointslice/endpointslice_test.go

@@ -6,13 +6,16 @@ package ciliumendpointslice
 import (
 	"context"
 	"errors"
+	"fmt"
 	"strconv"
 	"sync"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/stretchr/testify/assert"
+	k8s_errors "k8s.io/apimachinery/pkg/api/errors"
 	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/tools/cache"
 
 	v2 "github.com/cilium/cilium/pkg/k8s/apis/cilium.io/v2"
@@ -262,3 +265,30 @@ func TestUsedIdentitiesInCESs(t *testing.T) {
 	gotIdentities = usedIdentitiesInCESs(cesStore)
 	assertEqualIDs(t, wantIdentities, gotIdentities)
 }
+
+func TestHandleErr(t *testing.T) {
+	t.Run("Cache is updated on conflict", func(t *testing.T) {
+		_, client := client.NewFakeClientset()
+		cesController := NewCESController(context.Background(), &sync.WaitGroup{}, client, 5, cesIdentityBasedSlicing, 10, 20)
+		manager := cesController.Manager
+
+		key := "some-ces"
+		initialCES := createCESWithIDs(key, []int64{1, 2})
+		initialCES.Generation = 1
+		manager.createCES(key)
+		manager.updateCESInCache(initialCES, true)
+		updatedCES := createCESWithIDs(key, []int64{1, 2})
+		updatedCES.Generation = 2
+		cesController.ciliumEndpointSliceStore.Add(updatedCES)
+
+		var err error
+		err = k8s_errors.NewConflict(
+			schema.GroupResource{Group: "", Resource: "ciliumendpointslices"},
+			key,
+			fmt.Errorf("conflict"))
+		cesController.handleErr(err, key)
+
+		managerCES, _ := manager.getCESFromCache(key)
+		assert.Equal(t, updatedCES.Generation, managerCES.Generation)
+	})
+}