k8s, policy: enable toServices rules for high-scale ipcache

Normally, toServices rules only allow Endpoints that are external to the
cluster. This is to prevent conflict with entries already existing in
the ipcache.

However, this is not relevant with highscale ipcache, where podips are
normally excluded from the ipcache anyways. So, allow all services to be
specified in toServices egress rules when highscale is enabled.

Signed-off-by: Casey Callendrello <cdc@isovalent.com>

pkg/k8s/rule_translate.go

@@ -9,6 +9,7 @@ import (
 	"net/netip"
 
 	"github.com/cilium/cilium/pkg/k8s/slim/k8s/apis/labels"
+	"github.com/cilium/cilium/pkg/option"
 	"github.com/cilium/cilium/pkg/policy"
 	"github.com/cilium/cilium/pkg/policy/api"
 	"github.com/cilium/cilium/pkg/slices"
@@ -228,7 +229,14 @@ func PreprocessRules(r api.Rules, cache *ServiceCache) error {
 		}
 		for ns, ep := range cache.endpoints {
 			svc, ok := cache.services[ns]
-			if ok && svc.IsExternal() {
+			// Normally, only services without a label selector (i.e. empty services)
+			// are allowed as targets of a toServices rule.
+			// This is to minimize the chances of a pod IP being selected by this rule, which might
+			// cause conflicting entries in the ipcache.
+			//
+			// This requirement, however, is dropped for HighScale IPCache mode, because pod IPs are
+			// normally excluded from the ipcache regardless.
+			if ok && (option.Config.EnableHighScaleIPcache || svc.IsExternal()) {
 				eps := ep.GetEndpoints()
 				if eps != nil {
 					t := NewK8sTranslator(ns, Endpoints{}, *eps, false, svc.Labels)

pkg/k8s/watchers/watcher.go

@@ -608,7 +608,14 @@ func (k *K8sWatcher) k8sServiceHandler() {
 				scopedLog.WithError(err).Error("Unable to add/update service to implement k8s event")
 			}
 
-			if !svc.IsExternal() {
+			// Normally, only services without a label selector (i.e. "bottomless" or empty services)
+			// are allowed as targets of a toServices rule.
+			// This is to minimize the chances of a pod IP being selected by this rule, which might
+			// cause conflicting entries in the ipcache.
+			//
+			// This requirement, however, is dropped for HighScale IPCache mode, because pod IPs are
+			// normally excluded from the ipcache regardless.
+			if !option.Config.EnableHighScaleIPcache && !svc.IsExternal() {
 				return
 			}
 
@@ -638,7 +645,7 @@ func (k *K8sWatcher) k8sServiceHandler() {
 				scopedLog.WithError(err).Error("Unable to delete service to implement k8s event")
 			}
 
-			if !svc.IsExternal() {
+			if !option.Config.EnableHighScaleIPcache && !svc.IsExternal() {
 				return
 			}
 

test/k8s/manifests/high-scale-ipcache.yaml

@@ -81,3 +81,30 @@ spec:
     protocol: TCP
   selector:
     type: server
+---
+apiVersion: "cilium.io/v2"
+kind: CiliumNetworkPolicy
+metadata:
+  name: egress-dns
+spec:
+  endpointSelector:
+    matchLabels:
+      type: client
+  egress:
+  - toEndpoints:
+    - matchLabels:
+        k8s-app: kube-dns
+        io.kubernetes.pod.namespace: kube-system
+---
+apiVersion: "cilium.io/v2"
+kind: CiliumNetworkPolicy
+metadata:
+  name: egress-l3
+spec:
+  endpointSelector:
+    matchLabels:
+      type: client
+  egress:
+  - toServices:
+    - k8sService:
+        serviceName: http