connectivity: add full egress gateway test suite

Signed-off-by: Gilberto Bertin <jibi@cilium.io>

connectivity/check/context.go

@@ -821,6 +821,13 @@ func (ct *ConnectivityTest) PingCommand(peer TestPeer, ipFam IPFamily) []string
 	return cmd
 }
 
+func (ct *ConnectivityTest) DigCommand(peer TestPeer, ipFam IPFamily) []string {
+	cmd := []string{"dig", "+time=2", "kubernetes"}
+
+	cmd = append(cmd, fmt.Sprintf("@%s", peer.Address(ipFam)))
+	return cmd
+}
+
 func (ct *ConnectivityTest) RandomClientPod() *Pod {
 	for _, p := range ct.clientPods {
 		return &p

connectivity/check/deployment.go

@@ -1304,10 +1304,21 @@ func (ct *ConnectivityTest) patchEchoServicesWithExternalIPs(ctx context.Context
 
 	ensureFrontend := func() error {
 		for _, client := range ct.clients.clients() {
+			fmt.Println("client:", client.Config.Host)
+
 			for _, ciliumPod := range ct.CiliumPods() {
 				for _, service := range patchedServices {
 					for _, ip := range service.Spec.ExternalIPs {
-						cmd := []string{"sh", "-c",
+						cmd := []string{"sh", "-c", "cilium bpf lb list --frontends"}
+
+						out, err := client.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd)
+						if err != nil {
+							return err
+						}
+
+						fmt.Println(out.String())
+
+						cmd = []string{"sh", "-c",
 							fmt.Sprintf("cilium bpf lb list --frontends | grep %s:%d", ip, service.Spec.Ports[0].Port)}
 
 						fmt.Println(strings.Join(cmd, " "))

connectivity/check/test.go

@@ -452,6 +452,21 @@ func (t *Test) WithCiliumEgressGatewayPolicy(policy string) *Test {
 
 		// Set the egress gateway node
 		pl[i].Spec.EgressGateway.NodeSelector.MatchLabels["kubernetes.io/hostname"] = egressGatewayNode
+
+		// Set the excluded CIDR
+		if len(pl[i].Spec.ExcludedCIDRs) != 0 {
+			excludedCIDRs := []v2.IPv4CIDR{}
+			for _, nodeWithoutCilium := range t.NodesWithoutCilium() {
+				node, err := t.Context().K8sClient().GetNode(context.Background(), nodeWithoutCilium, metav1.GetOptions{})
+				if err != nil {
+					t.Fatalf("Can't get node")
+				}
+
+				excludedCIDRs = append(excludedCIDRs, v2.IPv4CIDR(fmt.Sprintf("%s/32", node.Status.Addresses[0].Address)))
+			}
+
+			pl[i].Spec.ExcludedCIDRs = excludedCIDRs
+		}
 	}
 
 	if err := t.addCEGPs(pl...); err != nil {

connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml

@@ -0,0 +1,18 @@
+apiVersion: cilium.io/v2
+kind: CiliumEgressGatewayPolicy
+metadata:
+  name: cegp-sample-excluded-cidrs
+spec:
+  selectors:
+  - podSelector:
+      matchLabels:
+        io.kubernetes.pod.namespace: cilium-test
+        kind: client
+  destinationCIDRs:
+  - 0.0.0.0/0
+  excludedCIDRs:
+  - NODE_WITHOUT_CILIUM_PLACEHOLDER/32
+  egressGateway:
+    nodeSelector:
+      matchLabels:
+        kubernetes.io/hostname: NODE_NAME_PLACEHOLDER

connectivity/manifests/egress-gateway-policy.yaml

@@ -14,3 +14,19 @@ spec:
     nodeSelector:
       matchLabels:
         kubernetes.io/hostname: NODE_NAME_PLACEHOLDER
+---
+apiVersion: cilium.io/v2
+kind: CiliumEgressGatewayPolicy
+metadata:
+  name: cegp-sample-echo-service
+spec:
+  selectors:
+  - podSelector:
+      matchLabels:
+        kind: echo
+  destinationCIDRs:
+  - 0.0.0.0/0
+  egressGateway:
+    nodeSelector:
+      matchLabels:
+        kubernetes.io/hostname: NODE_NAME_PLACEHOLDER

connectivity/suite.go

@@ -11,6 +11,8 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/cilium/cilium/pkg/versioncheck"
+
 	"github.com/cilium/cilium-cli/connectivity/check"
 	"github.com/cilium/cilium-cli/connectivity/manifests/template"
 	"github.com/cilium/cilium-cli/connectivity/tests"
@@ -157,6 +159,9 @@ var (
 
 	//go:embed manifests/egress-gateway-policy.yaml
 	egressGatewayPolicyYAML string
+
+	//go:embed manifests/egress-gateway-policy-excluded-cidrs.yaml
+	egressGatewayPolicyExcludedCIDRsYAML string
 )
 
 var (
@@ -734,6 +739,16 @@ func Run(ctx context.Context, ct *check.ConnectivityTest, addExtraTests func(*ch
 			tests.EgressGateway(),
 		)
 
+	if versioncheck.MustCompile(">=1.14.0")(ct.CiliumVersion) {
+		ct.NewTest("egress-gateway-excluded-cidrs").
+			WithCiliumEgressGatewayPolicy(egressGatewayPolicyExcludedCIDRsYAML).
+			WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureEgressGateway),
+				check.RequireFeatureEnabled(check.FeatureNodeWithoutCilium)).
+			WithScenarios(
+				tests.EgressGatewayExcludedCIDRs(),
+			)
+	}
+
 	// The following tests have DNS redirect policies. They should be executed last.
 
 	ct.NewTest("north-south-loadbalancing-with-l7-policy").

connectivity/tests/egressgateway-excluded-cirds.go

@@ -0,0 +1,169 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright Authors of Cilium
+
+package tests
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/cilium/cilium-cli/connectivity/check"
+	"github.com/cilium/cilium-cli/defaults"
+	"github.com/cilium/cilium-cli/internal/utils"
+)
+
+// EgressGatewayExcludedCIDRs is a test case which, given the cegp-sample
+// CiliumEgressGatewayExcludedCIDRsPolicy targeting:
+// - a couple of client pods (kind=client) as source
+// - the 0.0.0.0/0 destination CIDR
+// - kind-worker2 as gateway node
+//
+// ensures that traffic from both clients reaches the echo-external service with
+// the egress IP of the gateway node.
+func EgressGatewayExcludedCIDRs() check.Scenario {
+	return &egressGatewayExcludedCIDRs{}
+}
+
+type egressGatewayExcludedCIDRs struct {
+	egressGatewayNode string
+}
+
+func (s *egressGatewayExcludedCIDRs) Name() string {
+	return "egress-gateway-excluded-cidrs"
+}
+
+func (s *egressGatewayExcludedCIDRs) Run(ctx context.Context, t *check.Test) {
+	ct := t.Context()
+
+	s.egressGatewayNode = t.EgressGatewayNode()
+	if s.egressGatewayNode == "" {
+		t.Fatal("Cannot get egress gateway node")
+	}
+
+	s.waitForBpfPolicyEntries(ctx, t)
+
+	// Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service)
+	i := 0
+	for _, client := range ct.ClientPods() {
+		client := client
+
+		for _, externalEcho := range ct.ExternalEchoPods() {
+			t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, externalEcho, check.IPFamilyV4).Run(func(a *check.Action) {
+				a.ExecInPod(ctx, ct.CurlClientIPCommand(externalEcho, check.IPFamilyV4))
+				clientIP := extractClientIPFromResponse(a.CmdOutput())
+
+				if !clientIP.Equal(net.ParseIP(client.Pod.Status.HostIP)) {
+					t.Fatal("Request reached external echo service with wrong source IP")
+				}
+			})
+			i++
+		}
+	}
+}
+
+// bpfEgressGatewayExcludedCIDRsPolicyEntry represents an entry in the BPF egress gateway
+// policy map
+type bpfEgressGatewayExcludedCIDRsPolicyEntry struct {
+	SourceIP  string
+	DestCIDR  string
+	EgressIP  string
+	GatewayIP string
+}
+
+// matches is an helper used to compare the receiver bpfEgressGatewayExcludedCIDRsPolicyEntry
+// with another entry
+func (e *bpfEgressGatewayExcludedCIDRsPolicyEntry) matches(t bpfEgressGatewayExcludedCIDRsPolicyEntry) bool {
+	return t.SourceIP == e.SourceIP &&
+		t.DestCIDR == e.DestCIDR &&
+		t.EgressIP == e.EgressIP &&
+		t.GatewayIP == e.GatewayIP
+}
+
+// waitForBpfPolicyEntries waits for the egress gateway policy maps on each node
+// to be populated with the entries for the cegp-sample CiliumEgressGatewayExcludedCIDRsPolicy
+func (s *egressGatewayExcludedCIDRs) waitForBpfPolicyEntries(ctx context.Context, t *check.Test) {
+	ct := t.Context()
+
+	w := utils.NewWaitObserver(ctx, utils.WaitParameters{Timeout: 10 * time.Second})
+	defer w.Cancel()
+
+	ensureBpfPolicyEntries := func() error {
+		gatewayNodeInternalIP := getGatewayNodeInternalIP(ct, s.egressGatewayNode)
+		if gatewayNodeInternalIP == nil {
+			t.Fatalf("Cannot retrieve internal IP of gateway node")
+		}
+
+		for _, ciliumPod := range ct.CiliumPods() {
+			for _, nodeWithoutCilium := range t.NodesWithoutCilium() {
+				node, err := t.Context().K8sClient().GetNode(context.Background(), nodeWithoutCilium, metav1.GetOptions{})
+				if err != nil {
+					t.Fatalf("Cannot retrieve external node")
+				}
+
+				egressIP := "0.0.0.0"
+				if ciliumPod.Pod.Spec.NodeName == s.egressGatewayNode {
+					egressIP = gatewayNodeInternalIP.String()
+				}
+
+				targetEntries := []bpfEgressGatewayExcludedCIDRsPolicyEntry{}
+				for _, client := range ct.ClientPods() {
+					targetEntries = append(targetEntries,
+						bpfEgressGatewayExcludedCIDRsPolicyEntry{
+							SourceIP:  client.Pod.Status.PodIP,
+							DestCIDR:  "0.0.0.0/0",
+							EgressIP:  egressIP,
+							GatewayIP: gatewayNodeInternalIP.String(),
+						})
+
+					targetEntries = append(targetEntries,
+						bpfEgressGatewayExcludedCIDRsPolicyEntry{
+							SourceIP:  client.Pod.Status.PodIP,
+							DestCIDR:  fmt.Sprintf("%s/32", node.Status.Addresses[0].Address),
+							EgressIP:  egressIP,
+							GatewayIP: "Excluded CIDR",
+						})
+				}
+
+				cmd := strings.Split("cilium bpf egress list -o json", " ")
+				stdout, err := ciliumPod.K8sClient.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd)
+				if err != nil {
+					t.Fatal("failed to run cilium bpf egress list command: %w", err)
+				}
+
+				entries := []bpfEgressGatewayExcludedCIDRsPolicyEntry{}
+				json.Unmarshal(stdout.Bytes(), &entries)
+
+			nextTargetEntry:
+				for _, targetEntry := range targetEntries {
+					for _, entry := range entries {
+						if targetEntry.matches(entry) {
+							continue nextTargetEntry
+						}
+					}
+
+					return fmt.Errorf("Could not find egress gateway policy entry matching %+v", targetEntry)
+				}
+			}
+		}
+
+		return nil
+	}
+
+	for {
+		if err := ensureBpfPolicyEntries(); err != nil {
+			if err := w.Retry(err); err != nil {
+				t.Fatal("Failed to ensure egress gateway policy map is properly populated:", err)
+			}
+
+			continue
+		}
+
+		return
+	}
+}