connectivity: add full egress gateway test suite

connectivity/check/context.go

@@ -818,6 +818,13 @@ func (ct *ConnectivityTest) PingCommand(peer TestPeer, ipFam IPFamily) []string
 	return cmd
 }
 
+func (ct *ConnectivityTest) DigCommand(peer TestPeer, ipFam IPFamily) []string {
+	cmd := []string{"dig", "+time=2", "kubernetes"}
+
+	cmd = append(cmd, fmt.Sprintf("@%s", peer.Address(ipFam)))
+	return cmd
+}
+
 func (ct *ConnectivityTest) RandomClientPod() *Pod {
 	for _, p := range ct.clientPods {
 		return &p

connectivity/check/peer.go

@@ -231,6 +231,80 @@ func (s Service) FlowFilters() []*flow.FlowFilter {
 	return nil
 }
 
+func (s Service) ToNodeportService(node *v1.Node) NodeportService {
+	return NodeportService{
+		Service: s,
+		Node:    node,
+	}
+}
+
+// NodeportService wraps a Service and exposes it through its nodeport, acting as a peer in a connectivity test.
+// It implements interface TestPeer.
+type NodeportService struct {
+	Service Service
+	Node    *v1.Node
+}
+
+// Name returns name of the wrapped service.
+func (s NodeportService) Name() string {
+	return s.Service.Name()
+}
+
+// Scheme returns the scheme of the wrapped service.
+func (s NodeportService) Scheme() string {
+	return s.Service.Scheme()
+}
+
+// Path returns the path of the wrapped service.
+func (s NodeportService) Path() string {
+	return s.Service.Path()
+}
+
+// Address returns the node IP of the wrapped Service.
+func (s NodeportService) Address(family IPFamily) string {
+	if family == IPFamilyAny {
+		return s.Node.Status.Addresses[0].Address
+	}
+
+	for _, address := range s.Node.Status.Addresses {
+		if address.Type == v1.NodeInternalIP {
+			parsedAddress := net.ParseIP(address.Address)
+
+			switch family {
+			case IPFamilyV4:
+				if parsedAddress.To4() != nil {
+					return address.Address
+				}
+			case IPFamilyV6:
+				if parsedAddress.To16() != nil {
+					return address.Address
+				}
+			}
+		}
+	}
+
+	return ""
+}
+
+// Port returns the first nodeport of the wrapped Service.
+func (s NodeportService) Port() uint32 {
+	return uint32(s.Service.Service.Spec.Ports[0].NodePort)
+}
+
+// HasLabel checks if given label exists and value matches.
+func (s NodeportService) HasLabel(name, value string) bool {
+	return s.Service.HasLabel(name, value)
+}
+
+// Labels returns the copy of service labels
+func (s NodeportService) Labels() map[string]string {
+	return s.Service.Labels()
+}
+
+func (s NodeportService) FlowFilters() []*flow.FlowFilter {
+	return s.Service.FlowFilters()
+}
+
 // ExternalWorkload is an external workload acting as a peer in a
 // connectivity test. It implements interface TestPeer.
 type ExternalWorkload struct {

connectivity/check/policy.go

@@ -208,6 +208,15 @@ func deleteKNP(ctx context.Context, client *k8s.Client, knp *networkingv1.Networ
 	return nil
 }
 
+// deleteCEGP deletes a CiliumEgressGatewayPolicy from the cluster.
+func deleteCEGP(ctx context.Context, client *k8s.Client, cegp *ciliumv2.CiliumEgressGatewayPolicy) error {
+	if err := client.DeleteCiliumEgressGatewayPolicy(ctx, cegp.Name, metav1.DeleteOptions{}); err != nil {
+		return fmt.Errorf("%s/%s policy delete failed: %w", client.ClusterName(), cegp.Name, err)
+	}
+
+	return nil
+}
+
 func defaultDropReason(flow *flowpb.Flow) bool {
 	return flow.GetDropReasonDesc() != flowpb.DropReason_DROP_REASON_UNKNOWN
 }
@@ -525,7 +534,7 @@ func (t *Test) applyPolicies(ctx context.Context) error {
 
 // deletePolicies deletes a given set of network policies from the cluster.
 func (t *Test) deletePolicies(ctx context.Context) error {
-	if len(t.cnps) == 0 && len(t.knps) == 0 {
+	if len(t.cnps) == 0 && len(t.knps) == 0 && len(t.cegps) == 0 {
 		return nil
 	}
 
@@ -558,9 +567,21 @@ func (t *Test) deletePolicies(ctx context.Context) error {
 		}
 	}
 
-	// Wait for policies to be deleted on all Cilium nodes.
-	if err := t.waitCiliumPolicyRevisions(ctx, revs); err != nil {
-		return fmt.Errorf("timed out removing policies on Cilium agents: %w", err)
+	// Delete all the Test's CEGPs from all clients.
+	for _, cegp := range t.cegps {
+		t.Infof("📜 Deleting CiliumEgressGatewayPolicy '%s' from namespace '%s'..", cegp.Name, cegp.Namespace)
+		for _, client := range t.Context().clients.clients() {
+			if err := deleteCEGP(ctx, client, cegp); err != nil {
+				return fmt.Errorf("deleting CiliumEgressGatewayPolicy: %w", err)
+			}
+		}
+	}
+
+	if len(t.cnps) != 0 || len(t.knps) != 0 {
+		// Wait for policies to be deleted on all Cilium nodes.
+		if err := t.waitCiliumPolicyRevisions(ctx, revs); err != nil {
+			return fmt.Errorf("timed out removing policies on Cilium agents: %w", err)
+		}
 	}
 
 	if len(t.cnps) > 0 {
@@ -571,6 +592,10 @@ func (t *Test) deletePolicies(ctx context.Context) error {
 		t.Debugf("📜 Successfully deleted %d K8S NetworkPolicy", len(t.knps))
 	}
 
+	if len(t.cegps) > 0 {
+		t.Debugf("📜 Successfully deleted %d CiliumEgressGatewayPolicies", len(t.cegps))
+	}
+
 	return nil
 }
 

connectivity/check/test.go

@@ -9,6 +9,7 @@ import (
 	_ "embed"
 	"fmt"
 	"io"
+	"net"
 	"sync"
 	"time"
 
@@ -419,12 +420,24 @@ func (t *Test) WithK8SPolicy(policy string) *Test {
 	return t
 }
 
+const (
+	NoExcludedCIDRs = iota
+	ExternalNodeExcludedCIDRs
+)
+
+// CiliumEgressGatewayPolicyParams is used to configure how a CiliumEgressGatewayPolicy template should be configured
+// before being applied.
+type CiliumEgressGatewayPolicyParams struct {
+	// ExcludedCIDRs controls how the ExcludedCIDRs property should be configured
+	ExcludedCIDRs int
+}
+
 // WithCiliumEgressGatewayPolicy takes a string containing a YAML policy
 // document and adds the cilium egress gateway polic(y)(ies) to the scope of the
 // Test, to be applied when the test starts running. When calling this method,
 // note that the egress gateway enabled feature requirement is applied directly
 // here.
-func (t *Test) WithCiliumEgressGatewayPolicy(policy string) *Test {
+func (t *Test) WithCiliumEgressGatewayPolicy(policy string, params CiliumEgressGatewayPolicyParams) *Test {
 	pl, err := parseCiliumEgressGatewayPolicyYAML(policy)
 	if err != nil {
 		t.Fatalf("Parsing policy YAML: %s", err)
@@ -445,13 +458,28 @@ func (t *Test) WithCiliumEgressGatewayPolicy(policy string) *Test {
 			}
 		}
 
+		// Set the egress gateway node
 		egressGatewayNode := t.EgressGatewayNode()
 		if egressGatewayNode == "" {
 			t.Fatalf("Cannot find egress gateway node")
 		}
 
-		// Set the egress gateway node
 		pl[i].Spec.EgressGateway.NodeSelector.MatchLabels["kubernetes.io/hostname"] = egressGatewayNode
+
+		// Set the excluded CIDRs
+		pl[i].Spec.ExcludedCIDRs = []v2.IPv4CIDR{}
+
+		switch params.ExcludedCIDRs {
+		case ExternalNodeExcludedCIDRs:
+			for _, nodeWithoutCiliumIP := range t.Context().params.NodesWithoutCiliumIPs {
+				if parsedIP := net.ParseIP(nodeWithoutCiliumIP.IP); parsedIP.To4() == nil {
+					continue
+				}
+
+				cidr := v2.IPv4CIDR(fmt.Sprintf("%s/32", nodeWithoutCiliumIP.IP))
+				pl[i].Spec.ExcludedCIDRs = append(pl[i].Spec.ExcludedCIDRs, cidr)
+			}
+		}
 	}
 
 	if err := t.addCEGPs(pl...); err != nil {

connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml

@@ -0,0 +1,18 @@
+apiVersion: cilium.io/v2
+kind: CiliumEgressGatewayPolicy
+metadata:
+  name: cegp-sample-excluded-cidrs
+spec:
+  selectors:
+  - podSelector:
+      matchLabels:
+        io.kubernetes.pod.namespace: cilium-test
+        kind: client
+  destinationCIDRs:
+  - 0.0.0.0/0
+  excludedCIDRs:
+  - NODE_WITHOUT_CILIUM_PLACEHOLDER/32
+  egressGateway:
+    nodeSelector:
+      matchLabels:
+        kubernetes.io/hostname: NODE_NAME_PLACEHOLDER

connectivity/manifests/egress-gateway-policy.yaml

@@ -14,3 +14,19 @@ spec:
     nodeSelector:
       matchLabels:
         kubernetes.io/hostname: NODE_NAME_PLACEHOLDER
+---
+apiVersion: cilium.io/v2
+kind: CiliumEgressGatewayPolicy
+metadata:
+  name: cegp-sample-echo-service
+spec:
+  selectors:
+  - podSelector:
+      matchLabels:
+        kind: echo
+  destinationCIDRs:
+  - 0.0.0.0/0
+  egressGateway:
+    nodeSelector:
+      matchLabels:
+        kubernetes.io/hostname: NODE_NAME_PLACEHOLDER

connectivity/suite.go

@@ -11,6 +11,8 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	"github.com/cilium/cilium/pkg/versioncheck"
+
 	"github.com/cilium/cilium-cli/connectivity/check"
 	"github.com/cilium/cilium-cli/connectivity/manifests/template"
 	"github.com/cilium/cilium-cli/connectivity/tests"
@@ -157,6 +159,9 @@ var (
 
 	//go:embed manifests/egress-gateway-policy.yaml
 	egressGatewayPolicyYAML string
+
+	//go:embed manifests/egress-gateway-policy-excluded-cidrs.yaml
+	egressGatewayPolicyExcludedCIDRsYAML string
 )
 
 var (
@@ -726,13 +731,27 @@ func Run(ctx context.Context, ct *check.ConnectivityTest, addExtraTests func(*ch
 			tests.NodeToNodeEncryption(),
 		)
 
-	ct.NewTest("egress-gateway").
-		WithCiliumEgressGatewayPolicy(egressGatewayPolicyYAML).
-		WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureEgressGateway),
-			check.RequireFeatureEnabled(check.FeatureNodeWithoutCilium)).
-		WithScenarios(
-			tests.EgressGateway(),
-		)
+	if ct.Params().IncludeUnsafeTests {
+		ct.NewTest("egress-gateway").
+			WithCiliumEgressGatewayPolicy(egressGatewayPolicyYAML, check.CiliumEgressGatewayPolicyParams{}).
+			WithIPRoutesFromOutsideToPodCIDRs().
+			WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureEgressGateway),
+				check.RequireFeatureEnabled(check.FeatureNodeWithoutCilium)).
+			WithScenarios(
+				tests.EgressGateway(),
+			)
+	}
+
+	if versioncheck.MustCompile(">=1.14.0")(ct.CiliumVersion) {
+		ct.NewTest("egress-gateway-excluded-cidrs").
+			WithCiliumEgressGatewayPolicy(egressGatewayPolicyExcludedCIDRsYAML,
+				check.CiliumEgressGatewayPolicyParams{ExcludedCIDRs: check.ExternalNodeExcludedCIDRs}).
+			WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureEgressGateway),
+				check.RequireFeatureEnabled(check.FeatureNodeWithoutCilium)).
+			WithScenarios(
+				tests.EgressGatewayExcludedCIDRs(),
+			)
+	}
 
 	// The following tests have DNS redirect policies. They should be executed last.