diff --git a/connectivity/check/context.go b/connectivity/check/context.go index 2a54c3264b..0cb700a400 100644 --- a/connectivity/check/context.go +++ b/connectivity/check/context.go @@ -821,6 +821,13 @@ func (ct *ConnectivityTest) PingCommand(peer TestPeer, ipFam IPFamily) []string return cmd } +func (ct *ConnectivityTest) DigCommand(peer TestPeer, ipFam IPFamily) []string { + cmd := []string{"dig", "+time=2", "kubernetes"} + + cmd = append(cmd, fmt.Sprintf("@%s", peer.Address(ipFam))) + return cmd +} + func (ct *ConnectivityTest) RandomClientPod() *Pod { for _, p := range ct.clientPods { return &p diff --git a/connectivity/check/deployment.go b/connectivity/check/deployment.go index ac44539637..021ddb57e1 100644 --- a/connectivity/check/deployment.go +++ b/connectivity/check/deployment.go @@ -1304,10 +1304,21 @@ func (ct *ConnectivityTest) patchEchoServicesWithExternalIPs(ctx context.Context ensureFrontend := func() error { for _, client := range ct.clients.clients() { + fmt.Println("client:", client.Config.Host) + for _, ciliumPod := range ct.CiliumPods() { for _, service := range patchedServices { for _, ip := range service.Spec.ExternalIPs { - cmd := []string{"sh", "-c", + cmd := []string{"sh", "-c", "cilium bpf lb list --frontends"} + + out, err := client.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd) + if err != nil { + return err + } + + fmt.Println(out.String()) + + cmd = []string{"sh", "-c", fmt.Sprintf("cilium bpf lb list --frontends | grep %s:%d", ip, service.Spec.Ports[0].Port)} fmt.Println(strings.Join(cmd, " ")) diff --git a/connectivity/check/test.go b/connectivity/check/test.go index 722f96cc52..cfef4f8c62 100644 --- a/connectivity/check/test.go +++ b/connectivity/check/test.go @@ -452,6 +452,21 @@ func (t *Test) WithCiliumEgressGatewayPolicy(policy string) *Test { // Set the egress gateway node pl[i].Spec.EgressGateway.NodeSelector.MatchLabels["kubernetes.io/hostname"] = egressGatewayNode + + // Set the excluded CIDR + if len(pl[i].Spec.ExcludedCIDRs) != 0 { + excludedCIDRs := []v2.IPv4CIDR{} + for _, nodeWithoutCilium := range t.NodesWithoutCilium() { + node, err := t.Context().K8sClient().GetNode(context.Background(), nodeWithoutCilium, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Can't get node") + } + + excludedCIDRs = append(excludedCIDRs, v2.IPv4CIDR(fmt.Sprintf("%s/32", node.Status.Addresses[0].Address))) + } + + pl[i].Spec.ExcludedCIDRs = excludedCIDRs + } } if err := t.addCEGPs(pl...); err != nil { diff --git a/connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml b/connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml new file mode 100644 index 0000000000..3bdff0bff0 --- /dev/null +++ b/connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml @@ -0,0 +1,18 @@ +apiVersion: cilium.io/v2 +kind: CiliumEgressGatewayPolicy +metadata: + name: cegp-sample-excluded-cidrs +spec: + selectors: + - podSelector: + matchLabels: + io.kubernetes.pod.namespace: cilium-test + kind: client + destinationCIDRs: + - 0.0.0.0/0 + excludedCIDRs: + - NODE_WITHOUT_CILIUM_PLACEHOLDER/32 + egressGateway: + nodeSelector: + matchLabels: + kubernetes.io/hostname: NODE_NAME_PLACEHOLDER diff --git a/connectivity/manifests/egress-gateway-policy.yaml b/connectivity/manifests/egress-gateway-policy.yaml index 392856073e..6cc6f45a14 100644 --- a/connectivity/manifests/egress-gateway-policy.yaml +++ b/connectivity/manifests/egress-gateway-policy.yaml @@ -14,3 +14,19 @@ spec: nodeSelector: matchLabels: kubernetes.io/hostname: NODE_NAME_PLACEHOLDER +--- +apiVersion: cilium.io/v2 +kind: CiliumEgressGatewayPolicy +metadata: + name: cegp-sample-echo-service +spec: + selectors: + - podSelector: + matchLabels: + kind: echo + destinationCIDRs: + - 0.0.0.0/0 + egressGateway: + nodeSelector: + matchLabels: + kubernetes.io/hostname: NODE_NAME_PLACEHOLDER diff --git a/connectivity/suite.go b/connectivity/suite.go index 725253a191..a723a1e629 100644 --- a/connectivity/suite.go +++ b/connectivity/suite.go @@ -11,6 +11,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "github.com/cilium/cilium/pkg/versioncheck" + "github.com/cilium/cilium-cli/connectivity/check" "github.com/cilium/cilium-cli/connectivity/manifests/template" "github.com/cilium/cilium-cli/connectivity/tests" @@ -157,6 +159,9 @@ var ( //go:embed manifests/egress-gateway-policy.yaml egressGatewayPolicyYAML string + + //go:embed manifests/egress-gateway-policy-excluded-cidrs.yaml + egressGatewayPolicyExcludedCIDRsYAML string ) var ( @@ -734,6 +739,16 @@ func Run(ctx context.Context, ct *check.ConnectivityTest, addExtraTests func(*ch tests.EgressGateway(), ) + if versioncheck.MustCompile(">=1.14.0")(ct.CiliumVersion) { + ct.NewTest("egress-gateway-excluded-cidrs"). + WithCiliumEgressGatewayPolicy(egressGatewayPolicyExcludedCIDRsYAML). + WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureEgressGateway), + check.RequireFeatureEnabled(check.FeatureNodeWithoutCilium)). + WithScenarios( + tests.EgressGatewayExcludedCIDRs(), + ) + } + // The following tests have DNS redirect policies. They should be executed last. ct.NewTest("north-south-loadbalancing-with-l7-policy"). diff --git a/connectivity/tests/egressgateway-excluded-cirds.go b/connectivity/tests/egressgateway-excluded-cirds.go new file mode 100644 index 0000000000..fd79de3316 --- /dev/null +++ b/connectivity/tests/egressgateway-excluded-cirds.go @@ -0,0 +1,169 @@ +// SPDX-License-Identifier: Apache-2.0 +// Copyright Authors of Cilium + +package tests + +import ( + "context" + "encoding/json" + "fmt" + "net" + "strings" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/cilium/cilium-cli/connectivity/check" + "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/utils" +) + +// EgressGatewayExcludedCIDRs is a test case which, given the cegp-sample +// CiliumEgressGatewayExcludedCIDRsPolicy targeting: +// - a couple of client pods (kind=client) as source +// - the 0.0.0.0/0 destination CIDR +// - kind-worker2 as gateway node +// +// ensures that traffic from both clients reaches the echo-external service with +// the egress IP of the gateway node. +func EgressGatewayExcludedCIDRs() check.Scenario { + return &egressGatewayExcludedCIDRs{} +} + +type egressGatewayExcludedCIDRs struct { + egressGatewayNode string +} + +func (s *egressGatewayExcludedCIDRs) Name() string { + return "egress-gateway-excluded-cidrs" +} + +func (s *egressGatewayExcludedCIDRs) Run(ctx context.Context, t *check.Test) { + ct := t.Context() + + s.egressGatewayNode = t.EgressGatewayNode() + if s.egressGatewayNode == "" { + t.Fatal("Cannot get egress gateway node") + } + + s.waitForBpfPolicyEntries(ctx, t) + + // Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service) + i := 0 + for _, client := range ct.ClientPods() { + client := client + + for _, externalEcho := range ct.ExternalEchoPods() { + t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, externalEcho, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.CurlClientIPCommand(externalEcho, check.IPFamilyV4)) + clientIP := extractClientIPFromResponse(a.CmdOutput()) + + if !clientIP.Equal(net.ParseIP(client.Pod.Status.HostIP)) { + t.Fatal("Request reached external echo service with wrong source IP") + } + }) + i++ + } + } +} + +// bpfEgressGatewayExcludedCIDRsPolicyEntry represents an entry in the BPF egress gateway +// policy map +type bpfEgressGatewayExcludedCIDRsPolicyEntry struct { + SourceIP string + DestCIDR string + EgressIP string + GatewayIP string +} + +// matches is an helper used to compare the receiver bpfEgressGatewayExcludedCIDRsPolicyEntry +// with another entry +func (e *bpfEgressGatewayExcludedCIDRsPolicyEntry) matches(t bpfEgressGatewayExcludedCIDRsPolicyEntry) bool { + return t.SourceIP == e.SourceIP && + t.DestCIDR == e.DestCIDR && + t.EgressIP == e.EgressIP && + t.GatewayIP == e.GatewayIP +} + +// waitForBpfPolicyEntries waits for the egress gateway policy maps on each node +// to be populated with the entries for the cegp-sample CiliumEgressGatewayExcludedCIDRsPolicy +func (s *egressGatewayExcludedCIDRs) waitForBpfPolicyEntries(ctx context.Context, t *check.Test) { + ct := t.Context() + + w := utils.NewWaitObserver(ctx, utils.WaitParameters{Timeout: 10 * time.Second}) + defer w.Cancel() + + ensureBpfPolicyEntries := func() error { + gatewayNodeInternalIP := getGatewayNodeInternalIP(ct, s.egressGatewayNode) + if gatewayNodeInternalIP == nil { + t.Fatalf("Cannot retrieve internal IP of gateway node") + } + + for _, ciliumPod := range ct.CiliumPods() { + for _, nodeWithoutCilium := range t.NodesWithoutCilium() { + node, err := t.Context().K8sClient().GetNode(context.Background(), nodeWithoutCilium, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Cannot retrieve external node") + } + + egressIP := "0.0.0.0" + if ciliumPod.Pod.Spec.NodeName == s.egressGatewayNode { + egressIP = gatewayNodeInternalIP.String() + } + + targetEntries := []bpfEgressGatewayExcludedCIDRsPolicyEntry{} + for _, client := range ct.ClientPods() { + targetEntries = append(targetEntries, + bpfEgressGatewayExcludedCIDRsPolicyEntry{ + SourceIP: client.Pod.Status.PodIP, + DestCIDR: "0.0.0.0/0", + EgressIP: egressIP, + GatewayIP: gatewayNodeInternalIP.String(), + }) + + targetEntries = append(targetEntries, + bpfEgressGatewayExcludedCIDRsPolicyEntry{ + SourceIP: client.Pod.Status.PodIP, + DestCIDR: fmt.Sprintf("%s/32", node.Status.Addresses[0].Address), + EgressIP: egressIP, + GatewayIP: "Excluded CIDR", + }) + } + + cmd := strings.Split("cilium bpf egress list -o json", " ") + stdout, err := ciliumPod.K8sClient.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd) + if err != nil { + t.Fatal("failed to run cilium bpf egress list command: %w", err) + } + + entries := []bpfEgressGatewayExcludedCIDRsPolicyEntry{} + json.Unmarshal(stdout.Bytes(), &entries) + + nextTargetEntry: + for _, targetEntry := range targetEntries { + for _, entry := range entries { + if targetEntry.matches(entry) { + continue nextTargetEntry + } + } + + return fmt.Errorf("Could not find egress gateway policy entry matching %+v", targetEntry) + } + } + } + + return nil + } + + for { + if err := ensureBpfPolicyEntries(); err != nil { + if err := w.Retry(err); err != nil { + t.Fatal("Failed to ensure egress gateway policy map is properly populated:", err) + } + + continue + } + + return + } +} diff --git a/connectivity/tests/egressgateway.go b/connectivity/tests/egressgateway.go index 43e8d3a588..10d077ff7a 100644 --- a/connectivity/tests/egressgateway.go +++ b/connectivity/tests/egressgateway.go @@ -14,6 +14,8 @@ import ( "github.com/cilium/cilium-cli/connectivity/check" "github.com/cilium/cilium-cli/defaults" "github.com/cilium/cilium-cli/internal/utils" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // EgressGateway is a test case which, given the cegp-sample @@ -44,14 +46,47 @@ func (s *egressGateway) Run(ctx context.Context, t *check.Test) { t.Fatal("Cannot get egress gateway node") } - egressIP := s.getGatewayNodeInternalIP(ct) + egressIP := getGatewayNodeInternalIP(ct, s.egressGatewayNode) s.waitForBpfPolicyEntries(ctx, t) + // Ping hosts (pod to host connectivity) i := 0 for _, client := range ct.ClientPods() { client := client + for _, dst := range ct.HostNetNSPodsByNode() { + dst := dst + + t.NewAction(s, fmt.Sprintf("ping-%d", i), &client, &dst, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.PingCommand(dst, check.IPFamilyV4)) + }) + i++ + } + } + + // DNS query (pod to service connectivity) + i = 0 + for _, client := range ct.ClientPods() { + client := client + + kubeDNSService, err := ct.K8sClient().GetService(ctx, "kube-system", "kube-dns", metav1.GetOptions{}) + if err != nil { + t.Fatal("Cannot get kube-dns service") + } + kubeDNSServicePeer := check.Service{Service: kubeDNSService} + + t.NewAction(s, fmt.Sprintf("dig-%d", i), &client, kubeDNSServicePeer, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.DigCommand(kubeDNSServicePeer, check.IPFamilyV4)) + }) + i++ + } + + // Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service) + i = 0 + for _, client := range ct.ClientPods() { + client := client + for _, externalEcho := range ct.ExternalEchoPods() { t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, externalEcho, check.IPFamilyV4).Run(func(a *check.Action) { a.ExecInPod(ctx, ct.CurlClientIPCommand(externalEcho, check.IPFamilyV4)) @@ -64,30 +99,72 @@ func (s *egressGateway) Run(ctx context.Context, t *check.Test) { i++ } } -} -// getGatewayNodeInternalIP returns the k8s internal IP of the node acting as -// gateway for this test -func (s *egressGateway) getGatewayNodeInternalIP(ct *check.ConnectivityTest) net.IP { - gatewayNode, ok := ct.Nodes()[s.egressGatewayNode] - if !ok { - return nil - } + // When connecting from outside the cluster to a nodeport service whose pods are selected by an egress policy, + // the reply traffic should not be SNATed with the egress IP + i = 0 + for _, client := range ct.ExternalEchoPods() { + client := client - for _, addr := range gatewayNode.Status.Addresses { - if addr.Type != "InternalIP" { - continue + for _, echo := range ct.EchoServices() { + // convert the service to a ServiceExternalIP as we want to access it through its external IP + echo := echo.ToExternalIPService() + + t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, echo, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.CurlClientIPCommand(echo, check.IPFamilyV4)) + }) + i++ } + } - ip := net.ParseIP(addr.Address) - if ip == nil || ip.To4() == nil { - continue + if status, ok := ct.Feature(check.FeatureTunnel); ok && !status.Enabled { + // When connecting from outside the cluster directly to a pod which is selected by an egress policy, the + // reply traffic should not be SNATed with the egress IP (only connections originating from these pods + // should go through egress gateway). + // + // This test is executed only when Cilium is running in direct routing mode, since we can simply add a + // route on the node outside the cluster to direct pod's traffic to the node where the pod is running + // (while in tunneling mode we would need the external node to send the traffic over the tunnel) + + for _, echoPod := range ct.EchoPods() { + targetPodHostIP := echoPod.Pod.Status.HostIP + targetPodIP := echoPod.Pod.Status.PodIP + + for _, externalNode := range ct.NodesWithoutCilium() { + for node, hostNetNSPod := range ct.HostNetNSPodsByNode() { + if node != externalNode { + continue + } + + cmd := []string{"ip", "route", "add", targetPodIP, "via", targetPodHostIP} + _, err := hostNetNSPod.K8sClient.ExecInPod(ctx, hostNetNSPod.Pod.Namespace, hostNetNSPod.Pod.Name, "", cmd) + if err != nil { + t.Fatalf("failed to add ip route: %w", err) + } + + defer func(hostNetNSPod check.Pod) { + cmd = []string{"ip", "route", "del", targetPodIP, "via", targetPodHostIP} + _, err = hostNetNSPod.K8sClient.ExecInPod(ctx, hostNetNSPod.Pod.Namespace, hostNetNSPod.Pod.Name, "", cmd) + if err != nil { + t.Fatalf("failed to delete ip route: %w", err) + } + }(hostNetNSPod) + } + } } - return ip - } + i = 0 + for _, client := range ct.ExternalEchoPods() { + client := client - return nil + for _, echo := range ct.EchoPods() { + t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, echo, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.CurlClientIPCommand(echo, check.IPFamilyV4)) + }) + i++ + } + } + } } // bpfEgressGatewayPolicyEntry represents an entry in the BPF egress gateway @@ -117,7 +194,7 @@ func (s *egressGateway) waitForBpfPolicyEntries(ctx context.Context, t *check.Te defer w.Cancel() ensureBpfPolicyEntries := func() error { - gatewayNodeInternalIP := s.getGatewayNodeInternalIP(ct) + gatewayNodeInternalIP := getGatewayNodeInternalIP(ct, s.egressGatewayNode) if gatewayNodeInternalIP == nil { t.Fatalf("Cannot retrieve internal IP of gateway node") } @@ -139,6 +216,16 @@ func (s *egressGateway) waitForBpfPolicyEntries(ctx context.Context, t *check.Te }) } + for _, echo := range ct.EchoPods() { + targetEntries = append(targetEntries, + bpfEgressGatewayPolicyEntry{ + SourceIP: echo.Pod.Status.PodIP, + DestCIDR: "0.0.0.0/0", + EgressIP: egressIP, + GatewayIP: gatewayNodeInternalIP.String(), + }) + } + cmd := strings.Split("cilium bpf egress list -o json", " ") stdout, err := ciliumPod.K8sClient.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd) if err != nil { @@ -176,6 +263,30 @@ func (s *egressGateway) waitForBpfPolicyEntries(ctx context.Context, t *check.Te } } +// getGatewayNodeInternalIP returns the k8s internal IP of the node acting as +// gateway for this test +func getGatewayNodeInternalIP(ct *check.ConnectivityTest, egressGatewayNode string) net.IP { + gatewayNode, ok := ct.Nodes()[egressGatewayNode] + if !ok { + return nil + } + + for _, addr := range gatewayNode.Status.Addresses { + if addr.Type != "InternalIP" { + continue + } + + ip := net.ParseIP(addr.Address) + if ip == nil || ip.To4() == nil { + continue + } + + return ip + } + + return nil +} + // extractClientIPFromResponse extracts the client IP from the response of the // echo-external service func extractClientIPFromResponse(res string) net.IP { diff --git a/k8s/client.go b/k8s/client.go index abe38547b2..e7fa364008 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -673,6 +673,10 @@ func (c *Client) ListCiliumEnvoyConfigs(ctx context.Context, namespace string, o return c.CiliumClientset.CiliumV2().CiliumEnvoyConfigs(namespace).List(ctx, options) } +func (c *Client) GetNode(ctx context.Context, name string, opts metav1.GetOptions) (*corev1.Node, error) { + return c.Clientset.CoreV1().Nodes().Get(ctx, name, opts) +} + func (c *Client) ListNodes(ctx context.Context, options metav1.ListOptions) (*corev1.NodeList, error) { return c.Clientset.CoreV1().Nodes().List(ctx, options) }