diff --git a/connectivity/check/context.go b/connectivity/check/context.go index 8e7301280e..bf1b93da7e 100644 --- a/connectivity/check/context.go +++ b/connectivity/check/context.go @@ -302,6 +302,9 @@ func (ct *ConnectivityTest) SetupAndValidate(ctx context.Context) error { if err := ct.validateDeployment(ctx); err != nil { return err } + if err := ct.patchEchoServicesWithExternalIPs(ctx); err != nil { + return err + } if ct.params.Hubble { if err := ct.enableHubbleClient(ctx); err != nil { return fmt.Errorf("unable to create hubble client: %s", err) @@ -760,6 +763,13 @@ func (ct *ConnectivityTest) PingCommand(peer TestPeer, ipFam IPFamily) []string return cmd } +func (ct *ConnectivityTest) DigCommand(peer TestPeer, ipFam IPFamily) []string { + cmd := []string{"dig", "+time=2", "kubernetes"} + + cmd = append(cmd, fmt.Sprintf("@%s", peer.Address(ipFam))) + return cmd +} + func (ct *ConnectivityTest) RandomClientPod() *Pod { for _, p := range ct.clientPods { return &p diff --git a/connectivity/check/deployment.go b/connectivity/check/deployment.go index 1189aa7065..12c55f1d64 100644 --- a/connectivity/check/deployment.go +++ b/connectivity/check/deployment.go @@ -17,9 +17,11 @@ import ( networkingv1 "k8s.io/api/networking/v1" k8sErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "github.com/cilium/cilium-cli/defaults" + "github.com/cilium/cilium-cli/internal/utils" "github.com/cilium/cilium-cli/k8s" ) @@ -236,6 +238,7 @@ type daemonSetParameters struct { Labels map[string]string HostNetwork bool Tolerations []corev1.Toleration + Capabilities []corev1.Capability } func newDaemonSet(p daemonSetParameters) *appsv1.DaemonSet { @@ -266,7 +269,7 @@ func newDaemonSet(p daemonSetParameters) *appsv1.DaemonSet { ReadinessProbe: p.ReadinessProbe, SecurityContext: &corev1.SecurityContext{ Capabilities: &corev1.Capabilities{ - Add: []corev1.Capability{"NET_RAW"}, + Add: append([]corev1.Capability{"NET_RAW"}, p.Capabilities...), }, }, }, @@ -814,6 +817,7 @@ func (ct *ConnectivityTest) deploy(ctx context.Context) error { Tolerations: []corev1.Toleration{ {Operator: corev1.TolerationOpExists}, }, + Capabilities: []corev1.Capability{"NET_ADMIN"}, }) _, err = ct.clients.src.CreateDaemonSet(ctx, ct.params.TestNamespace, ds, metav1.CreateOptions{}) if err != nil { @@ -1202,6 +1206,111 @@ func (ct *ConnectivityTest) validateDeployment(ctx context.Context) error { return nil } +// patchEchoServicesWithExternalIPs patches the echo services (echo-same-node +// and echo-other-node) external IPs to the IP of the node running those pods. +// +// This needs to happen after validate as we need to wait for the pods to be +// scheduled on the nodes first, in order to retrieve the pods' host IPs. + +func (ct *ConnectivityTest) patchEchoServicesWithExternalIPs(ctx context.Context) error { + patchedServices := []*corev1.Service{} + + echoSamePodHostIPs := []string{} + for _, client := range ct.clients.clients() { + echoPods, err := client.ListPods(ctx, ct.params.TestNamespace, metav1.ListOptions{LabelSelector: "name=" + "echo-same-node"}) + if err != nil { + return fmt.Errorf("unable to list echo pods: %w", err) + } + + for _, echoPod := range echoPods.Items { + echoSamePodHostIPs = append(echoSamePodHostIPs, echoPod.Status.HostIP) + } + + } + + for _, client := range ct.clients.clients() { + patch := fmt.Sprintf(`{"spec":{"externalIPs":["%s"], "externalTrafficPolicy": "Local"}}`, strings.Join(echoSamePodHostIPs, ",")) + + service, err := client.PatchService(ctx, ct.params.TestNamespace, echoSameNodeDeploymentName, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}) + if err != nil { + return err + } + + fmt.Println("Patched service:", service.Spec.ExternalIPs) + + s := ct.echoServices[service.Name] + s.Service = service.DeepCopy() + ct.echoServices[service.Name] = s + + patchedServices = append(patchedServices, service) + } + + echoOtherPodHostIPs := []string{} + for _, client := range ct.clients.clients() { + echoPods, err := client.ListPods(ctx, ct.params.TestNamespace, metav1.ListOptions{LabelSelector: "name=" + "echo-other-node"}) + if err != nil { + return fmt.Errorf("unable to list echo pods: %w", err) + } + + for _, echoPod := range echoPods.Items { + echoOtherPodHostIPs = append(echoOtherPodHostIPs, echoPod.Status.HostIP) + } + } + + for _, client := range ct.clients.clients() { + patch := fmt.Sprintf(`{"spec":{"externalIPs":["%s"], "externalTrafficPolicy": "Local"}}`, strings.Join(echoOtherPodHostIPs, ",")) + + service, err := client.PatchService(ctx, ct.params.TestNamespace, echoOtherNodeDeploymentName, types.StrategicMergePatchType, []byte(patch), metav1.PatchOptions{}) + if err != nil { + return err + } + + fmt.Println("Patched service:", service.Spec.ExternalIPs) + + s := ct.echoServices[service.Name] + s.Service = service.DeepCopy() + ct.echoServices[service.Name] = s + + patchedServices = append(patchedServices, service) + } + + ensureFrontend := func() error { + for _, client := range ct.clients.clients() { + for _, ciliumPod := range ct.CiliumPods() { + for _, service := range patchedServices { + for _, ip := range service.Spec.ExternalIPs { + cmd := []string{"sh", "-c", + fmt.Sprintf("cilium bpf lb list --frontends | grep %s:%d", ip, service.Spec.Ports[0].Port)} + + fmt.Println(strings.Join(cmd, " ")) + if out, err := client.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd); err != nil { + fmt.Println(out.String()) + return err + } + } + } + } + } + + return nil + } + + w := utils.NewWaitObserver(ctx, utils.WaitParameters{Timeout: 10 * time.Second}) + defer w.Cancel() + for { + if err := ensureFrontend(); err != nil { + if err := w.Retry(err); err != nil { + return fmt.Errorf("Failed to ensure external IPs are exposed: %w", err) + } + + continue + } + break + } + + return nil +} + // Validate that srcPod can query the DNS server on dstPod successfully func (ct *ConnectivityTest) waitForPodDNS(ctx context.Context, srcPod, dstPod Pod) error { ct.Logf("⌛ [%s] Waiting for pod %s to reach DNS server on %s pod...", ct.client.ClusterName(), srcPod.Name(), dstPod.Name()) diff --git a/connectivity/check/peer.go b/connectivity/check/peer.go index 981d4dee78..6b6172b60f 100644 --- a/connectivity/check/peer.go +++ b/connectivity/check/peer.go @@ -199,6 +199,46 @@ func (s Service) Labels() map[string]string { return newMap } +func (s Service) ToExternalIPService() ExternalIPService { + return ExternalIPService{ + Service: s, + } +} + +// ExternalIPService TODO +// It implements interface TestPeer. +type ExternalIPService struct { + Service Service +} + +func (s ExternalIPService) Name() string { + return s.Service.Name() +} + +func (s ExternalIPService) Scheme() string { + return s.Service.Scheme() +} + +func (s ExternalIPService) Path() string { + return s.Service.Path() +} + +func (s ExternalIPService) Address(_ IPFamily) string { + return s.Service.Service.Spec.ExternalIPs[0] +} + +func (s ExternalIPService) Port() uint32 { + return uint32(s.Service.Service.Spec.Ports[0].NodePort) +} + +func (s ExternalIPService) HasLabel(name, value string) bool { + return s.Service.HasLabel(name, value) +} + +func (s ExternalIPService) Labels() map[string]string { + return s.Service.Labels() +} + // ExternalWorkload is an external workload acting as a peer in a // connectivity test. It implements interface TestPeer. type ExternalWorkload struct { diff --git a/connectivity/check/test.go b/connectivity/check/test.go index fc6056699f..77b8954a61 100644 --- a/connectivity/check/test.go +++ b/connectivity/check/test.go @@ -428,6 +428,22 @@ func (t *Test) WithCiliumEgressGatewayPolicy(policy string) *Test { // Set the egress gateway node pl[i].Spec.EgressGateway.NodeSelector.MatchLabels["kubernetes.io/hostname"] = egressGatewayNode + + // Set the excluded CIDR + /* + if len(pl[i].Spec.ExcludedCIDRs) != 0 { + excludedCIDRs := []v2.IPv4CIDR{} + for _, nodeWithoutCilium := range t.NodesWithoutCilium() { + fmt.Println("node without cilium:", nodeWithoutCilium) + for lol := range t.Context().Nodes() { + fmt.Println("nodes:", lol) + } + excludedCIDRs = append(excludedCIDRs, v2.IPv4CIDR(fmt.Sprintf("%s/32", t.Context().Nodes()[nodeWithoutCilium].Status.Addresses[0].Address))) + } + + pl[i].Spec.ExcludedCIDRs = excludedCIDRs + } + */ } if err := t.addCEGPs(pl...); err != nil { diff --git a/connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml b/connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml new file mode 100644 index 0000000000..4d1e2794d7 --- /dev/null +++ b/connectivity/manifests/egress-gateway-policy-excluded-cidrs.yaml @@ -0,0 +1,18 @@ +apiVersion: cilium.io/v2 +kind: CiliumEgressGatewayPolicy +metadata: + name: cegp-sample-excluded-cidrs +spec: + selectors: + - podSelector: + matchLabels: + io.kubernetes.pod.namespace: cilium-test + kind: client + destinationCIDRs: + - 0.0.0.0/0 + # excludedCIDRs: + # - NODE_WITHOUT_CILIUM_PLACEHOLDER/32 + egressGateway: + nodeSelector: + matchLabels: + kubernetes.io/hostname: NODE_NAME_PLACEHOLDER diff --git a/connectivity/manifests/egress-gateway-policy.yaml b/connectivity/manifests/egress-gateway-policy.yaml index 392856073e..6cc6f45a14 100644 --- a/connectivity/manifests/egress-gateway-policy.yaml +++ b/connectivity/manifests/egress-gateway-policy.yaml @@ -14,3 +14,19 @@ spec: nodeSelector: matchLabels: kubernetes.io/hostname: NODE_NAME_PLACEHOLDER +--- +apiVersion: cilium.io/v2 +kind: CiliumEgressGatewayPolicy +metadata: + name: cegp-sample-echo-service +spec: + selectors: + - podSelector: + matchLabels: + kind: echo + destinationCIDRs: + - 0.0.0.0/0 + egressGateway: + nodeSelector: + matchLabels: + kubernetes.io/hostname: NODE_NAME_PLACEHOLDER diff --git a/connectivity/suite.go b/connectivity/suite.go index 609eaa0782..c380acb9ef 100644 --- a/connectivity/suite.go +++ b/connectivity/suite.go @@ -14,6 +14,7 @@ import ( "github.com/cilium/cilium-cli/connectivity/check" "github.com/cilium/cilium-cli/connectivity/manifests/template" "github.com/cilium/cilium-cli/connectivity/tests" + "github.com/cilium/cilium/pkg/versioncheck" ) var ( @@ -157,6 +158,9 @@ var ( //go:embed manifests/egress-gateway-policy.yaml egressGatewayPolicyYAML string + + //go:embed manifests/egress-gateway-policy-excluded-cidrs.yaml + egressGatewayPolicyExcludedCIDRsYAML string ) var ( @@ -233,6 +237,16 @@ func Run(ctx context.Context, ct *check.ConnectivityTest, addExtraTests func(*ch tests.EgressGateway(), ) + if versioncheck.MustCompile(">=1.14.0")(ct.CiliumVersion) { + ct.NewTest("egress-gateway-excluded-cidrs"). + WithCiliumEgressGatewayPolicy(egressGatewayPolicyExcludedCIDRsYAML). + WithFeatureRequirements(check.RequireFeatureEnabled(check.FeatureEgressGateway), + check.RequireFeatureEnabled(check.FeatureNodeWithoutCilium)). + WithScenarios( + tests.EgressGateway(), + ) + } + return ct.Run(ctx) } diff --git a/connectivity/tests/egressgateway.go b/connectivity/tests/egressgateway.go index 43e8d3a588..7d7945e6db 100644 --- a/connectivity/tests/egressgateway.go +++ b/connectivity/tests/egressgateway.go @@ -14,6 +14,8 @@ import ( "github.com/cilium/cilium-cli/connectivity/check" "github.com/cilium/cilium-cli/defaults" "github.com/cilium/cilium-cli/internal/utils" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) // EgressGateway is a test case which, given the cegp-sample @@ -48,10 +50,43 @@ func (s *egressGateway) Run(ctx context.Context, t *check.Test) { s.waitForBpfPolicyEntries(ctx, t) + // Ping hosts (pod to host connectivity) i := 0 for _, client := range ct.ClientPods() { client := client + for _, dst := range ct.HostNetNSPodsByNode() { + dst := dst + + t.NewAction(s, fmt.Sprintf("ping-%d", i), &client, &dst, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.PingCommand(dst, check.IPFamilyV4)) + }) + i++ + } + } + + // DNS query (pod to service connectivity) + i = 0 + for _, client := range ct.ClientPods() { + client := client + + kubeDNSService, err := ct.K8sClient().GetService(ctx, "kube-system", "kube-dns", metav1.GetOptions{}) + if err != nil { + t.Fatal("Cannot get kube-dns service") + } + kubeDNSServicePeer := check.Service{Service: kubeDNSService} + + t.NewAction(s, fmt.Sprintf("dig-%d", i), &client, kubeDNSServicePeer, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.DigCommand(kubeDNSServicePeer, check.IPFamilyV4)) + }) + i++ + } + + // Traffic matching an egress gateway policy should leave the cluster masqueraded with the egress IP (pod to external service) + i = 0 + for _, client := range ct.ClientPods() { + client := client + for _, externalEcho := range ct.ExternalEchoPods() { t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, externalEcho, check.IPFamilyV4).Run(func(a *check.Action) { a.ExecInPod(ctx, ct.CurlClientIPCommand(externalEcho, check.IPFamilyV4)) @@ -64,6 +99,72 @@ func (s *egressGateway) Run(ctx context.Context, t *check.Test) { i++ } } + + // When connecting from outside the cluster to a nodeport service whose pods are selected by an egress policy, + // the reply traffic should not be SNATed with the egress IP + i = 0 + for _, client := range ct.ExternalEchoPods() { + client := client + + for _, echo := range ct.EchoServices() { + // convert the service to a ServiceExternalIP as we want to access it through its external IP + echo := echo.ToExternalIPService() + + t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, echo, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.CurlClientIPCommand(echo, check.IPFamilyV4)) + }) + i++ + } + } + + if status, ok := ct.Feature(check.FeatureTunnel); ok && !status.Enabled { + // When connecting from outside the cluster directly to a pod which is selected by an egress policy, the + // reply traffic should not be SNATed with the egress IP (only connections originating from these pods + // should go through egress gateway). + // + // This test is executed only when Cilium is running in direct routing mode, since we can simply add a + // route on the node outside the cluster to direct pod's traffic to the node where the pod is running + // (while in tunneling mode we would need the external node to send the traffic over the tunnel) + + for _, echoPod := range ct.EchoPods() { + targetPodHostIP := echoPod.Pod.Status.HostIP + targetPodIP := echoPod.Pod.Status.PodIP + + for _, externalNode := range ct.NodesWithoutCilium() { + for node, hostNetNSPod := range ct.HostNetNSPodsByNode() { + if node != externalNode { + continue + } + + cmd := []string{"ip", "route", "add", targetPodIP, "via", targetPodHostIP} + _, err := hostNetNSPod.K8sClient.ExecInPod(ctx, hostNetNSPod.Pod.Namespace, hostNetNSPod.Pod.Name, "", cmd) + if err != nil { + t.Fatalf("failed to add ip route: %w", err) + } + + defer func(hostNetNSPod check.Pod) { + cmd = []string{"ip", "route", "del", targetPodIP, "via", targetPodHostIP} + _, err = hostNetNSPod.K8sClient.ExecInPod(ctx, hostNetNSPod.Pod.Namespace, hostNetNSPod.Pod.Name, "", cmd) + if err != nil { + t.Fatalf("failed to delete ip route: %w", err) + } + }(hostNetNSPod) + } + } + } + + i = 0 + for _, client := range ct.ExternalEchoPods() { + client := client + + for _, echo := range ct.EchoPods() { + t.NewAction(s, fmt.Sprintf("curl-%d", i), &client, echo, check.IPFamilyV4).Run(func(a *check.Action) { + a.ExecInPod(ctx, ct.CurlClientIPCommand(echo, check.IPFamilyV4)) + }) + i++ + } + } + } } // getGatewayNodeInternalIP returns the k8s internal IP of the node acting as @@ -139,6 +240,16 @@ func (s *egressGateway) waitForBpfPolicyEntries(ctx context.Context, t *check.Te }) } + for _, echo := range ct.EchoPods() { + targetEntries = append(targetEntries, + bpfEgressGatewayPolicyEntry{ + SourceIP: echo.Pod.Status.PodIP, + DestCIDR: "0.0.0.0/0", + EgressIP: egressIP, + GatewayIP: gatewayNodeInternalIP.String(), + }) + } + cmd := strings.Split("cilium bpf egress list -o json", " ") stdout, err := ciliumPod.K8sClient.ExecInPod(ctx, ciliumPod.Pod.Namespace, ciliumPod.Pod.Name, defaults.AgentContainerName, cmd) if err != nil { diff --git a/k8s/client.go b/k8s/client.go index 71f97b59d4..40eee4bf02 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -253,6 +253,10 @@ func (c *Client) GetService(ctx context.Context, namespace, name string, opts me return c.Clientset.CoreV1().Services(namespace).Get(ctx, name, opts) } +func (c *Client) PatchService(ctx context.Context, namespace, name string, pt types.PatchType, data []byte, opts metav1.PatchOptions) (*corev1.Service, error) { + return c.Clientset.CoreV1().Services(namespace).Patch(ctx, name, pt, data, opts) +} + func (c *Client) CreateEndpoints(ctx context.Context, namespace string, ep *corev1.Endpoints, opts metav1.CreateOptions) (*corev1.Endpoints, error) { return c.Clientset.CoreV1().Endpoints(namespace).Create(ctx, ep, opts) }