bpf: test: add LB test for terminating backend

Once a LB connection has been established, we expect to continue using
its CT entry to obtain the backend. Even if the backend is in terminating
state, and the service has lost all of its backends.

Keeping this separate from the fix, in case we can't easily backport.

Signed-off-by: Julian Wiedmann <jwi@isovalent.com>

bpf/tests/lib/lb.h

@@ -3,7 +3,7 @@
 
 #ifdef ENABLE_IPV4
 static __always_inline void
-lb_v4_add_service(__be32 addr, __be16 port, __u16 backend_count, __u16 rev_nat_index)
+lb_v4_upsert_service(__be32 addr, __be16 port, __u16 backend_count, __u16 rev_nat_index)
 {
 	struct lb4_key svc_key = {
 		.address = addr,
@@ -19,6 +19,13 @@ lb_v4_add_service(__be32 addr, __be16 port, __u16 backend_count, __u16 rev_nat_i
 	/* Register with both scopes: */
 	svc_key.scope = LB_LOOKUP_SCOPE_INT;
 	map_update_elem(&LB4_SERVICES_MAP_V2, &svc_key, &svc_value, BPF_ANY);
+}
+
+static __always_inline void
+lb_v4_add_service(__be32 addr, __be16 port, __u16 backend_count, __u16 rev_nat_index)
+{
+	/* Register with both scopes: */
+	lb_v4_upsert_service(addr, port, backend_count, rev_nat_index);
 
 	/* Insert a reverse NAT entry for the above service */
 	struct lb4_reverse_nat revnat_value = {
@@ -29,19 +36,28 @@ lb_v4_add_service(__be32 addr, __be16 port, __u16 backend_count, __u16 rev_nat_i
 }
 
 static __always_inline void
-lb_v4_add_backend(__be32 svc_addr, __be16 svc_port, __u16 backend_slot,
-		  __u32 backend_id, __be32 backend_addr, __be16 backend_port,
-		  __u8 backend_proto, __u8 cluster_id)
+lb_v4_upsert_backend(__u32 backend_id, __be32 backend_addr, __be16 backend_port,
+		     __u8 backend_proto, __u8 flags, __u8 cluster_id)
 {
 	struct lb4_backend backend = {
 		.address = backend_addr,
 		.port = backend_port,
 		.proto = backend_proto,
-		.flags = BE_STATE_ACTIVE,
+		.flags = flags,
 		.cluster_id = cluster_id,
 	};
-	/* Create the actual backend: */
+
 	map_update_elem(&LB4_BACKEND_MAP, &backend_id, &backend, BPF_ANY);
+}
+
+static __always_inline void
+lb_v4_add_backend(__be32 svc_addr, __be16 svc_port, __u16 backend_slot,
+		  __u32 backend_id, __be32 backend_addr, __be16 backend_port,
+		  __u8 backend_proto, __u8 cluster_id)
+{
+	/* Create the actual backend: */
+	lb_v4_upsert_backend(backend_id, backend_addr, backend_port,
+			     backend_proto, BE_STATE_ACTIVE, cluster_id);
 
 	struct lb4_key svc_key = {
 		.address = svc_addr,

bpf/tests/tc_nodeport_lb_terminating_backend.c

@@ -0,0 +1,295 @@
+// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
+/* Copyright Authors of Cilium */
+
+/* Set ETH_HLEN to 14 to indicate that the packet has a 14 byte ethernet header */
+#define ETH_HLEN 14
+
+/* Enable code paths under test */
+#define ENABLE_IPV4		1
+#define ENABLE_NODEPORT		1
+#define ENABLE_HOST_ROUTING	1
+
+#define DISABLE_LOOPBACK_LB	1
+
+/* Skip ingress policy checks, not needed to validate hairpin flow */
+#define USE_BPF_PROG_FOR_INGRESS_POLICY
+#undef FORCE_LOCAL_POLICY_EVAL_AT_SOURCE
+
+#define CLIENT_IP		v4_ext_one
+#define CLIENT_PORT		__bpf_htons(111)
+
+#define FRONTEND_IP_LOCAL	v4_svc_one
+#define FRONTEND_PORT		tcp_svc_one
+
+#define LB_IP			v4_node_one
+#define IPV4_DIRECT_ROUTING	LB_IP
+
+#define BACKEND_IP_LOCAL	v4_pod_one
+#define BACKEND_PORT		__bpf_htons(8080)
+
+#define NATIVE_DEV_IFINDEX	24
+#define DEFAULT_IFACE		NATIVE_DEV_IFINDEX
+#define BACKEND_IFACE		25
+
+#define SVC_REV_NAT_ID		2
+
+#include "common.h"
+
+#include <bpf/ctx/skb.h>
+#include "pktgen.h"
+
+static volatile const __u8 *client_mac = mac_one;
+/* this matches the default node_config.h: */
+static volatile const __u8 lb_mac[ETH_ALEN]	= { 0xce, 0x72, 0xa7, 0x03, 0x88, 0x56 };
+static volatile const __u8 *node_mac = mac_three;
+static volatile const __u8 *local_backend_mac = mac_four;
+
+#define ctx_redirect mock_ctx_redirect
+
+static __always_inline __maybe_unused int
+mock_ctx_redirect(const struct __sk_buff *ctx __maybe_unused,
+		  int ifindex __maybe_unused, __u32 flags __maybe_unused)
+{
+	void *data = (void *)(long)ctx_data(ctx);
+	void *data_end = (void *)(long)ctx->data_end;
+	struct iphdr *ip4;
+
+	ip4 = data + sizeof(struct ethhdr);
+	if ((void *)ip4 + sizeof(*ip4) > data_end)
+		return CTX_ACT_DROP;
+
+	/* Forward to backend: */
+	if (ip4->saddr == CLIENT_IP && ifindex == BACKEND_IFACE)
+		return CTX_ACT_REDIRECT;
+
+	return CTX_ACT_DROP;
+}
+
+#define SECCTX_FROM_IPCACHE 1
+
+#include "config_replacement.h"
+
+#include "bpf_host.c"
+
+#include "lib/endpoint.h"
+#include "lib/ipcache.h"
+#include "lib/lb.h"
+
+#define FROM_NETDEV	0
+
+struct {
+	__uint(type, BPF_MAP_TYPE_PROG_ARRAY);
+	__uint(key_size, sizeof(__u32));
+	__uint(max_entries, 2);
+	__array(values, int());
+} entry_call_map __section(".maps") = {
+	.values = {
+		[FROM_NETDEV] = &cil_from_netdev,
+	},
+};
+
+/* Test that a SVC request (UDP) to a local backend
+ * - gets DNATed (but not SNATed)
+ * - gets redirected by TC (as ENABLE_HOST_ROUTING is set)
+ */
+PKTGEN("tc", "tc_nodeport_lb_terminating_backend_0")
+int tc_nodeport_lb_terminating_backend_0_pktgen(struct __ctx_buff *ctx)
+{
+	struct pktgen builder;
+	struct udphdr *l4;
+	void *data;
+
+	/* Init packet builder */
+	pktgen__init(&builder, ctx);
+
+	l4 = pktgen__push_ipv4_udp_packet(&builder,
+					  (__u8 *)client_mac, (__u8 *)lb_mac,
+					  CLIENT_IP, FRONTEND_IP_LOCAL,
+					  CLIENT_PORT, FRONTEND_PORT);
+	if (!l4)
+		return TEST_ERROR;
+
+	data = pktgen__push_data(&builder, default_data, sizeof(default_data));
+	if (!data)
+		return TEST_ERROR;
+
+	/* Calc lengths, set protocol fields and calc checksums */
+	pktgen__finish(&builder);
+
+	return 0;
+}
+
+SETUP("tc", "tc_nodeport_lb_terminating_backend_0")
+int tc_nodeport_lb_terminating_backend_0_setup(struct __ctx_buff *ctx)
+{
+	__u16 revnat_id = SVC_REV_NAT_ID;
+
+	lb_v4_add_service(FRONTEND_IP_LOCAL, FRONTEND_PORT, 1, revnat_id);
+	lb_v4_add_backend(FRONTEND_IP_LOCAL, FRONTEND_PORT, 1, 125,
+			  BACKEND_IP_LOCAL, BACKEND_PORT, IPPROTO_UDP, 0);
+
+	/* add local backend */
+	endpoint_v4_add_entry(BACKEND_IP_LOCAL, BACKEND_IFACE, 0, 0,
+			      (__u8 *)local_backend_mac, (__u8 *)node_mac);
+
+	ipcache_v4_add_entry(BACKEND_IP_LOCAL, 0, 112233, 0, 0);
+
+	/* Jump into the entrypoint */
+	tail_call_static(ctx, entry_call_map, FROM_NETDEV);
+	/* Fail if we didn't jump */
+	return TEST_ERROR;
+}
+
+CHECK("tc", "tc_nodeport_lb_terminating_backend_0")
+int tc_nodeport_lb_terminating_backend_0_check(const struct __ctx_buff *ctx)
+{
+	void *data, *data_end;
+	__u32 *status_code;
+	struct udphdr *l4;
+	struct ethhdr *l2;
+	struct iphdr *l3;
+
+	test_init();
+
+	data = (void *)(long)ctx_data(ctx);
+	data_end = (void *)(long)ctx->data_end;
+
+	if (data + sizeof(__u32) > data_end)
+		test_fatal("status code out of bounds");
+
+	status_code = data;
+
+	assert(*status_code == CTX_ACT_REDIRECT);
+
+	l2 = data + sizeof(__u32);
+	if ((void *)l2 + sizeof(struct ethhdr) > data_end)
+		test_fatal("l2 out of bounds");
+
+	l3 = (void *)l2 + sizeof(struct ethhdr);
+	if ((void *)l3 + sizeof(struct iphdr) > data_end)
+		test_fatal("l3 out of bounds");
+
+	l4 = (void *)l3 + sizeof(struct iphdr);
+	if ((void *)l4 + sizeof(*l4) > data_end)
+		test_fatal("l4 out of bounds");
+
+	if (memcmp(l2->h_source, (__u8 *)node_mac, ETH_ALEN) != 0)
+		test_fatal("src MAC is not the node MAC")
+	if (memcmp(l2->h_dest, (__u8 *)local_backend_mac, ETH_ALEN) != 0)
+		test_fatal("dst MAC is not the endpoint MAC")
+
+	if (l3->saddr != CLIENT_IP)
+		test_fatal("src IP has changed");
+
+	if (l3->daddr != BACKEND_IP_LOCAL)
+		test_fatal("dst IP hasn't been NATed to local backend IP");
+
+	if (l4->source != CLIENT_PORT)
+		test_fatal("src port has changed");
+
+	if (l4->dest != BACKEND_PORT)
+		test_fatal("dst port hasn't been NATed to backend port");
+
+	test_finish();
+}
+
+/* Test that a second request gets LBed to a terminating backend,
+ * even when the service has no active backends remaining.
+ */
+PKTGEN("tc", "tc_nodeport_lb_terminating_backend_1")
+int tc_nodeport_lb_terminating_backend_1_pktgen(struct __ctx_buff *ctx)
+{
+	struct pktgen builder;
+	struct udphdr *l4;
+	void *data;
+
+	/* Init packet builder */
+	pktgen__init(&builder, ctx);
+
+	l4 = pktgen__push_ipv4_udp_packet(&builder,
+					  (__u8 *)client_mac, (__u8 *)lb_mac,
+					  CLIENT_IP, FRONTEND_IP_LOCAL,
+					  CLIENT_PORT, FRONTEND_PORT);
+	if (!l4)
+		return TEST_ERROR;
+
+	data = pktgen__push_data(&builder, default_data, sizeof(default_data));
+	if (!data)
+		return TEST_ERROR;
+
+	/* Calc lengths, set protocol fields and calc checksums */
+	pktgen__finish(&builder);
+
+	return 0;
+}
+
+SETUP("tc", "tc_nodeport_lb_terminating_backend_1")
+int tc_nodeport_lb_terminating_backend_1_setup(struct __ctx_buff *ctx)
+{
+	__u16 revnat_id = SVC_REV_NAT_ID;
+
+	/* Remove the service's last backend, and flip the backend to
+	 * 'terminating' state.
+	 */
+	lb_v4_upsert_service(FRONTEND_IP_LOCAL, FRONTEND_PORT, 0, revnat_id);
+	lb_v4_upsert_backend(125, BACKEND_IP_LOCAL, BACKEND_PORT, IPPROTO_UDP,
+			     BE_STATE_TERMINATING, 0);
+
+	/* Jump into the entrypoint */
+	tail_call_static(ctx, entry_call_map, FROM_NETDEV);
+	/* Fail if we didn't jump */
+	return TEST_ERROR;
+}
+
+CHECK("tc", "tc_nodeport_lb_terminating_backend_1")
+int tc_nodeport_lb_terminating_backend_1_check(const struct __ctx_buff *ctx)
+{
+	void *data, *data_end;
+	__u32 *status_code;
+	struct udphdr *l4;
+	struct ethhdr *l2;
+	struct iphdr *l3;
+
+	test_init();
+
+	data = (void *)(long)ctx_data(ctx);
+	data_end = (void *)(long)ctx->data_end;
+
+	if (data + sizeof(__u32) > data_end)
+		test_fatal("status code out of bounds");
+
+	status_code = data;
+
+	assert(*status_code == CTX_ACT_REDIRECT);
+
+	l2 = data + sizeof(__u32);
+	if ((void *)l2 + sizeof(struct ethhdr) > data_end)
+		test_fatal("l2 out of bounds");
+
+	l3 = (void *)l2 + sizeof(struct ethhdr);
+	if ((void *)l3 + sizeof(struct iphdr) > data_end)
+		test_fatal("l3 out of bounds");
+
+	l4 = (void *)l3 + sizeof(struct iphdr);
+	if ((void *)l4 + sizeof(*l4) > data_end)
+		test_fatal("l4 out of bounds");
+
+	if (memcmp(l2->h_source, (__u8 *)node_mac, ETH_ALEN) != 0)
+		test_fatal("src MAC is not the node MAC")
+	if (memcmp(l2->h_dest, (__u8 *)local_backend_mac, ETH_ALEN) != 0)
+		test_fatal("dst MAC is not the endpoint MAC")
+
+	if (l3->saddr != CLIENT_IP)
+		test_fatal("src IP has changed");
+
+	if (l3->daddr != BACKEND_IP_LOCAL)
+		test_fatal("dst IP hasn't been NATed to local backend IP");
+
+	if (l4->source != CLIENT_PORT)
+		test_fatal("src port has changed");
+
+	if (l4->dest != BACKEND_PORT)
+		test_fatal("dst port hasn't been NATed to backend port");
+
+	test_finish();
+}