bpf: Remove declare_tailcall_if

Remove `declare_tailcall_if`, so we always emit the tailcall programs into the ELF. The followup commit will implement pruning logic based on the actual usage of the tail calls. This means that we will only need the `invoke_tailcall_if` without the need to keep both the declaration and invocation in sync. Signed-off-by: Dylan Reimerink <dylan.reimerink@isovalent.com>
cilium · Feb 29, 2024 · 46db413 · 46db413
1 parent ce25c55
commit 46db413
Show file tree

Hide file tree

Showing 5 changed files with 46 additions and 102 deletions.
diff --git a/bpf/bpf_host.c b/bpf/bpf_host.c
@@ -414,13 +414,15 @@ tail_handle_ipv6_cont(struct __ctx_buff *ctx, bool from_host)
 	return ret;
 }
 
-declare_tailcall_if(is_defined(ENABLE_HOST_FIREWALL), CILIUM_CALL_IPV6_CONT_FROM_HOST)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_CONT_FROM_HOST)
+static __always_inline
 int tail_handle_ipv6_cont_from_host(struct __ctx_buff *ctx)
 {
 	return tail_handle_ipv6_cont(ctx, true);
 }
 
-declare_tailcall_if(is_defined(ENABLE_HOST_FIREWALL), CILIUM_CALL_IPV6_CONT_FROM_NETDEV)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_CONT_FROM_NETDEV)
+static __always_inline
 int tail_handle_ipv6_cont_from_netdev(struct __ctx_buff *ctx)
 {
 	return tail_handle_ipv6_cont(ctx, false);
@@ -868,13 +870,15 @@ tail_handle_ipv4_cont(struct __ctx_buff *ctx, bool from_host)
 	return ret;
 }
 
-declare_tailcall_if(is_defined(ENABLE_HOST_FIREWALL), CILIUM_CALL_IPV4_CONT_FROM_HOST)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_CONT_FROM_HOST)
+static __always_inline
 int tail_handle_ipv4_cont_from_host(struct __ctx_buff *ctx)
 {
 	return tail_handle_ipv4_cont(ctx, true);
 }
 
-declare_tailcall_if(is_defined(ENABLE_HOST_FIREWALL), CILIUM_CALL_IPV4_CONT_FROM_NETDEV)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_CONT_FROM_NETDEV)
+static __always_inline
 int tail_handle_ipv4_cont_from_netdev(struct __ctx_buff *ctx)
 {
 	return tail_handle_ipv4_cont(ctx, false);
@@ -1535,8 +1539,8 @@ int cil_to_host(struct __ctx_buff *ctx)
 
 #if defined(ENABLE_HOST_FIREWALL)
 #ifdef ENABLE_IPV6
-declare_tailcall_if(__or(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
-			 is_defined(DEBUG)), CILIUM_CALL_IPV6_TO_HOST_POLICY_ONLY)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_TO_HOST_POLICY_ONLY)
+static __always_inline
 int tail_ipv6_host_policy_ingress(struct __ctx_buff *ctx)
 {
 	struct trace_ctx __maybe_unused trace = {
@@ -1556,8 +1560,8 @@ int tail_ipv6_host_policy_ingress(struct __ctx_buff *ctx)
 #endif /* ENABLE_IPV6 */
 
 #ifdef ENABLE_IPV4
-declare_tailcall_if(__or(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
-			 is_defined(DEBUG)), CILIUM_CALL_IPV4_TO_HOST_POLICY_ONLY)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_TO_HOST_POLICY_ONLY)
+static __always_inline
 int tail_ipv4_host_policy_ingress(struct __ctx_buff *ctx)
 {
 	struct trace_ctx __maybe_unused trace = {

diff --git a/bpf/bpf_lxc.c b/bpf/bpf_lxc.c
@@ -252,7 +252,8 @@ static __always_inline int drop_for_direction(struct __ctx_buff *ctx,
 #endif /* ENABLE_IPV4 || ENABLE_IPV6 */
 
 #define TAIL_CT_LOOKUP4(ID, NAME, DIR, CONDITION, TARGET_ID, TARGET_NAME)	\
-declare_tailcall_if(CONDITION, ID)						\
+__section_tail(CILIUM_MAP_CALLS, ID)						\
+static __always_inline								\
 int NAME(struct __ctx_buff *ctx)						\
 {										\
 	struct ct_buffer4 ct_buffer = {};					\
@@ -297,7 +298,8 @@ int NAME(struct __ctx_buff *ctx)						\
 }
 
 #define TAIL_CT_LOOKUP6(ID, NAME, DIR, CONDITION, TARGET_ID, TARGET_NAME)	\
-declare_tailcall_if(CONDITION, ID)						\
+__section_tail(CILIUM_MAP_CALLS, ID)						\
+static __always_inline								\
 int NAME(struct __ctx_buff *ctx)						\
 {										\
 	struct ct_buffer6 ct_buffer = {};					\
@@ -734,7 +736,8 @@ static __always_inline int handle_ipv6_from_lxc(struct __ctx_buff *ctx, __u32 *d
 	return CTX_ACT_OK;
 }
 
-declare_tailcall_if(is_defined(ENABLE_PER_PACKET_LB), CILIUM_CALL_IPV6_FROM_LXC_CONT)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_FROM_LXC_CONT)
+static __always_inline
 int tail_handle_ipv6_cont(struct __ctx_buff *ctx)
 {
 	__u32 dst_sec_identity = 0;
@@ -1311,7 +1314,8 @@ static __always_inline int handle_ipv4_from_lxc(struct __ctx_buff *ctx, __u32 *d
 	return CTX_ACT_OK;
 }
 
-declare_tailcall_if(is_defined(ENABLE_PER_PACKET_LB), CILIUM_CALL_IPV4_FROM_LXC_CONT)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_FROM_LXC_CONT)
+static __always_inline
 int tail_handle_ipv4_cont(struct __ctx_buff *ctx)
 {
 	__u32 dst_sec_identity = 0;
@@ -1640,8 +1644,8 @@ ipv6_policy(struct __ctx_buff *ctx, struct ipv6hdr *ip6, int ifindex, __u32 src_
 	return CTX_ACT_OK;
 }
 
-declare_tailcall_if(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
-		    CILIUM_CALL_IPV6_TO_LXC_POLICY_ONLY)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_TO_LXC_POLICY_ONLY)
+static __always_inline
 int tail_ipv6_policy(struct __ctx_buff *ctx)
 {
 	struct ipv6_ct_tuple tuple = {};
@@ -2000,8 +2004,8 @@ ipv4_policy(struct __ctx_buff *ctx, struct iphdr *ip4, int ifindex, __u32 src_la
 	return CTX_ACT_OK;
 }
 
-declare_tailcall_if(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
-		    CILIUM_CALL_IPV4_TO_LXC_POLICY_ONLY)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_TO_LXC_POLICY_ONLY)
+static __always_inline
 int tail_ipv4_policy(struct __ctx_buff *ctx)
 {
 	struct ipv4_ct_tuple tuple = {};

diff --git a/bpf/lib/nodeport.h b/bpf/lib/nodeport.h
@@ -1023,11 +1023,8 @@ nodeport_rev_dnat_ingress_ipv6(struct __ctx_buff *ctx, struct trace_ctx *trace,
 	return fib_redirect(ctx, true, &fib_params, allow_neigh_map, ext_err, &ifindex);
 }
 
-declare_tailcall_if(__or3(__not(is_defined(HAVE_LARGE_INSN_LIMIT)),
-			  __and(is_defined(ENABLE_HOST_FIREWALL),
-				is_defined(IS_BPF_HOST)),
-			  is_defined(IS_BPF_LXC)),
-		    CILIUM_CALL_IPV6_NODEPORT_REVNAT)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_NODEPORT_REVNAT)
+static __always_inline
 int tail_nodeport_rev_dnat_ingress_ipv6(struct __ctx_buff *ctx)
 {
 	struct trace_ctx trace = {
@@ -1061,7 +1058,8 @@ int tail_nodeport_rev_dnat_ingress_ipv6(struct __ctx_buff *ctx)
 	return send_drop_notify_error_ext(ctx, 0, ret, ext_err, CTX_ACT_DROP, METRIC_EGRESS);
 }
 
-declare_tailcall_if(__not(is_defined(IS_BPF_LXC)), CILIUM_CALL_IPV6_NODEPORT_NAT_INGRESS)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_NODEPORT_NAT_INGRESS)
+static __always_inline
 int tail_nodeport_nat_ingress_ipv6(struct __ctx_buff *ctx)
 {
 	struct ipv6_nat_target target = {
@@ -1135,7 +1133,8 @@ int tail_nodeport_nat_ingress_ipv6(struct __ctx_buff *ctx)
 					  METRIC_INGRESS);
 }
 
-declare_tailcall_if(__not(is_defined(IS_BPF_LXC)), CILIUM_CALL_IPV6_NODEPORT_NAT_EGRESS)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_NODEPORT_NAT_EGRESS)
+static __always_inline
 int tail_nodeport_nat_egress_ipv6(struct __ctx_buff *ctx)
 {
 	const bool nat_46x64 = nat46x64_cb_xlate(ctx);
@@ -1579,11 +1578,8 @@ handle_nat_fwd_ipv6(struct __ctx_buff *ctx, struct trace_ctx *trace,
 	return __handle_nat_fwd_ipv6(ctx, trace, ext_err);
 }
 
-declare_tailcall_if(__or(__and(is_defined(ENABLE_IPV4),
-			       is_defined(ENABLE_IPV6)),
-			 __and(is_defined(ENABLE_HOST_FIREWALL),
-			       is_defined(IS_BPF_HOST))),
-		    CILIUM_CALL_IPV6_NODEPORT_NAT_FWD)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_NODEPORT_NAT_FWD)
+static __always_inline
 int tail_handle_nat_fwd_ipv6(struct __ctx_buff *ctx)
 {
 	struct trace_ctx trace = {
@@ -2497,11 +2493,8 @@ nodeport_rev_dnat_ingress_ipv4(struct __ctx_buff *ctx, struct trace_ctx *trace,
 	return fib_redirect(ctx, true, &fib_params, allow_neigh_map, ext_err, &ifindex);
 }
 
-declare_tailcall_if(__or3(__not(is_defined(HAVE_LARGE_INSN_LIMIT)),
-			  __and(is_defined(ENABLE_HOST_FIREWALL),
-				is_defined(IS_BPF_HOST)),
-			  is_defined(IS_BPF_LXC)),
-		    CILIUM_CALL_IPV4_NODEPORT_REVNAT)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_NODEPORT_REVNAT)
+static __always_inline
 int tail_nodeport_rev_dnat_ingress_ipv4(struct __ctx_buff *ctx)
 {
 	struct trace_ctx trace = {
@@ -2540,7 +2533,8 @@ int tail_nodeport_rev_dnat_ingress_ipv4(struct __ctx_buff *ctx)
 					  CTX_ACT_DROP, METRIC_EGRESS);
 }
 
-declare_tailcall_if(__not(is_defined(IS_BPF_LXC)), CILIUM_CALL_IPV4_NODEPORT_NAT_INGRESS)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_NODEPORT_NAT_INGRESS)
+static __always_inline
 int tail_nodeport_nat_ingress_ipv4(struct __ctx_buff *ctx)
 {
 	struct ipv4_nat_target target = {
@@ -2630,7 +2624,8 @@ int tail_nodeport_nat_ingress_ipv4(struct __ctx_buff *ctx)
 					  METRIC_INGRESS);
 }
 
-declare_tailcall_if(__not(is_defined(IS_BPF_LXC)), CILIUM_CALL_IPV4_NODEPORT_NAT_EGRESS)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_NODEPORT_NAT_EGRESS)
+static __always_inline
 int tail_nodeport_nat_egress_ipv4(struct __ctx_buff *ctx)
 {
 	struct bpf_fib_lookup_padded fib_params = {
@@ -3168,15 +3163,8 @@ handle_nat_fwd_ipv4(struct __ctx_buff *ctx, struct trace_ctx *trace,
 	return __handle_nat_fwd_ipv4(ctx, cluster_id, trace, ext_err);
 }
 
-declare_tailcall_if(__or4(__and(is_defined(ENABLE_IPV4),
-				is_defined(ENABLE_IPV6)),
-			  __and(is_defined(ENABLE_HOST_FIREWALL),
-				is_defined(IS_BPF_HOST)),
-			  __and(is_defined(ENABLE_CLUSTER_AWARE_ADDRESSING),
-				is_defined(ENABLE_INTER_CLUSTER_SNAT)),
-			  __and(is_defined(ENABLE_EGRESS_GATEWAY_COMMON),
-				is_defined(IS_BPF_HOST))),
-		    CILIUM_CALL_IPV4_NODEPORT_NAT_FWD)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_NODEPORT_NAT_FWD)
+static __always_inline
 int tail_handle_nat_fwd_ipv4(struct __ctx_buff *ctx)
 {
 	struct trace_ctx trace = {

diff --git a/bpf/lib/tailcall.h b/bpf/lib/tailcall.h
@@ -35,71 +35,17 @@
 #define __not_1 0
 #define __not(x) __eval(__not_, x)
 
-/* declare_tailcall_if() and invoke_tailcall_if() is a pair
- * of helpers which based on COND either selects to emit a
- * tail call for the underlying function when true or emits
- * it as inlined when false. COND can be selected by one or
- * multiple compile time flags.
- *
- * Usage example:
- *
- * 1) Declaration:
- *
- * declare_tailcall_if(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
- *                     CILIUM_CALL_FOO)
- * int foo_fn(struct __ctx_buff *ctx)
- * {
- *    [...]
- * }
- *
- * 2) Call-site:
+/* invoke_tailcall_if() is a helper which based on COND either selects to emit
+ * a tail call for the underlying function when true or emits it as inlined
+ * when false. COND can be selected by one or multiple compile time flags.
  *
  * [...]
  * invoke_tailcall_if(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
  *                    CILIUM_CALL_FOO, foo_fn);
  * [...]
  *
- * 3) Compilation result:
- *
- * When compiled with -DENABLE_IPV4 and -DENABLE_IPV6 both
- * set, then above emits a tail call as follows:
- *
- * __attribute__((section("2" "/" "10"), used))
- * int foo_fn(struct __ctx_buff *ctx)
- * {
- *    [...]
- * }
- *
- * [...]
- * do { ep_tail_call(ctx, 10); ret = -140; } while (0);
- * [...]
- *
- * The fall-through side sets DROP_MISSED_TAIL_CALL as ret.
- *
- * When only one of them is set in the above example or none
- * of them, then the code emission looks like:
- *
- * static __inline __attribute__ ((__always_inline__))
- * int foo_fn(struct __ctx_buff *ctx)
- * {
- *    [...]
- * }
- *
- * [...]
- * return foo_fn(ctx);
- * [...]
- *
- * Selectors can be single is_defined(), or multiple ones
- * combined with __and() or __or() macros. COND must be
- * the same expression for declare_tailcall_if() and the
- * invoke_tailcall_if() part.
+ * The loader will only load tail calls if they are invoked at least once.
  */
-#define __declare_tailcall_if_0(NAME)         \
-	static __always_inline
-#define __declare_tailcall_if_1(NAME)         \
-	__section_tail(CILIUM_MAP_CALLS, NAME)
-#define declare_tailcall_if(COND, NAME)       \
-	__eval(__declare_tailcall_if_, COND)(NAME)
 
 #define __invoke_tailcall_if_0(NAME, FUNC, EXT_ERR)			\
 	FUNC(ctx)

diff --git a/bpf/tests/tc_nodeport_lb6_dsr_backend.c b/bpf/tests/tc_nodeport_lb6_dsr_backend.c
@@ -248,6 +248,7 @@ int nodeport_dsr_backend_check(struct __ctx_buff *ctx)
 	test_finish();
 }
 
+static __always_inline
 int build_reply(struct __ctx_buff *ctx)
 {
 	union v6addr backend_ip = BACKEND_IP;
@@ -276,6 +277,7 @@ int build_reply(struct __ctx_buff *ctx)
 	return 0;
 }
 
+static __always_inline
 int check_reply(const struct __ctx_buff *ctx)
 {
 	union v6addr frontend_ip = FRONTEND_IP;