[v1.15] Replaced declare_tailcall_if with logic in the loader

Documentation/contributing/testing/bpf.rst

@@ -189,7 +189,7 @@ This is an abbreviated example showing the key components:
         /* OMITTED setting up map state */
 
         /* Jump into the entrypoint */
-        tail_call_static(ctx, &entry_call_map, 0);
+        tail_call_static(ctx, entry_call_map, 0);
         /* Fail if we didn't jump */
         return TEST_ERROR;
     }

Makefile

@@ -817,6 +817,8 @@ endif
 	$(QUIET) contrib/scripts/rand-check.sh
 	@$(ECHO_CHECK) contrib/scripts/check-time.sh
 	$(QUIET) contrib/scripts/check-time.sh
+	@$(ECHO_CHECK) contrib/scripts/check-go-testdata.sh
+	$(QUIET) contrib/scripts/check-go-testdata.sh
 
 check-sources:
 	@$(ECHO_CHECK) pkg/datapath/loader/check-sources.sh

bpf/bpf_host.c

@@ -418,13 +418,15 @@ tail_handle_ipv6_cont(struct __ctx_buff *ctx, bool from_host)
 	return ret;
 }
 
-declare_tailcall_if(is_defined(ENABLE_HOST_FIREWALL), CILIUM_CALL_IPV6_CONT_FROM_HOST)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_CONT_FROM_HOST)
+static __always_inline
 int tail_handle_ipv6_cont_from_host(struct __ctx_buff *ctx)
 {
 	return tail_handle_ipv6_cont(ctx, true);
 }
 
-declare_tailcall_if(is_defined(ENABLE_HOST_FIREWALL), CILIUM_CALL_IPV6_CONT_FROM_NETDEV)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_CONT_FROM_NETDEV)
+static __always_inline
 int tail_handle_ipv6_cont_from_netdev(struct __ctx_buff *ctx)
 {
 	return tail_handle_ipv6_cont(ctx, false);
@@ -598,13 +600,13 @@ handle_ipv4(struct __ctx_buff *ctx, __u32 secctx __maybe_unused,
 			bool __maybe_unused is_dsr = false;
 
 			int ret = nodeport_lb4(ctx, ip4, ETH_HLEN, secctx, ext_err, &is_dsr);
-
+#ifdef ENABLE_IPV6
 			if (ret == NAT_46X64_RECIRC) {
 				ctx_store_meta(ctx, CB_SRC_LABEL, secctx);
 				ep_tail_call(ctx, CILIUM_CALL_IPV6_FROM_NETDEV);
 				return DROP_MISSED_TAIL_CALL;
 			}
-
+#endif
 			/* nodeport_lb4() returns with TC_ACT_REDIRECT for
 			 * traffic to L7 LB. Policy enforcement needs to take
 			 * place after L7 LB has processed the packet, so we
@@ -872,13 +874,15 @@ tail_handle_ipv4_cont(struct __ctx_buff *ctx, bool from_host)
 	return ret;
 }
 
-declare_tailcall_if(is_defined(ENABLE_HOST_FIREWALL), CILIUM_CALL_IPV4_CONT_FROM_HOST)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_CONT_FROM_HOST)
+static __always_inline
 int tail_handle_ipv4_cont_from_host(struct __ctx_buff *ctx)
 {
 	return tail_handle_ipv4_cont(ctx, true);
 }
 
-declare_tailcall_if(is_defined(ENABLE_HOST_FIREWALL), CILIUM_CALL_IPV4_CONT_FROM_NETDEV)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_CONT_FROM_NETDEV)
+static __always_inline
 int tail_handle_ipv4_cont_from_netdev(struct __ctx_buff *ctx)
 {
 	return tail_handle_ipv4_cont(ctx, false);
@@ -1547,8 +1551,8 @@ int cil_to_host(struct __ctx_buff *ctx)
 
 #if defined(ENABLE_HOST_FIREWALL)
 #ifdef ENABLE_IPV6
-declare_tailcall_if(__or(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
-			 is_defined(DEBUG)), CILIUM_CALL_IPV6_TO_HOST_POLICY_ONLY)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_TO_HOST_POLICY_ONLY)
+static __always_inline
 int tail_ipv6_host_policy_ingress(struct __ctx_buff *ctx)
 {
 	struct trace_ctx __maybe_unused trace = {
@@ -1568,8 +1572,8 @@ int tail_ipv6_host_policy_ingress(struct __ctx_buff *ctx)
 #endif /* ENABLE_IPV6 */
 
 #ifdef ENABLE_IPV4
-declare_tailcall_if(__or(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
-			 is_defined(DEBUG)), CILIUM_CALL_IPV4_TO_HOST_POLICY_ONLY)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_TO_HOST_POLICY_ONLY)
+static __always_inline
 int tail_ipv4_host_policy_ingress(struct __ctx_buff *ctx)
 {
 	struct trace_ctx __maybe_unused trace = {

bpf/bpf_lxc.c

@@ -253,7 +253,8 @@ static __always_inline int drop_for_direction(struct __ctx_buff *ctx,
 #endif /* ENABLE_IPV4 || ENABLE_IPV6 */
 
 #define TAIL_CT_LOOKUP4(ID, NAME, DIR, CONDITION, TARGET_ID, TARGET_NAME)	\
-declare_tailcall_if(CONDITION, ID)						\
+__section_tail(CILIUM_MAP_CALLS, ID)						\
+static __always_inline								\
 int NAME(struct __ctx_buff *ctx)						\
 {										\
 	struct ct_buffer4 ct_buffer = {};					\
@@ -295,7 +296,8 @@ int NAME(struct __ctx_buff *ctx)						\
 }
 
 #define TAIL_CT_LOOKUP6(ID, NAME, DIR, CONDITION, TARGET_ID, TARGET_NAME)	\
-declare_tailcall_if(CONDITION, ID)						\
+__section_tail(CILIUM_MAP_CALLS, ID)						\
+static __always_inline								\
 int NAME(struct __ctx_buff *ctx)						\
 {										\
 	struct ct_buffer6 ct_buffer = {};					\
@@ -592,7 +594,7 @@ static __always_inline int handle_ipv6_from_lxc(struct __ctx_buff *ctx, __u32 *d
 	 */
 	if (*dst_sec_identity == HOST_ID) {
 		ctx_store_meta(ctx, CB_FROM_HOST, 0);
-		tail_call_static(ctx, &POLICY_CALL_MAP, HOST_EP_ID);
+		tail_call_static(ctx, POLICY_CALL_MAP, HOST_EP_ID);
 		return DROP_HOST_NOT_READY;
 	}
 #endif /* ENABLE_HOST_FIREWALL && !ENABLE_ROUTING */
@@ -728,7 +730,8 @@ static __always_inline int handle_ipv6_from_lxc(struct __ctx_buff *ctx, __u32 *d
 	return CTX_ACT_OK;
 }
 
-declare_tailcall_if(is_defined(ENABLE_PER_PACKET_LB), CILIUM_CALL_IPV6_FROM_LXC_CONT)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_FROM_LXC_CONT)
+static __always_inline
 int tail_handle_ipv6_cont(struct __ctx_buff *ctx)
 {
 	__u32 dst_sec_identity = 0;
@@ -741,7 +744,7 @@ int tail_handle_ipv6_cont(struct __ctx_buff *ctx)
 
 #ifdef ENABLE_CUSTOM_CALLS
 	if (!encode_custom_prog_meta(ctx, ret, dst_sec_identity)) {
-		tail_call_static(ctx, &CUSTOM_CALLS_MAP,
+		tail_call_static(ctx, CUSTOM_CALLS_MAP,
 				 CUSTOM_CALLS_IDX_IPV6_EGRESS);
 		update_metrics(ctx_full_len(ctx), METRIC_EGRESS,
 			       REASON_MISSED_CUSTOM_CALL);
@@ -1066,7 +1069,7 @@ static __always_inline int handle_ipv4_from_lxc(struct __ctx_buff *ctx, __u32 *d
 	 */
 	if (*dst_sec_identity == HOST_ID) {
 		ctx_store_meta(ctx, CB_FROM_HOST, 0);
-		tail_call_static(ctx, &POLICY_CALL_MAP, HOST_EP_ID);
+		tail_call_static(ctx, POLICY_CALL_MAP, HOST_EP_ID);
 		return DROP_HOST_NOT_READY;
 	}
 #endif /* ENABLE_HOST_FIREWALL && !ENABLE_ROUTING */
@@ -1302,7 +1305,8 @@ static __always_inline int handle_ipv4_from_lxc(struct __ctx_buff *ctx, __u32 *d
 	return CTX_ACT_OK;
 }
 
-declare_tailcall_if(is_defined(ENABLE_PER_PACKET_LB), CILIUM_CALL_IPV4_FROM_LXC_CONT)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_FROM_LXC_CONT)
+static __always_inline
 int tail_handle_ipv4_cont(struct __ctx_buff *ctx)
 {
 	__u32 dst_sec_identity = 0;
@@ -1316,7 +1320,7 @@ int tail_handle_ipv4_cont(struct __ctx_buff *ctx)
 
 #ifdef ENABLE_CUSTOM_CALLS
 	if (!encode_custom_prog_meta(ctx, ret, dst_sec_identity)) {
-		tail_call_static(ctx, &CUSTOM_CALLS_MAP,
+		tail_call_static(ctx, CUSTOM_CALLS_MAP,
 				 CUSTOM_CALLS_IDX_IPV4_EGRESS);
 		update_metrics(ctx_full_len(ctx), METRIC_EGRESS,
 			       REASON_MISSED_CUSTOM_CALL);
@@ -1612,8 +1616,8 @@ ipv6_policy(struct __ctx_buff *ctx, struct ipv6hdr *ip6, int ifindex, __u32 src_
 	return CTX_ACT_OK;
 }
 
-declare_tailcall_if(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
-		    CILIUM_CALL_IPV6_TO_LXC_POLICY_ONLY)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV6_TO_LXC_POLICY_ONLY)
+static __always_inline
 int tail_ipv6_policy(struct __ctx_buff *ctx)
 {
 	struct ipv6_ct_tuple tuple = {};
@@ -1680,7 +1684,7 @@ int tail_ipv6_policy(struct __ctx_buff *ctx)
 	 * proxy).
 	 */
 	if (!proxy_redirect && !encode_custom_prog_meta(ctx, ret, src_label)) {
-		tail_call_static(ctx, &CUSTOM_CALLS_MAP,
+		tail_call_static(ctx, CUSTOM_CALLS_MAP,
 				 CUSTOM_CALLS_IDX_IPV6_INGRESS);
 		update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
 			       REASON_MISSED_CUSTOM_CALL);
@@ -1773,7 +1777,7 @@ int tail_ipv6_to_endpoint(struct __ctx_buff *ctx)
 	 */
 	if (!proxy_redirect &&
 	    !encode_custom_prog_meta(ctx, ret, src_sec_identity)) {
-		tail_call_static(ctx, &CUSTOM_CALLS_MAP,
+		tail_call_static(ctx, CUSTOM_CALLS_MAP,
 				 CUSTOM_CALLS_IDX_IPV6_INGRESS);
 		update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
 			       REASON_MISSED_CUSTOM_CALL);
@@ -1979,8 +1983,8 @@ ipv4_policy(struct __ctx_buff *ctx, struct iphdr *ip4, int ifindex, __u32 src_la
 	return CTX_ACT_OK;
 }
 
-declare_tailcall_if(__and(is_defined(ENABLE_IPV4), is_defined(ENABLE_IPV6)),
-		    CILIUM_CALL_IPV4_TO_LXC_POLICY_ONLY)
+__section_tail(CILIUM_MAP_CALLS, CILIUM_CALL_IPV4_TO_LXC_POLICY_ONLY)
+static __always_inline
 int tail_ipv4_policy(struct __ctx_buff *ctx)
 {
 	struct ipv4_ct_tuple tuple = {};
@@ -2051,7 +2055,7 @@ int tail_ipv4_policy(struct __ctx_buff *ctx)
 	 * proxy).
 	 */
 	if (!proxy_redirect && !encode_custom_prog_meta(ctx, ret, src_label)) {
-		tail_call_static(ctx, &CUSTOM_CALLS_MAP,
+		tail_call_static(ctx, CUSTOM_CALLS_MAP,
 				 CUSTOM_CALLS_IDX_IPV4_INGRESS);
 		update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
 			       REASON_MISSED_CUSTOM_CALL);
@@ -2231,7 +2235,7 @@ int tail_ipv4_to_endpoint(struct __ctx_buff *ctx)
 	 */
 	if (!proxy_redirect &&
 	    !encode_custom_prog_meta(ctx, ret, src_sec_identity)) {
-		tail_call_static(ctx, &CUSTOM_CALLS_MAP,
+		tail_call_static(ctx, CUSTOM_CALLS_MAP,
 				 CUSTOM_CALLS_IDX_IPV4_INGRESS);
 		update_metrics(ctx_full_len(ctx), METRIC_INGRESS,
 			       REASON_MISSED_CUSTOM_CALL);
@@ -2406,7 +2410,7 @@ int cil_to_container(struct __ctx_buff *ctx)
 	if (identity == HOST_ID) {
 		ctx_store_meta(ctx, CB_FROM_HOST, 1);
 		ctx_store_meta(ctx, CB_DST_ENDPOINT_ID, LXC_ID);
-		tail_call_static(ctx, &POLICY_CALL_MAP, HOST_EP_ID);
+		tail_call_static(ctx, POLICY_CALL_MAP, HOST_EP_ID);
 		return send_drop_notify(ctx, identity, sec_label, LXC_ID,
 					DROP_HOST_NOT_READY, CTX_ACT_DROP,
 					METRIC_INGRESS);

bpf/include/bpf/tailcall.h

@@ -7,29 +7,32 @@
 #include "compiler.h"
 
 #if defined(__bpf__)
-static __always_inline __maybe_unused void
-tail_call_static(const struct __ctx_buff *ctx, const void *map,
-		 const __u32 slot)
-{
-	if (!__builtin_constant_p(slot))
-		__throw_build_bug();
 
-	/* Don't gamble, but _guarantee_ that LLVM won't optimize setting
-	 * r2 and r3 from different paths ending up at the same call insn as
-	 * otherwise we won't be able to use the jmpq/nopl retpoline-free
-	 * patching by the x86-64 JIT in the kernel.
-	 *
-	 * Note on clobber list: we need to stay in-line with BPF calling
-	 * convention, so even if we don't end up using r0, r4, r5, we need
-	 * to mark them as clobber so that LLVM doesn't end up using them
-	 * before / after the call.
-	 */
-	asm volatile("r1 = %[ctx]\n\t"
-		     "r2 = %[map]\n\t"
-		     "r3 = %[slot]\n\t"
-		     "call 12\n\t"
-		     :: [ctx]"r"(ctx), [map]"r"(map), [slot]"i"(slot)
-		     : "r0", "r1", "r2", "r3", "r4", "r5");
+/* Don't gamble, but _guarantee_ that LLVM won't optimize setting
+ * r2 and r3 from different paths ending up at the same call insn as
+ * otherwise we won't be able to use the jmpq/nopl retpoline-free
+ * patching by the x86-64 JIT in the kernel.
+ *
+ * Note on clobber list: we need to stay in-line with BPF calling
+ * convention, so even if we don't end up using r0, r4, r5, we need
+ * to mark them as clobber so that LLVM doesn't end up using them
+ * before / after the call.
+ *
+ * WARNING: The loader relies on the exact instruction sequence
+ * emitted by this macro. Consult with the loader team before
+ * changing this macro.
+ */
+#define tail_call_static(ctx_ptr, map, slot)				\
+{								\
+	if (!__builtin_constant_p(slot))			\
+		__throw_build_bug();				\
+								\
+	asm volatile("r1 = %[ctx]\n\t"				\
+		"r2 = " __stringify(map) " ll\n\t"		\
+		"r3 = %[slot_idx]\n\t"				\
+		"call 12\n\t"					\
+		:: [ctx]"r"(ctx_ptr), [slot_idx]"i"(slot)	\
+		: "r0", "r1", "r2", "r3", "r4", "r5");		\
 }
 
 static __always_inline __maybe_unused void

bpf/lib/icmp6.h

@@ -276,7 +276,6 @@ int tail_icmp6_send_time_exceeded(struct __ctx_buff *ctx __maybe_unused)
 	return 0;
 # endif
 }
-#endif
 
 /*
  * icmp6_send_time_exceeded
@@ -297,6 +296,7 @@ static __always_inline int icmp6_send_time_exceeded(struct __ctx_buff *ctx,
 
 	return DROP_MISSED_TAIL_CALL;
 }
+#endif
 
 static __always_inline int __icmp6_handle_ns(struct __ctx_buff *ctx, int nh_off)
 {

bpf/lib/ids.h

@@ -4,6 +4,7 @@
 #ifndef __LIB_IDS_H_
 #define __LIB_IDS_H_
 
+/* WARNING: Do not change these ids without updating pkg/bpf/collection.go */
 #define CILIUM_MAP_POLICY	1
 #define CILIUM_MAP_CALLS	2
 #define CILIUM_MAP_CUSTOM_CALLS	3

bpf/lib/l3.h

@@ -28,15 +28,16 @@
 #ifdef ENABLE_IPV6
 static __always_inline int ipv6_l3(struct __ctx_buff *ctx, int l3_off,
 				   const __u8 *smac, const __u8 *dmac,
-				   __u8 direction)
+				   __u8 __maybe_unused direction)
 {
 	int ret;
 
 	ret = ipv6_dec_hoplimit(ctx, l3_off);
 	if (IS_ERR(ret)) {
+#ifndef SKIP_ICMPV6_HOPLIMIT_HANDLING
 		if (ret == DROP_TTL_EXCEEDED)
 			return icmp6_send_time_exceeded(ctx, l3_off, direction);
-
+#endif
 		return ret;
 	}
 

bpf/lib/maps.h

@@ -317,7 +317,7 @@ struct {
 static __always_inline void ep_tail_call(struct __ctx_buff *ctx __maybe_unused,
 					 const __u32 index __maybe_unused)
 {
-	tail_call_static(ctx, &CALLS_MAP, index);
+	tail_call_static(ctx, CALLS_MAP, index);
 }
 #endif /* SKIP_CALLS_MAP */
 #endif