bpf: add -nostdinc and a few more misc compilation options

bpf/Makefile

@@ -60,6 +60,8 @@ LB_OPTIONS = \
 	-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_IPSEC:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP \
 	-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_IPSEC:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_NODEPORT_ACCELERATION \
 	-DENABLE_IPV4:-DENABLE_IPV6:-DENABLE_IPSEC:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_NODEPORT_ACCELERATION:-DENABLE_SESSION_AFFINITY \
+	-DENABLE_IPV4:-DENABLE_IPSEC:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_NODEPORT_ACCELERATION:-DENABLE_SESSION_AFFINITY \
+	-DENABLE_IPV6:-DENABLE_IPSEC:-DLB_L3:-DLB_L4:-DENABLE_HOST_SERVICES_UDP:-DENABLE_HOST_SERVICES_TCP:-DENABLE_NODEPORT:-DENABLE_EXTERNAL_IP:-DENABLE_NODEPORT_ACCELERATION:-DENABLE_SESSION_AFFINITY \
 
 # These options are intended to max out the BPF program complexity. it is load
 # tested as well.
@@ -82,25 +84,16 @@ bpf_sock.o: bpf_sock.ll
 	@$(ECHO_CC)
 	$(QUIET) ${LLC} ${LLC_FLAGS} -filetype=obj -o $@ $(patsubst %.o,%.ll,$@)
 
-# The following option combinations are compile tested
-OVERLAY_OPTIONS = \
-	 -DENABLE_IPV4 \
-	 -DENABLE_IPV6
-
-# These options are intended to max out the BPF program complexity. it is load
-# tested as well.
-MAX_OVERLAY_OPTIONS = -DENABLE_IPV4 -DENABLE_IPV6 -DENABLE_IPSEC
-
 bpf_overlay.ll: bpf_overlay.c $(LIB)
 	$(QUIET) set -e; \
 	if [ $(BUILD_PERMUTATIONS) != "" ]; then \
-		$(foreach OPTS,$(OVERLAY_OPTIONS), \
+		$(foreach OPTS,$(LB_OPTIONS), \
 			$(ECHO_CC) " [$(subst :,$(space),$(OPTS))]"; \
 			${CLANG} $(subst :,$(space),$(OPTS)) ${CLANG_FLAGS} -c $< -o $@; \
 			${LLC} ${LLC_FLAGS} -o /dev/null $@; ) \
 	fi
 	@$(ECHO_CC)
-	$(QUIET) ${CLANG} ${MAX_OVERLAY_OPTIONS} ${CLANG_FLAGS} -c $< -o $@
+	$(QUIET) ${CLANG} ${MAX_LB_OPTIONS} ${CLANG_FLAGS} -c $< -o $@
 
 bpf_overlay.o: bpf_overlay.ll
 	@$(ECHO_CC)

bpf/Makefile.bpf

@@ -3,11 +3,14 @@
 
 FLAGS := -I$(ROOT_DIR)/bpf/include -I$(ROOT_DIR)/bpf -D__NR_CPUS__=$(shell nproc) -O2 -g
 
-CLANG_FLAGS := ${FLAGS} -target bpf -emit-llvm
+CLANG_FLAGS := ${FLAGS} -target bpf -std=gnu89 -nostdinc -emit-llvm
 # eBPF verifier enforces unaligned access checks where necessary, so don't
 # let clang complain too early.
-CLANG_FLAGS += -Wall -Wextra -Werror
-CLANG_FLAGS += -Wno-address-of-packed-member -Wno-unknown-warning-option -Wno-gnu-variable-sized-type-not-at-end
+CLANG_FLAGS += -Wall -Wextra -Werror -Wshadow
+CLANG_FLAGS += -Wno-address-of-packed-member
+CLANG_FLAGS += -Wno-unknown-warning-option
+CLANG_FLAGS += -Wno-gnu-variable-sized-type-not-at-end
+CLANG_FLAGS += -Wdeclaration-after-statement
 LLC_FLAGS   := -march=bpf -mcpu=probe -mattr=dwarfris
 
 # BUILD_PERMUTATIONS_DEP is a dummy file dependency that ensures all targets

bpf/bpf_lxc.c

@@ -884,13 +884,15 @@ ipv6_policy(struct __ctx_buff *ctx, int ifindex, __u32 src_label, __u8 *reason,
 		send_policy_verdict_notify(ctx, src_label, tuple.dport, tuple.nexthdr,
 						 POLICY_INGRESS, 1, verdict, policy_match_type);
 #ifdef ENABLE_DSR
+	{
 		bool dsr = false;
 
 		ret = handle_dsr_v6(ctx, &dsr);
 		if (ret != 0)
 			return ret;
 
 		ct_state_new.dsr = dsr;
+	}
 #endif /* ENABLE_DSR */
 
 		ct_state_new.src_sec_id = src_label;
@@ -1099,13 +1101,15 @@ ipv4_policy(struct __ctx_buff *ctx, int ifindex, __u32 src_label, __u8 *reason,
 		send_policy_verdict_notify(ctx, src_label, tuple.dport, tuple.nexthdr,
 						 POLICY_INGRESS, 0, verdict, policy_match_type);
 #ifdef ENABLE_DSR
+	{
 		bool dsr = false;
 
 		ret = handle_dsr_v4(ctx, &dsr);
 		if (ret != 0)
 			return ret;
 
 		ct_state_new.dsr = dsr;
+	}
 #endif /* ENABLE_DSR */
 
 		ct_state_new.src_sec_id = src_label;

bpf/bpf_overlay.c

@@ -36,7 +36,7 @@ static __always_inline int handle_ipv6(struct __ctx_buff *ctx,
 		return DROP_INVALID;
 #ifdef ENABLE_NODEPORT
 	if (!bpf_skip_nodeport(ctx)) {
-		int ret = nodeport_lb6(ctx, *identity);
+		ret = nodeport_lb6(ctx, *identity);
 		if (ret < 0)
 			return ret;
 	}
@@ -95,12 +95,14 @@ static __always_inline int handle_ipv6(struct __ctx_buff *ctx,
 
 	/* Lookup IPv6 address in list of local endpoints */
 	if ((ep = lookup_ip6_endpoint(ip6)) != NULL) {
+		__u8 nexthdr;
+
 		/* Let through packets to the node-ip so they are
 		 * processed by the local ip stack */
 		if (ep->flags & ENDPOINT_F_HOST)
 			goto to_host;
 
-		__u8 nexthdr = ip6->nexthdr;
+		nexthdr = ip6->nexthdr;
 		hdrlen = ipv6_hdrlen(ctx, l3_off, &nexthdr);
 		if (hdrlen < 0)
 			return hdrlen;
@@ -113,7 +115,6 @@ static __always_inline int handle_ipv6(struct __ctx_buff *ctx,
 	if (1) {
 		union macaddr host_mac = HOST_IFINDEX_MAC;
 		union macaddr router_mac = NODE_MAC;
-		int ret;
 
 		ret = ipv6_l3(ctx, ETH_HLEN, (__u8 *) &router_mac.addr, (__u8 *) &host_mac.addr, METRIC_INGRESS);
 		if (ret != CTX_ACT_OK)

bpf/bpf_sock.c

@@ -77,11 +77,17 @@ void ctx_set_port(struct bpf_sock_addr *ctx, __be16 dport)
 }
 
 static __always_inline __maybe_unused bool
-ctx_in_hostns(void *ctx __maybe_unused)
+ctx_in_hostns(void *ctx __maybe_unused, __u64 *cookie)
 {
 #ifdef BPF_HAVE_NETNS_COOKIE
-	return get_netns_cookie(ctx) == get_netns_cookie(NULL);
+	__u64 own_cookie = get_netns_cookie(ctx);
+
+	if (cookie)
+		*cookie = own_cookie;
+	return own_cookie == get_netns_cookie(NULL);
 #else
+	if (cookie)
+		*cookie = 0;
 	return true;
 #endif
 }
@@ -237,17 +243,17 @@ static __always_inline int __sock4_xlate(struct bpf_sock_addr *ctx,
 					 struct bpf_sock_addr *ctx_full,
 					 const bool udp_only)
 {
-	const bool in_hostns = ctx_in_hostns(ctx_full);
+	union lb4_affinity_client_id id;
+	const bool in_hostns = ctx_in_hostns(ctx_full, &id.client_cookie);
 	struct lb4_backend *backend;
 	struct lb4_service *svc;
 	struct lb4_key key = {
 		.address	= ctx->user_ip4,
 		.dport		= ctx_dst_port(ctx),
 	};
 	struct lb4_service *slave_svc;
-	__u32 backend_id = 0;
-	/* Indicates whether a backend was selected from session affinity */
 	bool backend_from_affinity = false;
+	__u32 backend_id = 0;
 
 	if (!udp_only && !sock_proto_enabled(ctx->protocol))
 		return -ENOTSUP;
@@ -276,56 +282,48 @@ static __always_inline int __sock4_xlate(struct bpf_sock_addr *ctx,
 	if (sock4_skip_xlate(svc, in_hostns, ctx->user_ip4))
 		return -EPERM;
 
-#if defined(ENABLE_SESSION_AFFINITY) && defined(BPF_HAVE_NETNS_COOKIE)
-	/* Session affinity id (a netns cookie) */
-	union lb4_affinity_client_id client_id = { .client_cookie = 0 };
 	if (svc->affinity) {
-		client_id.client_cookie = get_netns_cookie(ctx_full);
-		backend_id = lb4_affinity_backend_id(svc->rev_nat_index,
-						     svc->affinity_timeout,
-						     true, client_id);
+		backend_id = lb4_affinity_backend_id_by_netns(svc, &id);
 		backend_from_affinity = true;
 	}
-#endif
 
 	if (backend_id == 0) {
-reselect_backend: __maybe_unused
+reselect_backend:
 		backend_from_affinity = false;
+
 		key.slave = (sock_local_cookie(ctx_full) % svc->count) + 1;
 		slave_svc = __lb4_lookup_slave(&key);
 		if (!slave_svc) {
 			update_metrics(0, METRIC_EGRESS, REASON_LB_NO_SLAVE);
 			return -ENOENT;
 		}
+
 		backend_id = slave_svc->backend_id;
 	}
 
 	backend = __lb4_lookup_backend(backend_id);
 	if (!backend) {
-#if defined(ENABLE_SESSION_AFFINITY) && defined(BPF_HAVE_NETNS_COOKIE)
 		if (backend_from_affinity) {
 			/* Backend from the session affinity no longer exists,
 			 * thus select a new one. Also, remove the affinity,
-			 * so that if the svc doesn't have any backend,
-			 * a subsequent request to the svc doesn't hit
-			 * the reselection again. */
-			lb4_delete_affinity(svc->rev_nat_index, true, client_id);
+			 * so that if the svc doesn't have any backend, a
+			 * subsequent request to the svc doesn't hit the
+			 * reselection again.
+			 */
+			lb4_delete_affinity_by_netns(svc, &id);
 			goto reselect_backend;
 		}
-#endif
+
 		update_metrics(0, METRIC_EGRESS, REASON_LB_NO_BACKEND);
 		return -ENOENT;
 	}
 
-#if defined(ENABLE_SESSION_AFFINITY) && defined(BPF_HAVE_NETNS_COOKIE)
 	if (svc->affinity)
-		lb4_update_affinity(svc->rev_nat_index, true, client_id, backend_id);
-#endif
+		lb4_update_affinity_by_netns(svc, &id, backend_id);
 
 	/* revnat entry is not required for TCP protocol */
-	if (!udp_only && ctx->protocol == IPPROTO_TCP) {
+	if (!udp_only && ctx->protocol == IPPROTO_TCP)
 		goto update_dst;
-	}
 
 	if (sock4_update_revnat(ctx_full, backend, &key,
 				svc->rev_nat_index) < 0) {
@@ -357,7 +355,8 @@ static __always_inline int __sock4_post_bind(struct bpf_sock *ctx,
 		.dport		= ctx_src_port(ctx),
 	};
 
-	if (!sock_proto_enabled(ctx->protocol) || !ctx_in_hostns(ctx_full))
+	if (!sock_proto_enabled(ctx->protocol) ||
+	    !ctx_in_hostns(ctx_full, NULL))
 		return 0;
 
 	svc = lb4_lookup_service(&key);
@@ -602,7 +601,8 @@ int sock6_xlate_v4_in_v6(struct bpf_sock_addr *ctx __maybe_unused,
 }
 
 #if defined(ENABLE_NODEPORT) || defined(ENABLE_EXTERNAL_IP)
-static __always_inline int sock6_post_bind_v4_in_v6(struct bpf_sock *ctx)
+static __always_inline int
+sock6_post_bind_v4_in_v6(struct bpf_sock *ctx __maybe_unused)
 {
 #ifdef ENABLE_IPV4
 	struct bpf_sock fake_ctx;
@@ -629,7 +629,8 @@ static __always_inline int __sock6_post_bind(struct bpf_sock *ctx)
 		.dport		= ctx_src_port(ctx),
 	};
 
-	if (!sock_proto_enabled(ctx->protocol) || !ctx_in_hostns(ctx))
+	if (!sock_proto_enabled(ctx->protocol) ||
+	    !ctx_in_hostns(ctx, NULL))
 		return 0;
 
 	ctx_get_v6_src_address(ctx, &key.address);
@@ -662,7 +663,8 @@ static __always_inline int __sock6_xlate(struct bpf_sock_addr *ctx,
 					 const bool udp_only)
 {
 #ifdef ENABLE_IPV6
-	const bool in_hostns = ctx_in_hostns(ctx);
+	union lb6_affinity_client_id id;
+	const bool in_hostns = ctx_in_hostns(ctx, &id.client_cookie);
 	struct lb6_backend *backend;
 	struct lb6_service *svc;
 	struct lb6_key key = {
@@ -694,49 +696,41 @@ static __always_inline int __sock6_xlate(struct bpf_sock_addr *ctx,
 	if (sock6_skip_xlate(svc, in_hostns, &v6_orig))
 		return -EPERM;
 
-#if defined(ENABLE_SESSION_AFFINITY) && defined(BPF_HAVE_NETNS_COOKIE)
-	union lb6_affinity_client_id client_id = { .client_cookie = 0 };
 	if (svc->affinity) {
-		client_id.client_cookie = get_netns_cookie(ctx);
-		backend_id = lb6_affinity_backend_id(svc->rev_nat_index,
-						     svc->affinity_timeout,
-						     true, &client_id);
+		backend_id = lb6_affinity_backend_id_by_netns(svc, &id);
 		backend_from_affinity = true;
 	}
-#endif
 
 	if (backend_id == 0) {
-reselect_backend: __maybe_unused
+reselect_backend:
 		backend_from_affinity = false;
+
 		key.slave = (sock_local_cookie(ctx) % svc->count) + 1;
 		slave_svc = __lb6_lookup_slave(&key);
 		if (!slave_svc) {
 			update_metrics(0, METRIC_EGRESS, REASON_LB_NO_SLAVE);
 			return -ENOENT;
 		}
+
 		backend_id = slave_svc->backend_id;
 	}
 
 	backend = __lb6_lookup_backend(backend_id);
 	if (!backend) {
-#if defined(ENABLE_SESSION_AFFINITY) && defined(BPF_HAVE_NETNS_COOKIE)
 		if (backend_from_affinity) {
-			lb6_delete_affinity(svc->rev_nat_index, true, &client_id);
+			lb6_delete_affinity_by_netns(svc, &id);
 			goto reselect_backend;
 		}
-#endif
+
 		update_metrics(0, METRIC_EGRESS, REASON_LB_NO_BACKEND);
 		return -ENOENT;
 	}
 
-#if defined(ENABLE_SESSION_AFFINITY) && defined(BPF_HAVE_NETNS_COOKIE)
 	if (svc->affinity)
-		lb6_update_affinity(svc->rev_nat_index, true, &client_id, backend_id);
-#endif
+		lb6_update_affinity_by_netns(svc, &id, backend_id);
 
-	if (!udp_only && ctx->protocol == IPPROTO_TCP) {
+	if (!udp_only && ctx->protocol == IPPROTO_TCP)
 		goto update_dst;
-	}
 
 	if (sock6_update_revnat(ctx, backend, &key,
 			        svc->rev_nat_index) < 0) {

bpf/include/bpf/ctx/skb.h

@@ -49,6 +49,13 @@
 
 #define ctx_adjust_meta		({ -ENOTSUPP; })
 
+/* Avoid expensive calls into the kernel flow dissector if it's not an L4
+ * hash. We currently only use the hash for debugging. If needed later, we
+ * can map it to BPF_FUNC(get_hash_recalc) to get the L4 hash.
+ */
+#define get_hash(ctx)		ctx->hash
+#define get_hash_recalc(ctx)	get_hash(ctx)
+
 static __always_inline __maybe_unused int
 ctx_redirect(struct __sk_buff *ctx __maybe_unused, int ifindex, __u32 flags)
 {

bpf/include/bpf/ctx/xdp.h

@@ -100,7 +100,8 @@ xdp_store_bytes(const struct xdp_md *ctx, __u64 off, const void *from,
 
 #define ctx_adjust_meta			xdp_adjust_meta
 
-#define get_hash_recalc(ctx)		({ 0; })
+#define get_hash(ctx)			({ 0; })
+#define get_hash_recalc(ctx)		get_hash(ctx)
 
 /* Checksum pieces from Linux kernel. */
 static inline __sum16 csum_fold(__wsum csum)

bpf/include/bpf/helpers_skb.h

@@ -12,9 +12,6 @@
 
 /* Only used helpers in Cilium go below. */
 
-/* Packet misc meta data */
-static __u32 BPF_FUNC(get_hash_recalc, struct __sk_buff *skb);
-
 /* Packet redirection */
 static int BPF_FUNC(redirect, int ifindex, __u32 flags);
 

bpf/init.sh

@@ -268,11 +268,12 @@ function bpf_compile()
 	TYPE=$3
 	EXTRA_OPTS=$4
 
-	clang -O2 -target bpf -emit-llvm				\
-	      -Wall -Wextra -Werror					\
+	clang -O2 -target bpf -std=gnu89 -nostdinc -emit-llvm		\
+	      -Wall -Wextra -Werror -Wshadow				\
 	      -Wno-address-of-packed-member				\
 	      -Wno-unknown-warning-option				\
 	      -Wno-gnu-variable-sized-type-not-at-end			\
+	      -Wdeclaration-after-statement				\
 	      -I. -I$DIR -I$LIB -I$LIB/include				\
 	      -D__NR_CPUS__=$(nproc)					\
 	      -DENABLE_ARP_RESPONDER=1					\