bpf: use per-cpu scratch space from xdp context to store meta data

bpf/bpf_xdp.c

@@ -71,18 +71,20 @@ struct bpf_elf_map __section_maps CIDR6_LMAP_NAME = {
 static __always_inline __maybe_unused int
 bpf_xdp_exit(struct __ctx_buff *ctx, const int verdict)
 {
-	/* Undo meta data, so GRO can perform natural aggregation. */
 	if (verdict == CTX_ACT_OK) {
 		__u32 meta_xfer = ctx_load_meta(ctx, XFER_MARKER);
 
 		/* We transfer data from XFER_MARKER. This specifically
 		 * does not break packet trains in GRO.
 		 */
 		if (meta_xfer) {
-			ctx_adjust_meta(ctx, META_PIVOT - sizeof(__u32));
-			ctx_store_meta(ctx, 0, meta_xfer);
-		} else {
-			ctx_adjust_meta(ctx, META_PIVOT);
+			if (!ctx_adjust_meta(ctx, -(int)sizeof(meta_xfer))) {
+				__u32 *data_meta = ctx_data_meta(ctx);
+				__u32 *data = ctx_data(ctx);
+
+				if (!ctx_no_room(data_meta + 1, data))
+					data_meta[0] = meta_xfer;
+			}
 		}
 	}
 
@@ -224,8 +226,6 @@ static __always_inline int check_filters(struct __ctx_buff *ctx)
 
 	if (!validate_ethertype(ctx, &proto))
 		return CTX_ACT_OK;
-	if (ctx_adjust_meta(ctx, -META_PIVOT))
-		return CTX_ACT_OK;
 
 	ctx_store_meta(ctx, XFER_MARKER, 0);
 	bpf_skip_nodeport_clear(ctx);

bpf/include/bpf/api.h

@@ -16,23 +16,6 @@
 #include "verifier.h"
 #include "tailcall.h"
 #include "errno.h"
-
-#define PIN_NONE		0
-#define PIN_OBJECT_NS		1
-#define PIN_GLOBAL_NS		2
-
-struct bpf_elf_map {
-	__u32 type;
-	__u32 size_key;
-	__u32 size_value;
-	__u32 max_elem;
-	__u32 flags;
-	__u32 id;
-	__u32 pinning;
-#ifdef SOCKMAP
-	__u32 inner_id;
-	__u32 inner_idx;
-#endif
-};
+#include "loader.h"
 
 #endif /* __BPF_API__ */

bpf/include/bpf/compiler.h

@@ -44,6 +44,10 @@
 # define unlikely(X)		__builtin_expect(!!(X), 0)
 #endif
 
+#ifndef always_succeeds		/* Mainly for documentation purpose. */
+# define always_succeeds(X)	likely(X)
+#endif
+
 #undef __always_inline		/* stddef.h defines its own */
 #define __always_inline		inline __attribute__((always_inline))
 

bpf/include/bpf/ctx/xdp.h

@@ -12,6 +12,8 @@
 #include "common.h"
 #include "../helpers_xdp.h"
 #include "../builtins.h"
+#include "../section.h"
+#include "../loader.h"
 
 #define CTX_ACT_OK			XDP_PASS
 #define CTX_ACT_DROP			XDP_DROP
@@ -270,31 +272,33 @@ ctx_full_len(const struct xdp_md *ctx)
 	return ctx_data_end(ctx) - ctx_data(ctx);
 }
 
+struct bpf_elf_map __section_maps cilium_xdp_scratch = {
+	.type		= BPF_MAP_TYPE_PERCPU_ARRAY,
+	.size_key	= sizeof(int),
+	.size_value	= META_PIVOT,
+	.pinning	= PIN_GLOBAL_NS,
+	.max_elem	= 1,
+};
+
 static __always_inline __maybe_unused void
-ctx_store_meta(struct xdp_md *ctx, const __u64 off, __u32 datum)
+ctx_store_meta(struct xdp_md *ctx __maybe_unused, const __u64 off, __u32 datum)
 {
-	__u32 *data_meta = ctx_data_meta(ctx);
-	void *data = ctx_data(ctx);
+	__u32 zero = 0, *data_meta = map_lookup_elem(&cilium_xdp_scratch, &zero);
 
-	if (!ctx_no_room(data_meta + off + 1, data)) {
+	if (always_succeeds(data_meta))
 		data_meta[off] = datum;
-	} else {
-		build_bug_on((off + 1) * sizeof(__u32) > META_PIVOT);
-	}
+	build_bug_on((off + 1) * sizeof(__u32) > META_PIVOT);
 }
 
 static __always_inline __maybe_unused __u32
-ctx_load_meta(const struct xdp_md *ctx, const __u64 off)
+ctx_load_meta(const struct xdp_md *ctx __maybe_unused, const __u64 off)
 {
-	__u32 *data_meta = ctx_data_meta(ctx);
-	void *data = ctx_data(ctx);
+	__u32 zero = 0, *data_meta = map_lookup_elem(&cilium_xdp_scratch, &zero);
 
-	if (!ctx_no_room(data_meta + off + 1, data)) {
+	if (always_succeeds(data_meta))
 		return data_meta[off];
-	} else {
-		build_bug_on((off + 1) * sizeof(__u32) > META_PIVOT);
-		return 0;
-	}
+	build_bug_on((off + 1) * sizeof(__u32) > META_PIVOT);
+	return 0;
 }
 
 static __always_inline __maybe_unused __u32

bpf/include/bpf/loader.h

@@ -0,0 +1,27 @@
+/* SPDX-License-Identifier: GPL-2.0 */
+/* Copyright (C) 2016-2020 Authors of Cilium */
+
+#ifndef __BPF_LOADER__
+#define __BPF_LOADER__
+
+#include <linux/types.h>
+
+#define PIN_NONE		0
+#define PIN_OBJECT_NS		1
+#define PIN_GLOBAL_NS		2
+
+struct bpf_elf_map {
+	__u32 type;
+	__u32 size_key;
+	__u32 size_value;
+	__u32 max_elem;
+	__u32 flags;
+	__u32 id;
+	__u32 pinning;
+#ifdef SOCKMAP
+	__u32 inner_id;
+	__u32 inner_idx;
+#endif
+};
+
+#endif /* __BPF_LOADER__ */