bpf: dynamic-width static data inlining

bpf/include/bpf/compiler.h

@@ -54,7 +54,7 @@
 #endif
 
 #ifndef __fetch
-# define __fetch(X)		(__u32)(__u64)(&(X))
+# define __fetch(X)		(__u64)(&(X))
 #endif
 
 #ifndef __aligned

bpf/lib/common.h

@@ -232,13 +232,11 @@ __revalidate_data_pull(struct __ctx_buff *ctx, void **data, void **data_end,
 
 /* Macros for working with L3 cilium defined IPV6 addresses */
 #define BPF_V6(dst, ...)	BPF_V6_1(dst, fetch_ipv6(__VA_ARGS__))
-#define BPF_V6_1(dst, ...)	BPF_V6_4(dst, __VA_ARGS__)
-#define BPF_V6_4(dst, a1, a2, a3, a4)		\
+#define BPF_V6_1(dst, ...)	BPF_V6_2(dst, __VA_ARGS__)
+#define BPF_V6_2(dst, a1, a2)		\
 	({					\
-		dst.p1 = a1;			\
-		dst.p2 = a2;			\
-		dst.p3 = a3;			\
-		dst.p4 = a4;			\
+		dst.d1 = a1;			\
+		dst.d2 = a2;			\
 	})
 
 #define ENDPOINT_KEY_IPV4 1

bpf/lib/endian.h

@@ -7,34 +7,75 @@
 #include <bpf/ctx/ctx.h>
 #include <bpf/api.h>
 
-#if defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && \
-	__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
-# define __bpf_ntohs(x)		__builtin_bswap16(x)
-# define __bpf_htons(x)		__builtin_bswap16(x)
-# define __bpf_ntohl(x)		__builtin_bswap32(x)
-# define __bpf_htonl(x)		__builtin_bswap32(x)
-#elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && \
-	__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
-# define __bpf_ntohs(x)		(x)
-# define __bpf_htons(x)		(x)
-# define __bpf_ntohl(x)		(x)
-# define __bpf_htonl(x)		(x)
+#define ___bpf_mvb(x, b, n, m) ((__u##b)(x) << (b-(n+1)*8) >> (b-8) << (m*8))
+
+#define ___bpf_swab16(x) ((__u16)(			\
+			  ___bpf_mvb(x, 16, 0, 1) |	\
+			  ___bpf_mvb(x, 16, 1, 0)))
+
+#define ___bpf_swab32(x) ((__u32)(			\
+			  ___bpf_mvb(x, 32, 0, 3) |	\
+			  ___bpf_mvb(x, 32, 1, 2) |	\
+			  ___bpf_mvb(x, 32, 2, 1) |	\
+			  ___bpf_mvb(x, 32, 3, 0)))
+
+#define ___bpf_swab64(x) ((__u64)(			\
+			  ___bpf_mvb(x, 64, 0, 7) |	\
+			  ___bpf_mvb(x, 64, 1, 6) |	\
+			  ___bpf_mvb(x, 64, 2, 5) |	\
+			  ___bpf_mvb(x, 64, 3, 4) |	\
+			  ___bpf_mvb(x, 64, 4, 3) |	\
+			  ___bpf_mvb(x, 64, 5, 2) |	\
+			  ___bpf_mvb(x, 64, 6, 1) |	\
+			  ___bpf_mvb(x, 64, 7, 0)))
+
+#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
+# define __bpf_ntohs(x)			__builtin_bswap16(x)
+# define __bpf_htons(x)			__builtin_bswap16(x)
+# define __bpf_constant_ntohs(x)	___bpf_swab16(x)
+# define __bpf_constant_htons(x)	___bpf_swab16(x)
+# define __bpf_ntohl(x)			__builtin_bswap32(x)
+# define __bpf_htonl(x)			__builtin_bswap32(x)
+# define __bpf_constant_ntohl(x)	___bpf_swab32(x)
+# define __bpf_constant_htonl(x)	___bpf_swab32(x)
+# define __bpf_be64_to_cpu(x)		__builtin_bswap64(x)
+# define __bpf_cpu_to_be64(x)		__builtin_bswap64(x)
+# define __bpf_constant_be64_to_cpu(x)	___bpf_swab64(x)
+# define __bpf_constant_cpu_to_be64(x)	___bpf_swab64(x)
+#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
+# define __bpf_ntohs(x)			(x)
+# define __bpf_htons(x)			(x)
+# define __bpf_constant_ntohs(x)	(x)
+# define __bpf_constant_htons(x)	(x)
+# define __bpf_ntohl(x)			(x)
+# define __bpf_htonl(x)			(x)
+# define __bpf_constant_ntohl(x)	(x)
+# define __bpf_constant_htonl(x)	(x)
+# define __bpf_be64_to_cpu(x)		(x)
+# define __bpf_cpu_to_be64(x)		(x)
+# define __bpf_constant_be64_to_cpu(x)  (x)
+# define __bpf_constant_cpu_to_be64(x)  (x)
 #else
-# error "Endianness detection needs to be set up for your compiler?!"
+# error "Fix your compiler's __BYTE_ORDER__?!"
 #endif
 
 #define bpf_htons(x)				\
 	(__builtin_constant_p(x) ?		\
-	 __constant_htons(x) : __bpf_htons(x))
+	 __bpf_constant_htons(x) : __bpf_htons(x))
 #define bpf_ntohs(x)				\
 	(__builtin_constant_p(x) ?		\
-	 __constant_ntohs(x) : __bpf_ntohs(x))
-
+	 __bpf_constant_ntohs(x) : __bpf_ntohs(x))
 #define bpf_htonl(x)				\
 	(__builtin_constant_p(x) ?		\
-	 __constant_htonl(x) : __bpf_htonl(x))
+	 __bpf_constant_htonl(x) : __bpf_htonl(x))
 #define bpf_ntohl(x)				\
 	(__builtin_constant_p(x) ?		\
-	 __constant_ntohl(x) : __bpf_ntohl(x))
+	 __bpf_constant_ntohl(x) : __bpf_ntohl(x))
+#define bpf_cpu_to_be64(x)			\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_cpu_to_be64(x) : __bpf_cpu_to_be64(x))
+#define bpf_be64_to_cpu(x)			\
+	(__builtin_constant_p(x) ?		\
+	 __bpf_constant_be64_to_cpu(x) : __bpf_be64_to_cpu(x))
 
 #endif /* __LIB_ENDIAN_H_ */

bpf/lib/static_data.h

@@ -11,21 +11,27 @@
 
 /* fetch_* macros assist in fetching variously sized static data */
 #define fetch_u16(x) (__u16)__fetch(x)
-#define fetch_u32(x) __fetch(x)
-#define fetch_u32_i(x, i) __fetch(x ## _ ## i)
-#define fetch_ipv6(x) fetch_u32_i(x, 1), fetch_u32_i(x, 2), fetch_u32_i(x, 3), fetch_u32_i(x, 4)
+#define fetch_u32(x) (__u32)__fetch(x)
+#define fetch_u32_i(x, i) fetch_u32(x ## _ ## i)
+#define fetch_u64(x) __fetch(x)
+#define fetch_u64_i(x, i) fetch_u64(x ## _ ## i)
+#define fetch_ipv6(x) fetch_u64_i(x, 1), fetch_u64_i(x, 2)
 #define fetch_mac(x) { { fetch_u32_i(x, 1), (__u16)fetch_u32_i(x, 2) } }
 
 /* DEFINE_* macros help to declare static data. */
 #define DEFINE_U16(NAME, value) volatile __u16 NAME = value
 #define DEFINE_U32(NAME, value) volatile __u32 NAME = value
 #define DEFINE_U32_I(NAME, i) volatile __u32 NAME ## _ ## i
-#define DEFINE_IPV6(NAME,									\
-		    a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16)	\
-DEFINE_U32_I(NAME, 1) = bpf_htonl( (a1) << 24 |  (a2) << 16 |  (a3) << 8 |  (a4));		\
-DEFINE_U32_I(NAME, 2) = bpf_htonl( (a5) << 24 |  (a6) << 16 |  (a7) << 8 |  (a8));		\
-DEFINE_U32_I(NAME, 3) = bpf_htonl( (a9) << 24 | (a10) << 16 | (a11) << 8 | (a12));		\
-DEFINE_U32_I(NAME, 4) = bpf_htonl((a13) << 24 | (a14) << 16 | (a15) << 8 | (a16))
+#define DEFINE_U64(NAME, value) volatile __u64 NAME = value
+#define DEFINE_U64_I(NAME, i) volatile __u64 NAME ## _ ## i
+
+#define DEFINE_IPV6(NAME, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16) \
+DEFINE_U64_I(NAME, 1) = bpf_cpu_to_be64( \
+			(__u64)(a1) << 56 | (__u64)(a2) << 48 | (__u64)(a3) << 40 | \
+			(__u64)(a4) << 32 | (a5) << 24 | (a6) << 16 | (a7) << 8 | (a8)); \
+DEFINE_U64_I(NAME, 2) = bpf_cpu_to_be64( \
+			(__u64)(a9) << 56 | (__u64)(a10) << 48 | (__u64)(a11) << 40 | \
+			(__u64)(a12) << 32 | (a13) << 24 | (a14) << 16 | (a15) << 8 | (a16));
 
 #define DEFINE_MAC(NAME, a1, a2, a3, a4, a5, a6)			\
 DEFINE_U32_I(NAME, 1) = (a1) << 24 | (a2) << 16 |  (a3) << 8 | (a4);	\

bpf/tests/config_replacement.h

@@ -10,11 +10,13 @@
 #define ___EP_CONFIG____
 
 #ifndef LXC_IP
-#define LXC_IP_1 bpf_htonl((0xbe) << 24 | (0xef) << 16 | (0) << 8 | (0))
-#define LXC_IP_2 bpf_htonl((0) << 24 | (0) << 16 | (0) << 8 | (0x01))
-#define LXC_IP_3 bpf_htonl((0) << 24 | (0) << 16 | (0) << 8 | (0x01))
-#define LXC_IP_4 bpf_htonl((0x01) << 24 | (0x65) << 16 | (0x82) << 8 | (0xbc))
-#define LXC_IP { { LXC_IP_1, LXC_IP_2, LXC_IP_3, LXC_IP_4 } }
+#define LXC_IP_1 bpf_cpu_to_be64( \
+		(__u64)(0xbe) << 56 | (__u64)(0xef) << 48 | (__u64)(0) << 40 | (__u64)(0) << 32 | \
+		(0) << 24 | (0) << 16 | (0) << 8 | (0x01))
+#define LXC_IP_2 bpf_cpu_to_be64( \
+		(__u64)(0) << 56 | (__u64)(0) << 48 | (__u64)(0) << 40 | (__u64)(0x01) << 32 | \
+		(0x01) << 24 | (0x65) << 16 | (0x82) << 8 | (0xbc))
+#define LXC_IP { { LXC_IP_1, LXC_IP_2 } }
 #endif /* LXC_IP */
 
 #ifndef LXC_IPV4

pkg/bpf/collection.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/asm"
+	"github.com/cilium/ebpf/btf"
 )
 
 const globalDataMap = ".data"
@@ -229,19 +230,15 @@ func classifyProgramTypes(spec *ebpf.CollectionSpec) error {
 // loader. This is done for compatibility with kernels that don't support
 // global data maps yet.
 //
-// Currently, all map reads are expected to be 32 bits wide until BTF MapKV
-// can be fully accessed by the caller, which would allow for querying value
-// widths.
-//
 // This works in conjunction with the __fetch macros in the datapath, which
 // emit direct array accesses instead of memory loads with an offset from the
 // map's pointer.
 func inlineGlobalData(spec *ebpf.CollectionSpec) error {
-	data, err := globalData(spec)
+	vars, err := globalData(spec)
 	if err != nil {
 		return err
 	}
-	if data == nil {
+	if vars == nil {
 		// No static data, nothing to replace.
 		return nil
 	}
@@ -270,11 +267,14 @@ func inlineGlobalData(spec *ebpf.CollectionSpec) error {
 			// Equivalent to Instruction.mapOffset().
 			off := uint32(uint64(ins.Constant) >> 32)
 
-			if off%4 != 0 {
-				return fmt.Errorf("global const access at offset %d not 32-bit aligned", off)
+			// Look up the value of the variable stored at the Datasec offset pointed
+			// at by the instruction.
+			value, ok := vars[off]
+			if !ok {
+				return fmt.Errorf("no global constant found in %s at offset %d", globalDataMap, off)
 			}
 
-			imm := spec.ByteOrder.Uint32(data[off : off+4])
+			imm := spec.ByteOrder.Uint64(value)
 
 			// Replace the map load with an immediate load. Must be a dword load
 			// to match the instruction width of a map load.
@@ -292,22 +292,50 @@ func inlineGlobalData(spec *ebpf.CollectionSpec) error {
 	return nil
 }
 
+type varOffsets map[uint32][]byte
+
 // globalData gets the contents of the first entry in the global data map
 // and removes it from the spec to prevent it from being created in the kernel.
-func globalData(spec *ebpf.CollectionSpec) ([]byte, error) {
-	data := spec.Maps[globalDataMap]
-	if data == nil {
+func globalData(spec *ebpf.CollectionSpec) (varOffsets, error) {
+	dm := spec.Maps[globalDataMap]
+	if dm == nil {
 		return nil, nil
 	}
 
-	if dl := len(data.Contents); dl != 1 {
+	if dl := len(dm.Contents); dl != 1 {
 		return nil, fmt.Errorf("expected one key in %s, found %d", globalDataMap, dl)
 	}
 
-	out, ok := (data.Contents[0].Value).([]byte)
+	ds, ok := dm.Value.(*btf.Datasec)
+	if !ok {
+		return nil, fmt.Errorf("no BTF datasec found for %s", globalDataMap)
+	}
+
+	data, ok := (dm.Contents[0].Value).([]byte)
 	if !ok {
 		return nil, fmt.Errorf("expected %s value to be a byte slice, got: %T",
-			globalDataMap, data.Contents[0].Value)
+			globalDataMap, dm.Contents[0].Value)
+	}
+
+	// Slice up the binary contents of the global data map according to the
+	// variables described in its Datasec.
+	out := make(varOffsets)
+	for _, vsi := range ds.Vars {
+		if vsi.Size > 8 {
+			return nil, fmt.Errorf("variables larger than 8 bytes are not supported (got %d)", vsi.Size)
+		}
+
+		if _, ok := out[vsi.Offset]; ok {
+			return nil, fmt.Errorf("duplicate VarSecInfo for offset %d", vsi.Offset)
+		}
+
+		// Allocate a fixed slice of 8 bytes so it can be used to store in an imm64
+		// instruction later using ByteOrder.Uint64().
+		v := make([]byte, 8)
+		copy(v, data[vsi.Offset:vsi.Offset+vsi.Size])
+
+		// Emit the variable's value by its offset in the datasec.
+		out[vsi.Offset] = v
 	}
 
 	// Remove the map definition to skip loading it into the kernel.

pkg/bpf/collection_test.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/cilium/ebpf"
 	"github.com/cilium/ebpf/asm"
+	"github.com/cilium/ebpf/btf"
 
 	"github.com/cilium/cilium/pkg/testutils"
 )
@@ -63,9 +64,21 @@ func TestInlineGlobalData(t *testing.T) {
 		ByteOrder: binary.LittleEndian,
 		Maps: map[string]*ebpf.MapSpec{
 			globalDataMap: {
+				Value: &btf.Datasec{
+					Vars: []btf.VarSecinfo{
+						{Offset: 0, Size: 4},
+						{Offset: 4, Size: 2},
+						{Offset: 8, Size: 8},
+					},
+				},
 				Contents: []ebpf.MapKV{{Value: []byte{
+					// var 1
 					0x0, 0x0, 0x0, 0x80,
-					0x1, 0x0, 0x0, 0x0,
+					// var 2 has padding since var 3 aligns to 64 bits. Fill the padding
+					// with garbage to test if it gets masked correctly by the inliner.
+					0x1, 0x0, 0xff, 0xff,
+					// var 3
+					0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7f,
 				}}},
 			},
 		},
@@ -78,6 +91,8 @@ func TestInlineGlobalData(t *testing.T) {
 					asm.LoadMapValue(asm.R0, 0, 0).WithReference(globalDataMap).WithSymbol("func1"),
 					// Pseudo-load at offset 4.
 					asm.LoadMapValue(asm.R0, 0, 4).WithReference(globalDataMap),
+					// Pseudo-load at offset 8, pointing at a u64.
+					asm.LoadMapValue(asm.R0, 0, 8).WithReference(globalDataMap),
 					asm.Return(),
 				},
 			},
@@ -88,17 +103,20 @@ func TestInlineGlobalData(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	ins := spec.Programs["prog1"].Instructions[0]
-	if want, got := 0x80000000, int(ins.Constant); want != got {
+	insns := spec.Programs["prog1"].Instructions
+	if want, got := 0x80000000, int(insns[0].Constant); want != got {
 		t.Errorf("unexpected Instruction constant: want: 0x%x, got: 0x%x", want, got)
 	}
 
-	if want, got := "func1", ins.Symbol(); want != got {
+	if want, got := "func1", insns[0].Symbol(); want != got {
 		t.Errorf("unexpected Symbol value of Instruction: want: %s, got: %s", want, got)
 	}
 
-	ins = spec.Programs["prog1"].Instructions[1]
-	if want, got := 0x1, int(ins.Constant); want != got {
+	if want, got := 0x1, int(insns[1].Constant); want != got {
+		t.Errorf("unexpected Instruction constant: want: 0x%x, got: 0x%x", want, got)
+	}
+
+	if want, got := 0x7f00000000000000, int(insns[2].Constant); want != got {
 		t.Errorf("unexpected Instruction constant: want: 0x%x, got: 0x%x", want, got)
 	}
 }

pkg/byteorder/byteorder_bigendian.go

@@ -11,5 +11,7 @@ var Native binary.ByteOrder = binary.BigEndian
 
 func HostToNetwork16(u uint16) uint16 { return u }
 func HostToNetwork32(u uint32) uint32 { return u }
+func HostToNetwork64(u uint64) uint64 { return u }
 func NetworkToHost16(u uint16) uint16 { return u }
 func NetworkToHost32(u uint32) uint32 { return u }
+func NetworkToHost64(u uint64) uint64 { return u }

pkg/byteorder/byteorder_littleendian.go

@@ -14,5 +14,7 @@ var Native binary.ByteOrder = binary.LittleEndian
 
 func HostToNetwork16(u uint16) uint16 { return bits.ReverseBytes16(u) }
 func HostToNetwork32(u uint32) uint32 { return bits.ReverseBytes32(u) }
+func HostToNetwork64(u uint64) uint64 { return bits.ReverseBytes64(u) }
 func NetworkToHost16(u uint16) uint16 { return bits.ReverseBytes16(u) }
 func NetworkToHost32(u uint32) uint32 { return bits.ReverseBytes32(u) }
+func NetworkToHost64(u uint64) uint64 { return bits.ReverseBytes64(u) }