Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

bpf: Inline assembly for packet context access #25336

Merged
merged 2 commits into from May 10, 2023
Merged

bpf: Inline assembly for packet context access #25336

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
f37ad28
bpf: tests: Minimise calls to ctx_data{,_end}() in prep for inline asm
qmonnet May 9, 2023
278058f
bpf: Avoid 32bit assignment of packet pointer
pchaigno Feb 3, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
15 changes: 0 additions & 15 deletions bpf/include/bpf/ctx/common.h
View file
Open in desktop
Expand Up @@ -13,21 +13,6 @@
#define __ctx_skb 1
#define __ctx_xdp 2

static __always_inline void *ctx_data(const struct __ctx_buff *ctx)
{
return (void *)(unsigned long)ctx->data;
}

static __always_inline void *ctx_data_meta(const struct __ctx_buff *ctx)
{
return (void *)(unsigned long)ctx->data_meta;
}

static __always_inline void *ctx_data_end(const struct __ctx_buff *ctx)
{
return (void *)(unsigned long)ctx->data_end;
}

static __always_inline bool ctx_no_room(const void *needed, const void *limit)
{
return unlikely(needed > limit);
Expand Down
23 changes: 23 additions & 0 deletions bpf/include/bpf/ctx/skb.h
View file
Open in desktop
Expand Up @@ -60,6 +60,29 @@
#define get_hash(ctx) ctx->hash
#define get_hash_recalc(ctx) get_hash(ctx)

#define DEFINE_FUNC_CTX_POINTER(FIELD) \
gentoo-root marked this conversation as resolved.
Show resolved Hide resolved
static __always_inline void * \
ctx_ ## FIELD(const struct __sk_buff *ctx) \
{ \
void *ptr; \
\
/* LLVM may generate u32 assignments of ctx->{data,data_end,data_meta}. \
* With this inline asm, LLVM loses track of the fact this field is on \
* 32 bits. \
*/ \
asm volatile("%0 = *(u32 *)(%1 + %2)" \
: "=r"(ptr) \
: "r"(ctx), "i"(offsetof(struct __sk_buff, FIELD))); \
return ptr; \
}
/* This defines ctx_data(). */
DEFINE_FUNC_CTX_POINTER(data)
/* This defines ctx_data_end(). */
DEFINE_FUNC_CTX_POINTER(data_end)
/* This defines ctx_data_meta(). */
DEFINE_FUNC_CTX_POINTER(data_meta)
#undef DEFINE_FUNC_CTX_POINTER

static __always_inline __maybe_unused int
ctx_redirect(const struct __sk_buff *ctx __maybe_unused, int ifindex, __u32 flags)
{
Expand Down
23 changes: 23 additions & 0 deletions bpf/include/bpf/ctx/xdp.h
View file
Open in desktop
Expand Up @@ -110,6 +110,29 @@ xdp_store_bytes(const struct xdp_md *ctx, __u64 off, const void *from,
#define get_hash(ctx) ({ 0; })
#define get_hash_recalc(ctx) get_hash(ctx)

#define DEFINE_FUNC_CTX_POINTER(FIELD) \
static __always_inline void * \
ctx_ ## FIELD(const struct xdp_md *ctx) \
{ \
void *ptr; \
\
/* LLVM may generate u32 assignments of ctx->{data,data_end,data_meta}. \
* With this inline asm, LLVM loses track of the fact this field is on \
* 32 bits. \
*/ \
asm volatile("%0 = *(u32 *)(%1 + %2)" \
: "=r"(ptr) \
: "r"(ctx), "i"(offsetof(struct xdp_md, FIELD))); \
return ptr; \
}
/* This defines ctx_data(). */
DEFINE_FUNC_CTX_POINTER(data)
/* This defines ctx_data_end(). */
DEFINE_FUNC_CTX_POINTER(data_end)
/* This defines ctx_data_meta(). */
DEFINE_FUNC_CTX_POINTER(data_meta)
#undef DEFINE_FUNC_CTX_POINTER

static __always_inline __maybe_unused void
__csum_replace_by_diff(__sum16 *sum, __wsum diff)
{
Expand Down
36 changes: 26 additions & 10 deletions bpf/tests/ipv6_test.c
View file
Open in desktop
Expand Up @@ -75,28 +75,36 @@ int ipv6_without_extension_header_setup(__maybe_unused struct __ctx_buff *ctx)
CHECK("xdp", "ipv6_without_extension_header")
int ipv6_without_extension_header_check(struct __ctx_buff *ctx)
{
void *data, *data_end;
struct ethhdr *l2;
struct ipv6hdr *l3;
__u32 *status_code;
__u8 nexthdr;

test_init();

if (ctx_data(ctx) + sizeof(__u32) > ctx_data_end(ctx))
data = ctx_data(ctx);
data_end = ctx_data_end(ctx);

if (data + sizeof(__u32) > data_end)
test_fatal("status code out of bounds");

status_code = ctx_data(ctx);
status_code = data;
assert(*status_code == 123);

xdp_adjust_head(ctx, 4);
l2 = ctx_data(ctx);
if ((void *)(l2 + 1) > ctx_data_end(ctx))

data = ctx_data(ctx);
data_end = ctx_data_end(ctx);

l2 = data;
if ((void *)(l2 + 1) > data_end)
test_fatal("l2 out of bounds");

assert(l2->h_proto == __bpf_htons(ETH_P_IPV6));

l3 = (void *)l2 + ETH_HLEN;
if ((void *)(l3 + 1) > ctx_data_end(ctx))
if ((void *)(l3 + 1) > data_end)
test_fatal("l3 out of bounds");

nexthdr = l3->nexthdr;
Expand Down Expand Up @@ -175,28 +183,36 @@ int ipv6_with_hop_auth_tcp_setup(__maybe_unused struct __ctx_buff *ctx)
CHECK("xdp", "ipv6_with_auth_hop_tcp")
int ipv6_with_hop_auth_tcp_check(struct __ctx_buff *ctx)
{
void *data, *data_end;
struct ethhdr *l2;
struct ipv6hdr *l3;
__u32 *status_code;
__u8 nexthdr;

test_init();

if (ctx_data(ctx) + sizeof(__u32) > ctx_data_end(ctx))
data = ctx_data(ctx);
data_end = ctx_data_end(ctx);

if (data + sizeof(__u32) > data_end)
test_fatal("status code out of bounds");

status_code = ctx_data(ctx);
status_code = data;
assert(*status_code == 1234);

xdp_adjust_head(ctx, 4);
l2 = ctx_data(ctx);
if ((void *)(l2 + 1) > ctx_data_end(ctx))

data = ctx_data(ctx);
data_end = ctx_data_end(ctx);

l2 = data;
if ((void *)(l2 + 1) > data_end)
test_fatal("l2 out of bounds");

assert(l2->h_proto == __bpf_htons(ETH_P_IPV6));

l3 = (void *)l2 + ETH_HLEN;
if ((void *)(l3 + 1) > ctx_data_end(ctx))
if ((void *)(l3 + 1) > data_end)
test_fatal("l3 out of bounds");

nexthdr = l3->nexthdr;
Expand Down
13 changes: 9 additions & 4 deletions bpf/tests/pktgen.h
View file
Open in desktop
Expand Up @@ -184,10 +184,12 @@ struct ethhdr *pktgen__push_ethhdr(struct pktgen *builder)
struct __ctx_buff *ctx = builder->ctx;
struct ethhdr *layer;
int layer_idx;
void *data;

/* Request additional tailroom, and check that we got it. */
ctx_adjust_troom(ctx, builder->cur_off + sizeof(struct ethhdr) - ctx_full_len(ctx));
if (ctx_data(ctx) + builder->cur_off + sizeof(struct ethhdr) > ctx_data_end(ctx))
data = ctx_data(ctx);
if (data + builder->cur_off + sizeof(struct ethhdr) > ctx_data_end(ctx))
return 0;

/* Check that any value within the struct will not exceed a u16 which
Expand All @@ -196,7 +198,7 @@ struct ethhdr *pktgen__push_ethhdr(struct pktgen *builder)
if (builder->cur_off >= MAX_PACKET_OFF - sizeof(struct ethhdr))
return 0;

layer = ctx_data(ctx) + builder->cur_off;
layer = data + builder->cur_off;
layer_idx = pktgen__free_layer(builder);

if (layer_idx < 0)
Expand Down Expand Up @@ -224,6 +226,7 @@ struct iphdr *pktgen__push_iphdr(struct pktgen *builder, __u32 option_bytes)
{
__u32 length = sizeof(struct iphdr) + option_bytes;
struct __ctx_buff *ctx = builder->ctx;
void *data, *data_end;
struct iphdr *layer;
int layer_idx;

Expand All @@ -232,7 +235,9 @@ struct iphdr *pktgen__push_iphdr(struct pktgen *builder, __u32 option_bytes)

/* Request additional tailroom, and check that we got it. */
ctx_adjust_troom(ctx, builder->cur_off + length - ctx_full_len(ctx));
if (ctx_data(ctx) + builder->cur_off + length > ctx_data_end(ctx))
data = ctx_data(ctx);
data_end = ctx_data_end(ctx);
if (data + builder->cur_off + length > data_end)
return 0;

/* Check that any value within the struct will not exceed a u16 which
Expand All @@ -241,7 +246,7 @@ struct iphdr *pktgen__push_iphdr(struct pktgen *builder, __u32 option_bytes)
if (builder->cur_off >= MAX_PACKET_OFF - length)
return 0;

layer = ctx_data(ctx) + builder->cur_off;
layer = data + builder->cur_off;
layer_idx = pktgen__free_layer(builder);

if (layer_idx < 0)
Expand Down