Skip to content

Commit e384c7b

Browse files
Kui-Feng Leeanakryiko
Kui-Feng Lee
authored and
anakryiko
committed
bpf, x86: Create bpf_tramp_run_ctx on the caller thread's stack
BPF trampolines will create a bpf_tramp_run_ctx, a bpf_run_ctx, on stacks and set/reset the current bpf_run_ctx before/after calling a bpf_prog. Signed-off-by: Kui-Feng Lee <kuifeng@fb.com> Signed-off-by: Alexei Starovoitov <ast@kernel.org> Signed-off-by: Andrii Nakryiko <andrii@kernel.org> Acked-by: Andrii Nakryiko <andrii@kernel.org> Link: https://lore.kernel.org/bpf/20220510205923.3206889-3-kuifeng@fb.com
1 parent f7e0bea commit e384c7b

File tree

4 files changed

+66
-19
lines changed

4 files changed

+66
-19
lines changed

‎arch/x86/net/bpf_jit_comp.c‎

Lines changed: 32 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1763,14 +1763,30 @@ static void restore_regs(const struct btf_func_model *m, u8 **prog, int nr_args,
17631763

17641764
static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
17651765
struct bpf_tramp_link *l, int stack_size,
1766-
bool save_ret)
1766+
int run_ctx_off, bool save_ret)
17671767
{
17681768
u8 *prog = *pprog;
17691769
u8 *jmp_insn;
1770+
int ctx_cookie_off = offsetof(struct bpf_tramp_run_ctx, bpf_cookie);
17701771
struct bpf_prog *p = l->link.prog;
17711772

1773+
/* mov rdi, 0 */
1774+
emit_mov_imm64(&prog, BPF_REG_1, 0, 0);
1775+
1776+
/* Prepare struct bpf_tramp_run_ctx.
1777+
*
1778+
* bpf_tramp_run_ctx is already preserved by
1779+
* arch_prepare_bpf_trampoline().
1780+
*
1781+
* mov QWORD PTR [rbp - run_ctx_off + ctx_cookie_off], rdi
1782+
*/
1783+
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_1, -run_ctx_off + ctx_cookie_off);
1784+
17721785
/* arg1: mov rdi, progs[i] */
17731786
emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
1787+
/* arg2: lea rsi, [rbp - ctx_cookie_off] */
1788+
EMIT4(0x48, 0x8D, 0x75, -run_ctx_off);
1789+
17741790
if (emit_call(&prog,
17751791
p->aux->sleepable ? __bpf_prog_enter_sleepable :
17761792
__bpf_prog_enter, prog))
@@ -1816,6 +1832,8 @@ static int invoke_bpf_prog(const struct btf_func_model *m, u8 **pprog,
18161832
emit_mov_imm64(&prog, BPF_REG_1, (long) p >> 32, (u32) (long) p);
18171833
/* arg2: mov rsi, rbx <- start time in nsec */
18181834
emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_6);
1835+
/* arg3: lea rdx, [rbp - run_ctx_off] */
1836+
EMIT4(0x48, 0x8D, 0x55, -run_ctx_off);
18191837
if (emit_call(&prog,
18201838
p->aux->sleepable ? __bpf_prog_exit_sleepable :
18211839
__bpf_prog_exit, prog))
@@ -1853,14 +1871,14 @@ static int emit_cond_near_jump(u8 **pprog, void *func, void *ip, u8 jmp_cond)
18531871

18541872
static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
18551873
struct bpf_tramp_links *tl, int stack_size,
1856-
bool save_ret)
1874+
int run_ctx_off, bool save_ret)
18571875
{
18581876
int i;
18591877
u8 *prog = *pprog;
18601878

18611879
for (i = 0; i < tl->nr_links; i++) {
18621880
if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size,
1863-
save_ret))
1881+
run_ctx_off, save_ret))
18641882
return -EINVAL;
18651883
}
18661884
*pprog = prog;
@@ -1869,7 +1887,7 @@ static int invoke_bpf(const struct btf_func_model *m, u8 **pprog,
18691887

18701888
static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
18711889
struct bpf_tramp_links *tl, int stack_size,
1872-
u8 **branches)
1890+
int run_ctx_off, u8 **branches)
18731891
{
18741892
u8 *prog = *pprog;
18751893
int i;
@@ -1880,7 +1898,7 @@ static int invoke_bpf_mod_ret(const struct btf_func_model *m, u8 **pprog,
18801898
emit_mov_imm32(&prog, false, BPF_REG_0, 0);
18811899
emit_stx(&prog, BPF_DW, BPF_REG_FP, BPF_REG_0, -8);
18821900
for (i = 0; i < tl->nr_links; i++) {
1883-
if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, true))
1901+
if (invoke_bpf_prog(m, &prog, tl->links[i], stack_size, run_ctx_off, true))
18841902
return -EINVAL;
18851903

18861904
/* mod_ret prog stored return value into [rbp - 8]. Emit:
@@ -1986,7 +2004,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
19862004
void *orig_call)
19872005
{
19882006
int ret, i, nr_args = m->nr_args;
1989-
int regs_off, ip_off, args_off, stack_size = nr_args * 8;
2007+
int regs_off, ip_off, args_off, stack_size = nr_args * 8, run_ctx_off;
19902008
struct bpf_tramp_links *fentry = &tlinks[BPF_TRAMP_FENTRY];
19912009
struct bpf_tramp_links *fexit = &tlinks[BPF_TRAMP_FEXIT];
19922010
struct bpf_tramp_links *fmod_ret = &tlinks[BPF_TRAMP_MODIFY_RETURN];
@@ -2016,6 +2034,8 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
20162034
* RBP - args_off [ args count ] always
20172035
*
20182036
* RBP - ip_off [ traced function ] BPF_TRAMP_F_IP_ARG flag
2037+
*
2038+
* RBP - run_ctx_off [ bpf_tramp_run_ctx ]
20192039
*/
20202040

20212041
/* room for return value of orig_call or fentry prog */
@@ -2034,6 +2054,9 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
20342054

20352055
ip_off = stack_size;
20362056

2057+
stack_size += (sizeof(struct bpf_tramp_run_ctx) + 7) & ~0x7;
2058+
run_ctx_off = stack_size;
2059+
20372060
if (flags & BPF_TRAMP_F_SKIP_FRAME) {
20382061
/* skip patched call instruction and point orig_call to actual
20392062
* body of the kernel function.
@@ -2081,7 +2104,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
20812104
}
20822105

20832106
if (fentry->nr_links)
2084-
if (invoke_bpf(m, &prog, fentry, regs_off,
2107+
if (invoke_bpf(m, &prog, fentry, regs_off, run_ctx_off,
20852108
flags & BPF_TRAMP_F_RET_FENTRY_RET))
20862109
return -EINVAL;
20872110

@@ -2092,7 +2115,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
20922115
return -ENOMEM;
20932116

20942117
if (invoke_bpf_mod_ret(m, &prog, fmod_ret, regs_off,
2095-
branches)) {
2118+
run_ctx_off, branches)) {
20962119
ret = -EINVAL;
20972120
goto cleanup;
20982121
}
@@ -2129,7 +2152,7 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *im, void *image, void *i
21292152
}
21302153

21312154
if (fexit->nr_links)
2132-
if (invoke_bpf(m, &prog, fexit, regs_off, false)) {
2155+
if (invoke_bpf(m, &prog, fexit, regs_off, run_ctx_off, false)) {
21332156
ret = -EINVAL;
21342157
goto cleanup;
21352158
}

‎include/linux/bpf.h‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -730,6 +730,8 @@ struct bpf_tramp_links {
730730
int nr_links;
731731
};
732732

733+
struct bpf_tramp_run_ctx;
734+
733735
/* Different use cases for BPF trampoline:
734736
* 1. replace nop at the function entry (kprobe equivalent)
735737
* flags = BPF_TRAMP_F_RESTORE_REGS
@@ -756,10 +758,11 @@ int arch_prepare_bpf_trampoline(struct bpf_tramp_image *tr, void *image, void *i
756758
struct bpf_tramp_links *tlinks,
757759
void *orig_call);
758760
/* these two functions are called from generated trampoline */
759-
u64 notrace __bpf_prog_enter(struct bpf_prog *prog);
760-
void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start);
761-
u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog);
762-
void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start);
761+
u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
762+
void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx);
763+
u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx);
764+
void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
765+
struct bpf_tramp_run_ctx *run_ctx);
763766
void notrace __bpf_tramp_enter(struct bpf_tramp_image *tr);
764767
void notrace __bpf_tramp_exit(struct bpf_tramp_image *tr);
765768

@@ -1351,6 +1354,12 @@ struct bpf_trace_run_ctx {
13511354
u64 bpf_cookie;
13521355
};
13531356

1357+
struct bpf_tramp_run_ctx {
1358+
struct bpf_run_ctx run_ctx;
1359+
u64 bpf_cookie;
1360+
struct bpf_run_ctx *saved_run_ctx;
1361+
};
1362+
13541363
static inline struct bpf_run_ctx *bpf_set_run_ctx(struct bpf_run_ctx *new_ctx)
13551364
{
13561365
struct bpf_run_ctx *old_ctx = NULL;

‎kernel/bpf/syscall.c‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5020,6 +5020,7 @@ static bool syscall_prog_is_valid_access(int off, int size,
50205020
BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
50215021
{
50225022
struct bpf_prog * __maybe_unused prog;
5023+
struct bpf_tramp_run_ctx __maybe_unused run_ctx;
50235024

50245025
switch (cmd) {
50255026
case BPF_MAP_CREATE:
@@ -5047,13 +5048,15 @@ BPF_CALL_3(bpf_sys_bpf, int, cmd, union bpf_attr *, attr, u32, attr_size)
50475048
return -EINVAL;
50485049
}
50495050

5050-
if (!__bpf_prog_enter_sleepable(prog)) {
5051+
run_ctx.bpf_cookie = 0;
5052+
run_ctx.saved_run_ctx = NULL;
5053+
if (!__bpf_prog_enter_sleepable(prog, &run_ctx)) {
50515054
/* recursion detected */
50525055
bpf_prog_put(prog);
50535056
return -EBUSY;
50545057
}
50555058
attr->test.retval = bpf_prog_run(prog, (void *) (long) attr->test.ctx_in);
5056-
__bpf_prog_exit_sleepable(prog, 0 /* bpf_prog_run does runtime stats */);
5059+
__bpf_prog_exit_sleepable(prog, 0 /* bpf_prog_run does runtime stats */, &run_ctx);
50575060
bpf_prog_put(prog);
50585061
return 0;
50595062
#endif

‎kernel/bpf/trampoline.c‎

Lines changed: 16 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -568,11 +568,14 @@ static void notrace inc_misses_counter(struct bpf_prog *prog)
568568
* [2..MAX_U64] - execute bpf prog and record execution time.
569569
* This is start time.
570570
*/
571-
u64 notrace __bpf_prog_enter(struct bpf_prog *prog)
571+
u64 notrace __bpf_prog_enter(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
572572
__acquires(RCU)
573573
{
574574
rcu_read_lock();
575575
migrate_disable();
576+
577+
run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
578+
576579
if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
577580
inc_misses_counter(prog);
578581
return 0;
@@ -602,29 +605,38 @@ static void notrace update_prog_stats(struct bpf_prog *prog,
602605
}
603606
}
604607

605-
void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start)
608+
void notrace __bpf_prog_exit(struct bpf_prog *prog, u64 start, struct bpf_tramp_run_ctx *run_ctx)
606609
__releases(RCU)
607610
{
611+
bpf_reset_run_ctx(run_ctx->saved_run_ctx);
612+
608613
update_prog_stats(prog, start);
609614
__this_cpu_dec(*(prog->active));
610615
migrate_enable();
611616
rcu_read_unlock();
612617
}
613618

614-
u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog)
619+
u64 notrace __bpf_prog_enter_sleepable(struct bpf_prog *prog, struct bpf_tramp_run_ctx *run_ctx)
615620
{
616621
rcu_read_lock_trace();
617622
migrate_disable();
618623
might_fault();
624+
619625
if (unlikely(__this_cpu_inc_return(*(prog->active)) != 1)) {
620626
inc_misses_counter(prog);
621627
return 0;
622628
}
629+
630+
run_ctx->saved_run_ctx = bpf_set_run_ctx(&run_ctx->run_ctx);
631+
623632
return bpf_prog_start_time();
624633
}
625634

626-
void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start)
635+
void notrace __bpf_prog_exit_sleepable(struct bpf_prog *prog, u64 start,
636+
struct bpf_tramp_run_ctx *run_ctx)
627637
{
638+
bpf_reset_run_ctx(run_ctx->saved_run_ctx);
639+
628640
update_prog_stats(prog, start);
629641
__this_cpu_dec(*(prog->active));
630642
migrate_enable();

0 commit comments

Comments
 (0)