Merge branch 'bpf-arm64-support-exceptions'

Puranjay Mohan says: ==================== bpf, arm64: Support Exceptions Changes in V2->V3: V2: https://lore.kernel.org/all/20230917000045.56377-1-puranjay12@gmail.com/ - Use unwinder from stacktrace.c rather than open coding the unwind logic. - Fix a bug in the prologue related to BPF_FP (Xu Kuohai) Changes in V1->V2: V1: https://lore.kernel.org/all/20230912233942.6734-1-puranjay12@gmail.com/ - Remove exceptions from DENYLIST.aarch64 as they are supported now. The base support for exceptions was merged with [1] and it was enabled for x86-64. This patch set enables the support on ARM64, all sefltests are passing: # ./test_progs -a exceptions torvalds#74/1 exceptions/exception_throw_always_1:OK torvalds#74/2 exceptions/exception_throw_always_2:OK torvalds#74/3 exceptions/exception_throw_unwind_1:OK torvalds#74/4 exceptions/exception_throw_unwind_2:OK torvalds#74/5 exceptions/exception_throw_default:OK torvalds#74/6 exceptions/exception_throw_default_value:OK torvalds#74/7 exceptions/exception_tail_call:OK torvalds#74/8 exceptions/exception_ext:OK torvalds#74/9 exceptions/exception_ext_mod_cb_runtime:OK torvalds#74/10 exceptions/exception_throw_subprog:OK torvalds#74/11 exceptions/exception_assert_nz_gfunc:OK torvalds#74/12 exceptions/exception_assert_zero_gfunc:OK torvalds#74/13 exceptions/exception_assert_neg_gfunc:OK torvalds#74/14 exceptions/exception_assert_pos_gfunc:OK torvalds#74/15 exceptions/exception_assert_negeq_gfunc:OK torvalds#74/16 exceptions/exception_assert_poseq_gfunc:OK torvalds#74/17 exceptions/exception_assert_nz_gfunc_with:OK torvalds#74/18 exceptions/exception_assert_zero_gfunc_with:OK torvalds#74/19 exceptions/exception_assert_neg_gfunc_with:OK torvalds#74/20 exceptions/exception_assert_pos_gfunc_with:OK torvalds#74/21 exceptions/exception_assert_negeq_gfunc_with:OK torvalds#74/22 exceptions/exception_assert_poseq_gfunc_with:OK torvalds#74/23 exceptions/exception_bad_assert_nz_gfunc:OK torvalds#74/24 exceptions/exception_bad_assert_zero_gfunc:OK torvalds#74/25 exceptions/exception_bad_assert_neg_gfunc:OK torvalds#74/26 exceptions/exception_bad_assert_pos_gfunc:OK torvalds#74/27 exceptions/exception_bad_assert_negeq_gfunc:OK torvalds#74/28 exceptions/exception_bad_assert_poseq_gfunc:OK torvalds#74/29 exceptions/exception_bad_assert_nz_gfunc_with:OK torvalds#74/30 exceptions/exception_bad_assert_zero_gfunc_with:OK torvalds#74/31 exceptions/exception_bad_assert_neg_gfunc_with:OK torvalds#74/32 exceptions/exception_bad_assert_pos_gfunc_with:OK torvalds#74/33 exceptions/exception_bad_assert_negeq_gfunc_with:OK torvalds#74/34 exceptions/exception_bad_assert_poseq_gfunc_with:OK torvalds#74/35 exceptions/exception_assert_range:OK torvalds#74/36 exceptions/exception_assert_range_with:OK torvalds#74/37 exceptions/exception_bad_assert_range:OK torvalds#74/38 exceptions/exception_bad_assert_range_with:OK torvalds#74/39 exceptions/non-throwing fentry -> exception_cb:OK torvalds#74/40 exceptions/throwing fentry -> exception_cb:OK torvalds#74/41 exceptions/non-throwing fexit -> exception_cb:OK torvalds#74/42 exceptions/throwing fexit -> exception_cb:OK torvalds#74/43 exceptions/throwing extension (with custom cb) -> exception_cb:OK torvalds#74/44 exceptions/throwing extension -> global func in exception_cb:OK torvalds#74/45 exceptions/exception_ext_mod_cb_runtime:OK torvalds#74/46 exceptions/throwing extension (with custom cb) -> global func in exception_cb:OK torvalds#74/47 exceptions/exception_ext:OK torvalds#74/48 exceptions/non-throwing fentry -> non-throwing subprog:OK torvalds#74/49 exceptions/throwing fentry -> non-throwing subprog:OK torvalds#74/50 exceptions/non-throwing fentry -> throwing subprog:OK torvalds#74/51 exceptions/throwing fentry -> throwing subprog:OK torvalds#74/52 exceptions/non-throwing fexit -> non-throwing subprog:OK torvalds#74/53 exceptions/throwing fexit -> non-throwing subprog:OK torvalds#74/54 exceptions/non-throwing fexit -> throwing subprog:OK torvalds#74/55 exceptions/throwing fexit -> throwing subprog:OK torvalds#74/56 exceptions/non-throwing fmod_ret -> non-throwing subprog:OK torvalds#74/57 exceptions/non-throwing fmod_ret -> non-throwing global subprog:OK torvalds#74/58 exceptions/non-throwing extension -> non-throwing subprog:OK torvalds#74/59 exceptions/non-throwing extension -> throwing subprog:OK torvalds#74/60 exceptions/non-throwing extension -> non-throwing subprog:OK torvalds#74/61 exceptions/non-throwing extension -> throwing global subprog:OK torvalds#74/62 exceptions/throwing extension -> throwing global subprog:OK torvalds#74/63 exceptions/throwing extension -> non-throwing global subprog:OK torvalds#74/64 exceptions/non-throwing extension -> main subprog:OK torvalds#74/65 exceptions/throwing extension -> main subprog:OK torvalds#74/66 exceptions/reject_exception_cb_type_1:OK torvalds#74/67 exceptions/reject_exception_cb_type_2:OK torvalds#74/68 exceptions/reject_exception_cb_type_3:OK torvalds#74/69 exceptions/reject_exception_cb_type_4:OK torvalds#74/70 exceptions/reject_async_callback_throw:OK torvalds#74/71 exceptions/reject_with_lock:OK torvalds#74/72 exceptions/reject_subprog_with_lock:OK torvalds#74/73 exceptions/reject_with_rcu_read_lock:OK torvalds#74/74 exceptions/reject_subprog_with_rcu_read_lock:OK torvalds#74/75 exceptions/reject_with_rbtree_add_throw:OK torvalds#74/76 exceptions/reject_with_reference:OK torvalds#74/77 exceptions/reject_with_cb_reference:OK torvalds#74/78 exceptions/reject_with_cb:OK torvalds#74/79 exceptions/reject_with_subprog_reference:OK torvalds#74/80 exceptions/reject_throwing_exception_cb:OK torvalds#74/81 exceptions/reject_exception_cb_call_global_func:OK torvalds#74/82 exceptions/reject_exception_cb_call_static_func:OK torvalds#74/83 exceptions/reject_multiple_exception_cb:OK torvalds#74/84 exceptions/reject_exception_throw_cb:OK torvalds#74/85 exceptions/reject_exception_throw_cb_diff:OK torvalds#74/86 exceptions/reject_set_exception_cb_bad_ret1:OK torvalds#74/87 exceptions/reject_set_exception_cb_bad_ret2:OK torvalds#74/88 exceptions/check_assert_eq_int_min:OK torvalds#74/89 exceptions/check_assert_eq_int_max:OK torvalds#74/90 exceptions/check_assert_eq_zero:OK torvalds#74/91 exceptions/check_assert_eq_llong_min:OK torvalds#74/92 exceptions/check_assert_eq_llong_max:OK torvalds#74/93 exceptions/check_assert_lt_pos:OK torvalds#74/94 exceptions/check_assert_lt_zero:OK torvalds#74/95 exceptions/check_assert_lt_neg:OK torvalds#74/96 exceptions/check_assert_le_pos:OK torvalds#74/97 exceptions/check_assert_le_zero:OK torvalds#74/98 exceptions/check_assert_le_neg:OK torvalds#74/99 exceptions/check_assert_gt_pos:OK torvalds#74/100 exceptions/check_assert_gt_zero:OK torvalds#74/101 exceptions/check_assert_gt_neg:OK torvalds#74/102 exceptions/check_assert_ge_pos:OK torvalds#74/103 exceptions/check_assert_ge_zero:OK torvalds#74/104 exceptions/check_assert_ge_neg:OK torvalds#74/105 exceptions/check_assert_range_s64:OK torvalds#74/106 exceptions/check_assert_range_u64:OK torvalds#74/107 exceptions/check_assert_single_range_s64:OK torvalds#74/108 exceptions/check_assert_single_range_u64:OK torvalds#74/109 exceptions/check_assert_generic:OK torvalds#74/110 exceptions/check_assert_with_return:OK torvalds#74 exceptions:OK Summary: 1/110 PASSED, 0 SKIPPED, 0 FAILED [1] https://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf-next.git/commit/?h=for-next&id=ec6f1b4db95b7eedb3fe85f4f14e08fa0e9281c3 ==================== Link: https://lore.kernel.org/r/20240201125225.72796-1-puranjay12@gmail.com Signed-off-by: Alexei Starovoitov <ast@kernel.org>
jonhunter · Feb 27, 2024 · e59997d · e59997d
2 parents 2ab256e + 22fc0e8
commit e59997d
Show file tree

Hide file tree

Showing 3 changed files with 94 additions and 20 deletions.
diff --git a/arch/arm64/kernel/stacktrace.c b/arch/arm64/kernel/stacktrace.c
@@ -7,6 +7,7 @@
 #include <linux/kernel.h>
 #include <linux/efi.h>
 #include <linux/export.h>
+#include <linux/filter.h>
 #include <linux/ftrace.h>
 #include <linux/kprobes.h>
 #include <linux/sched.h>
@@ -266,6 +267,31 @@ noinline noinstr void arch_stack_walk(stack_trace_consume_fn consume_entry,
 	kunwind_stack_walk(arch_kunwind_consume_entry, &data, task, regs);
 }
 
+struct bpf_unwind_consume_entry_data {
+	bool (*consume_entry)(void *cookie, u64 ip, u64 sp, u64 fp);
+	void *cookie;
+};
+
+static bool
+arch_bpf_unwind_consume_entry(const struct kunwind_state *state, void *cookie)
+{
+	struct bpf_unwind_consume_entry_data *data = cookie;
+
+	return data->consume_entry(data->cookie, state->common.pc, 0,
+				   state->common.fp);
+}
+
+noinline noinstr void arch_bpf_stack_walk(bool (*consume_entry)(void *cookie, u64 ip, u64 sp,
+								u64 fp), void *cookie)
+{
+	struct bpf_unwind_consume_entry_data data = {
+		.consume_entry = consume_entry,
+		.cookie = cookie,
+	};
+
+	kunwind_stack_walk(arch_bpf_unwind_consume_entry, &data, current, NULL);
+}
+
 static bool dump_backtrace_entry(void *arg, unsigned long where)
 {
 	char *loglvl = arg;

diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c
@@ -285,7 +285,8 @@ static bool is_lsi_offset(int offset, int scale)
 /* Tail call offset to jump into */
 #define PROLOGUE_OFFSET (BTI_INSNS + 2 + PAC_INSNS + 8)
 
-static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
+static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf,
+			  bool is_exception_cb)
 {
 	const struct bpf_prog *prog = ctx->prog;
 	const bool is_main_prog = !bpf_is_subprog(prog);
@@ -333,19 +334,34 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 	emit(A64_MOV(1, A64_R(9), A64_LR), ctx);
 	emit(A64_NOP, ctx);
 
-	/* Sign lr */
-	if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL))
-		emit(A64_PACIASP, ctx);
-
-	/* Save FP and LR registers to stay align with ARM64 AAPCS */
-	emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
-	emit(A64_MOV(1, A64_FP, A64_SP), ctx);
-
-	/* Save callee-saved registers */
-	emit(A64_PUSH(r6, r7, A64_SP), ctx);
-	emit(A64_PUSH(r8, r9, A64_SP), ctx);
-	emit(A64_PUSH(fp, tcc, A64_SP), ctx);
-	emit(A64_PUSH(fpb, A64_R(28), A64_SP), ctx);
+	if (!is_exception_cb) {
+		/* Sign lr */
+		if (IS_ENABLED(CONFIG_ARM64_PTR_AUTH_KERNEL))
+			emit(A64_PACIASP, ctx);
+		/* Save FP and LR registers to stay align with ARM64 AAPCS */
+		emit(A64_PUSH(A64_FP, A64_LR, A64_SP), ctx);
+		emit(A64_MOV(1, A64_FP, A64_SP), ctx);
+
+		/* Save callee-saved registers */
+		emit(A64_PUSH(r6, r7, A64_SP), ctx);
+		emit(A64_PUSH(r8, r9, A64_SP), ctx);
+		emit(A64_PUSH(fp, tcc, A64_SP), ctx);
+		emit(A64_PUSH(fpb, A64_R(28), A64_SP), ctx);
+	} else {
+		/*
+		 * Exception callback receives FP of Main Program as third
+		 * parameter
+		 */
+		emit(A64_MOV(1, A64_FP, A64_R(2)), ctx);
+		/*
+		 * Main Program already pushed the frame record and the
+		 * callee-saved registers. The exception callback will not push
+		 * anything and re-use the main program's stack.
+		 *
+		 * 10 registers are on the stack
+		 */
+		emit(A64_SUB_I(1, A64_SP, A64_FP, 80), ctx);
+	}
 
 	/* Set up BPF prog stack base register */
 	emit(A64_MOV(1, fp, A64_SP), ctx);
@@ -365,6 +381,20 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 		emit_bti(A64_BTI_J, ctx);
 	}
 
+	/*
+	 * Program acting as exception boundary should save all ARM64
+	 * Callee-saved registers as the exception callback needs to recover
+	 * all ARM64 Callee-saved registers in its epilogue.
+	 */
+	if (prog->aux->exception_boundary) {
+		/*
+		 * As we are pushing two more registers, BPF_FP should be moved
+		 * 16 bytes
+		 */
+		emit(A64_SUB_I(1, fp, fp, 16), ctx);
+		emit(A64_PUSH(A64_R(23), A64_R(24), A64_SP), ctx);
+	}
+
 	emit(A64_SUB_I(1, fpb, fp, ctx->fpb_offset), ctx);
 
 	/* Stack must be multiples of 16B */
@@ -653,7 +683,7 @@ static void build_plt(struct jit_ctx *ctx)
 		plt->target = (u64)&dummy_tramp;
 }
 
-static void build_epilogue(struct jit_ctx *ctx)
+static void build_epilogue(struct jit_ctx *ctx, bool is_exception_cb)
 {
 	const u8 r0 = bpf2a64[BPF_REG_0];
 	const u8 r6 = bpf2a64[BPF_REG_6];
@@ -666,6 +696,15 @@ static void build_epilogue(struct jit_ctx *ctx)
 	/* We're done with BPF stack */
 	emit(A64_ADD_I(1, A64_SP, A64_SP, ctx->stack_size), ctx);
 
+	/*
+	 * Program acting as exception boundary pushes R23 and R24 in addition
+	 * to BPF callee-saved registers. Exception callback uses the boundary
+	 * program's stack frame, so recover these extra registers in the above
+	 * two cases.
+	 */
+	if (ctx->prog->aux->exception_boundary || is_exception_cb)
+		emit(A64_POP(A64_R(23), A64_R(24), A64_SP), ctx);
+
 	/* Restore x27 and x28 */
 	emit(A64_POP(fpb, A64_R(28), A64_SP), ctx);
 	/* Restore fs (x25) and x26 */
@@ -1575,7 +1614,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	 * BPF line info needs ctx->offset[i] to be the offset of
 	 * instruction[i] in jited image, so build prologue first.
 	 */
-	if (build_prologue(&ctx, was_classic)) {
+	if (build_prologue(&ctx, was_classic, prog->aux->exception_cb)) {
 		prog = orig_prog;
 		goto out_off;
 	}
@@ -1586,7 +1625,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	}
 
 	ctx.epilogue_offset = ctx.idx;
-	build_epilogue(&ctx);
+	build_epilogue(&ctx, prog->aux->exception_cb);
 	build_plt(&ctx);
 
 	extable_align = __alignof__(struct exception_table_entry);
@@ -1614,15 +1653,15 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
 	ctx.idx = 0;
 	ctx.exentry_idx = 0;
 
-	build_prologue(&ctx, was_classic);
+	build_prologue(&ctx, was_classic, prog->aux->exception_cb);
 
 	if (build_body(&ctx, extra_pass)) {
 		bpf_jit_binary_free(header);
 		prog = orig_prog;
 		goto out_off;
 	}
 
-	build_epilogue(&ctx);
+	build_epilogue(&ctx, prog->aux->exception_cb);
 	build_plt(&ctx);
 
 	/* 3. Extra pass to validate JITed code. */
@@ -2310,3 +2349,13 @@ bool bpf_jit_supports_ptr_xchg(void)
 {
 	return true;
 }
+
+bool bpf_jit_supports_exceptions(void)
+{
+	/* We unwind through both kernel frames starting from within bpf_throw
+	 * call and BPF frames. Therefore we require FP unwinder to be enabled
+	 * to walk kernel frames and reach BPF frames in the stack trace.
+	 * ARM64 kernel is aways compiled with CONFIG_FRAME_POINTER=y
+	 */
+	return true;
+}
diff --git a/tools/testing/selftests/bpf/DENYLIST.aarch64 b/tools/testing/selftests/bpf/DENYLIST.aarch64
@@ -1,6 +1,5 @@
 bpf_cookie/multi_kprobe_attach_api               # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
 bpf_cookie/multi_kprobe_link_api                 # kprobe_multi_link_api_subtest:FAIL:fentry_raw_skel_load unexpected error: -3
-exceptions					 # JIT does not support calling kfunc bpf_throw: -524
 fexit_sleep                                      # The test never returns. The remaining tests cannot start.
 kprobe_multi_bench_attach                        # needs CONFIG_FPROBE
 kprobe_multi_test                                # needs CONFIG_FPROBE