Skip to content

Commit 29c677c

Browse files
borkmannborkmann
committed
Merge branch 'bpf-jmp-seq-limit'
Alexei Starovoitov says: ==================== Patch 1 - jmp sequence limit Patch 2 - improve existing tests Patch 3 - add pyperf-based realistic bpf program that takes advantage of higher limit and use it as a stress test v1->v2: fixed nit in patch 3. added Andrii's acks ==================== Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
2 parents 9efc779 + 7c94410 commit 29c677c

File tree

7 files changed

+319
-30
lines changed

7 files changed

+319
-30
lines changed

‎kernel/bpf/verifier.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -176,7 +176,7 @@ struct bpf_verifier_stack_elem {
176176
struct bpf_verifier_stack_elem *next;
177177
};
178178

179-
#define BPF_COMPLEXITY_LIMIT_STACK 1024
179+
#define BPF_COMPLEXITY_LIMIT_JMP_SEQ 8192
180180
#define BPF_COMPLEXITY_LIMIT_STATES 64
181181

182182
#define BPF_MAP_PTR_UNPRIV 1UL
@@ -782,8 +782,9 @@ static struct bpf_verifier_state *push_stack(struct bpf_verifier_env *env,
782782
if (err)
783783
goto err;
784784
elem->st.speculative |= speculative;
785-
if (env->stack_size > BPF_COMPLEXITY_LIMIT_STACK) {
786-
verbose(env, "BPF program is too complex\n");
785+
if (env->stack_size > BPF_COMPLEXITY_LIMIT_JMP_SEQ) {
786+
verbose(env, "The sequence of %d jumps is too complex.\n",
787+
env->stack_size);
787788
goto err;
788789
}
789790
return &elem->st;

‎tools/testing/selftests/bpf/prog_tests/bpf_verif_scale.c‎

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,15 @@ static int libbpf_debug_print(enum libbpf_print_level level,
1212
return vfprintf(stderr, "%s", args);
1313
}
1414

15-
static int check_load(const char *file)
15+
static int check_load(const char *file, enum bpf_prog_type type)
1616
{
1717
struct bpf_prog_load_attr attr;
1818
struct bpf_object *obj = NULL;
1919
int err, prog_fd;
2020

2121
memset(&attr, 0, sizeof(struct bpf_prog_load_attr));
2222
attr.file = file;
23-
attr.prog_type = BPF_PROG_TYPE_SCHED_CLS;
23+
attr.prog_type = type;
2424
attr.log_level = 4;
2525
err = bpf_prog_load_xattr(&attr, &obj, &prog_fd);
2626
bpf_object__close(obj);
@@ -31,19 +31,24 @@ static int check_load(const char *file)
3131

3232
void test_bpf_verif_scale(void)
3333
{
34-
const char *file1 = "./test_verif_scale1.o";
35-
const char *file2 = "./test_verif_scale2.o";
36-
const char *file3 = "./test_verif_scale3.o";
37-
int err;
34+
const char *scale[] = {
35+
"./test_verif_scale1.o", "./test_verif_scale2.o", "./test_verif_scale3.o"
36+
};
37+
const char *pyperf[] = {
38+
"./pyperf50.o", "./pyperf100.o", "./pyperf180.o"
39+
};
40+
int err, i;
3841

3942
if (verifier_stats)
4043
libbpf_set_print(libbpf_debug_print);
4144

42-
err = check_load(file1);
43-
err |= check_load(file2);
44-
err |= check_load(file3);
45-
if (!err)
46-
printf("test_verif_scale:OK\n");
47-
else
48-
printf("test_verif_scale:FAIL\n");
45+
for (i = 0; i < ARRAY_SIZE(scale); i++) {
46+
err = check_load(scale[i], BPF_PROG_TYPE_SCHED_CLS);
47+
printf("test_scale:%s:%s\n", scale[i], err ? "FAIL" : "OK");
48+
}
49+
50+
for (i = 0; i < ARRAY_SIZE(pyperf); i++) {
51+
err = check_load(pyperf[i], BPF_PROG_TYPE_RAW_TRACEPOINT);
52+
printf("test_scale:%s:%s\n", pyperf[i], err ? "FAIL" : "OK");
53+
}
4954
}

‎tools/testing/selftests/bpf/progs/pyperf.h‎

Lines changed: 268 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,268 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
// Copyright (c) 2019 Facebook
3+
#include <linux/sched.h>
4+
#include <linux/ptrace.h>
5+
#include <stdint.h>
6+
#include <stddef.h>
7+
#include <stdbool.h>
8+
#include <linux/bpf.h>
9+
#include "bpf_helpers.h"
10+
11+
#define FUNCTION_NAME_LEN 64
12+
#define FILE_NAME_LEN 128
13+
#define TASK_COMM_LEN 16
14+
15+
typedef struct {
16+
int PyThreadState_frame;
17+
int PyThreadState_thread;
18+
int PyFrameObject_back;
19+
int PyFrameObject_code;
20+
int PyFrameObject_lineno;
21+
int PyCodeObject_filename;
22+
int PyCodeObject_name;
23+
int String_data;
24+
int String_size;
25+
} OffsetConfig;
26+
27+
typedef struct {
28+
uintptr_t current_state_addr;
29+
uintptr_t tls_key_addr;
30+
OffsetConfig offsets;
31+
bool use_tls;
32+
} PidData;
33+
34+
typedef struct {
35+
uint32_t success;
36+
} Stats;
37+
38+
typedef struct {
39+
char name[FUNCTION_NAME_LEN];
40+
char file[FILE_NAME_LEN];
41+
} Symbol;
42+
43+
typedef struct {
44+
uint32_t pid;
45+
uint32_t tid;
46+
char comm[TASK_COMM_LEN];
47+
int32_t kernel_stack_id;
48+
int32_t user_stack_id;
49+
bool thread_current;
50+
bool pthread_match;
51+
bool stack_complete;
52+
int16_t stack_len;
53+
int32_t stack[STACK_MAX_LEN];
54+
55+
int has_meta;
56+
int metadata;
57+
char dummy_safeguard;
58+
} Event;
59+
60+
61+
struct bpf_elf_map {
62+
__u32 type;
63+
__u32 size_key;
64+
__u32 size_value;
65+
__u32 max_elem;
66+
__u32 flags;
67+
};
68+
69+
typedef int pid_t;
70+
71+
typedef struct {
72+
void* f_back; // PyFrameObject.f_back, previous frame
73+
void* f_code; // PyFrameObject.f_code, pointer to PyCodeObject
74+
void* co_filename; // PyCodeObject.co_filename
75+
void* co_name; // PyCodeObject.co_name
76+
} FrameData;
77+
78+
static inline __attribute__((__always_inline__)) void*
79+
get_thread_state(void* tls_base, PidData* pidData)
80+
{
81+
void* thread_state;
82+
int key;
83+
84+
bpf_probe_read(&key, sizeof(key), (void*)(long)pidData->tls_key_addr);
85+
bpf_probe_read(&thread_state, sizeof(thread_state),
86+
tls_base + 0x310 + key * 0x10 + 0x08);
87+
return thread_state;
88+
}
89+
90+
static inline __attribute__((__always_inline__)) bool
91+
get_frame_data(void* frame_ptr, PidData* pidData, FrameData* frame, Symbol* symbol)
92+
{
93+
// read data from PyFrameObject
94+
bpf_probe_read(&frame->f_back,
95+
sizeof(frame->f_back),
96+
frame_ptr + pidData->offsets.PyFrameObject_back);
97+
bpf_probe_read(&frame->f_code,
98+
sizeof(frame->f_code),
99+
frame_ptr + pidData->offsets.PyFrameObject_code);
100+
101+
// read data from PyCodeObject
102+
if (!frame->f_code)
103+
return false;
104+
bpf_probe_read(&frame->co_filename,
105+
sizeof(frame->co_filename),
106+
frame->f_code + pidData->offsets.PyCodeObject_filename);
107+
bpf_probe_read(&frame->co_name,
108+
sizeof(frame->co_name),
109+
frame->f_code + pidData->offsets.PyCodeObject_name);
110+
// read actual names into symbol
111+
if (frame->co_filename)
112+
bpf_probe_read_str(&symbol->file,
113+
sizeof(symbol->file),
114+
frame->co_filename + pidData->offsets.String_data);
115+
if (frame->co_name)
116+
bpf_probe_read_str(&symbol->name,
117+
sizeof(symbol->name),
118+
frame->co_name + pidData->offsets.String_data);
119+
return true;
120+
}
121+
122+
struct bpf_elf_map SEC("maps") pidmap = {
123+
.type = BPF_MAP_TYPE_HASH,
124+
.size_key = sizeof(int),
125+
.size_value = sizeof(PidData),
126+
.max_elem = 1,
127+
};
128+
129+
struct bpf_elf_map SEC("maps") eventmap = {
130+
.type = BPF_MAP_TYPE_HASH,
131+
.size_key = sizeof(int),
132+
.size_value = sizeof(Event),
133+
.max_elem = 1,
134+
};
135+
136+
struct bpf_elf_map SEC("maps") symbolmap = {
137+
.type = BPF_MAP_TYPE_HASH,
138+
.size_key = sizeof(Symbol),
139+
.size_value = sizeof(int),
140+
.max_elem = 1,
141+
};
142+
143+
struct bpf_elf_map SEC("maps") statsmap = {
144+
.type = BPF_MAP_TYPE_ARRAY,
145+
.size_key = sizeof(Stats),
146+
.size_value = sizeof(int),
147+
.max_elem = 1,
148+
};
149+
150+
struct bpf_elf_map SEC("maps") perfmap = {
151+
.type = BPF_MAP_TYPE_PERF_EVENT_ARRAY,
152+
.size_key = sizeof(int),
153+
.size_value = sizeof(int),
154+
.max_elem = 32,
155+
};
156+
157+
struct bpf_elf_map SEC("maps") stackmap = {
158+
.type = BPF_MAP_TYPE_STACK_TRACE,
159+
.size_key = sizeof(int),
160+
.size_value = sizeof(long long) * 127,
161+
.max_elem = 1000,
162+
};
163+
164+
static inline __attribute__((__always_inline__)) int __on_event(struct pt_regs *ctx)
165+
{
166+
uint64_t pid_tgid = bpf_get_current_pid_tgid();
167+
pid_t pid = (pid_t)(pid_tgid >> 32);
168+
PidData* pidData = bpf_map_lookup_elem(&pidmap, &pid);
169+
if (!pidData)
170+
return 0;
171+
172+
int zero = 0;
173+
Event* event = bpf_map_lookup_elem(&eventmap, &zero);
174+
if (!event)
175+
return 0;
176+
177+
event->pid = pid;
178+
179+
event->tid = (pid_t)pid_tgid;
180+
bpf_get_current_comm(&event->comm, sizeof(event->comm));
181+
182+
event->user_stack_id = bpf_get_stackid(ctx, &stackmap, BPF_F_USER_STACK);
183+
event->kernel_stack_id = bpf_get_stackid(ctx, &stackmap, 0);
184+
185+
void* thread_state_current = (void*)0;
186+
bpf_probe_read(&thread_state_current,
187+
sizeof(thread_state_current),
188+
(void*)(long)pidData->current_state_addr);
189+
190+
struct task_struct* task = (struct task_struct*)bpf_get_current_task();
191+
void* tls_base = (void*)task;
192+
193+
void* thread_state = pidData->use_tls ? get_thread_state(tls_base, pidData)
194+
: thread_state_current;
195+
event->thread_current = thread_state == thread_state_current;
196+
197+
if (pidData->use_tls) {
198+
uint64_t pthread_created;
199+
uint64_t pthread_self;
200+
bpf_probe_read(&pthread_self, sizeof(pthread_self), tls_base + 0x10);
201+
202+
bpf_probe_read(&pthread_created,
203+
sizeof(pthread_created),
204+
thread_state + pidData->offsets.PyThreadState_thread);
205+
event->pthread_match = pthread_created == pthread_self;
206+
} else {
207+
event->pthread_match = 1;
208+
}
209+
210+
if (event->pthread_match || !pidData->use_tls) {
211+
void* frame_ptr;
212+
FrameData frame;
213+
Symbol sym = {};
214+
int cur_cpu = bpf_get_smp_processor_id();
215+
216+
bpf_probe_read(&frame_ptr,
217+
sizeof(frame_ptr),
218+
thread_state + pidData->offsets.PyThreadState_frame);
219+
220+
int32_t* symbol_counter = bpf_map_lookup_elem(&symbolmap, &sym);
221+
if (symbol_counter == NULL)
222+
return 0;
223+
#pragma unroll
224+
/* Unwind python stack */
225+
for (int i = 0; i < STACK_MAX_LEN; ++i) {
226+
if (frame_ptr && get_frame_data(frame_ptr, pidData, &frame, &sym)) {
227+
int32_t new_symbol_id = *symbol_counter * 64 + cur_cpu;
228+
int32_t *symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
229+
if (!symbol_id) {
230+
bpf_map_update_elem(&symbolmap, &sym, &zero, 0);
231+
symbol_id = bpf_map_lookup_elem(&symbolmap, &sym);
232+
if (!symbol_id)
233+
return 0;
234+
}
235+
if (*symbol_id == new_symbol_id)
236+
(*symbol_counter)++;
237+
event->stack[i] = *symbol_id;
238+
event->stack_len = i + 1;
239+
frame_ptr = frame.f_back;
240+
}
241+
}
242+
event->stack_complete = frame_ptr == NULL;
243+
} else {
244+
event->stack_complete = 1;
245+
}
246+
247+
Stats* stats = bpf_map_lookup_elem(&statsmap, &zero);
248+
if (stats)
249+
stats->success++;
250+
251+
event->has_meta = 0;
252+
bpf_perf_event_output(ctx, &perfmap, 0, event, offsetof(Event, metadata));
253+
return 0;
254+
}
255+
256+
SEC("raw_tracepoint/kfree_skb")
257+
int on_event(struct pt_regs* ctx)
258+
{
259+
int i, ret = 0;
260+
ret |= __on_event(ctx);
261+
ret |= __on_event(ctx);
262+
ret |= __on_event(ctx);
263+
ret |= __on_event(ctx);
264+
ret |= __on_event(ctx);
265+
return ret;
266+
}
267+
268+
char _license[] SEC("license") = "GPL";

‎tools/testing/selftests/bpf/progs/pyperf100.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
// Copyright (c) 2019 Facebook
3+
#define STACK_MAX_LEN 100
4+
#include "pyperf.h"

‎tools/testing/selftests/bpf/progs/pyperf180.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
// Copyright (c) 2019 Facebook
3+
#define STACK_MAX_LEN 180
4+
#include "pyperf.h"

‎tools/testing/selftests/bpf/progs/pyperf50.c‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
// SPDX-License-Identifier: GPL-2.0
2+
// Copyright (c) 2019 Facebook
3+
#define STACK_MAX_LEN 50
4+
#include "pyperf.h"

0 commit comments

Comments
 (0)