From c0f4cd093e23b257a4c766b31a1320f22c2de16b Mon Sep 17 00:00:00 2001 From: Jiping Yin Date: Tue, 23 Apr 2024 19:20:34 +0800 Subject: [PATCH] [eBPF] Fix memory leak when reading perf buffer lost data Caused a memory leak problem in the optimization PR(https://github.com/deepflowio/deepflow/pull/5669) submitted earlier. When the user-mode program receives data from the perf buffer, the address of the handle is fwd_info instead of tracer. In this case, an error occurs when tracer->lost is read. Here is the memory check information: ==16372== Invalid write of size 8 ==16372== at 0xC33AF4: atomic64_add (atomic.h:138) ==16372== by 0xC33AF4: reader_lost_cb_a (perf_profiler.c:298) ==16372== by 0xC3ACD5: perf_reader_event_read (perf_reader.c:205) ==16372== by 0xC34AFD: reader_event_read (perf_reader.h:76) ==16372== by 0xC34AFD: process_bpf_stacktraces (perf_profiler.c:985) ==16372== by 0xC34AFD: cp_reader_work (perf_profiler.c:1076) ==16372== by 0x4C51AC2: start_thread (pthread_create.c:442) ==16372== by 0x4CE2A03: clone (clone.S:100) ==16372== Address 0x40970898 is 24 bytes inside a block of size 30 alloc'd ==16372== at 0x4848899: malloc (in /usr/libexec/valgrind/vgpreload_memcheck-amd64-linux.so) ==16372== by 0x65F333: alloc::raw_vec::RawVec::allocate_in (alloc.rs:98) ==16372== by 0x6B9A7E: ::parse_payload (raw_vec.rs:145) ==16372== by 0x9F2E6A: deepflow_agent::flow_generator::perf::FlowLog::l7_parse_log (l7_protocol_log.rs:218) ==16372== by 0x9ED85C: deepflow_agent::flow_generator::flow_map::FlowMap::collect_metric (mod.rs:455) ==16372== by 0x9EA3F6: deepflow_agent::flow_generator::flow_map::FlowMap::inject_meta_packet (flow_map.rs:869) ==16372== by 0x9DFBB2: deepflow_agent::ebpf_dispatcher::ebpf_dispatcher::EbpfDispatcher::run (ebpf_dispatcher.rs:251) ==16372== by 0x855337: std::sys_common::backtrace::__rust_begin_short_backtrace (ebpf_dispatcher.rs:736) ==16372== by 0x8694AD: core::ops::function::FnOnce::call_once{{vtable.shim}} (mod.rs:529) ==16372== by 0x1765694: std::sys::pal::unix::thread::Thread::new::thread_start (boxed.rs:2015) ==16372== by 0x4C51AC2: start_thread (pthread_create.c:442) ==16372== by 0x4CE2A03: clone (clone.S:100) ==16372== --- agent/src/ebpf/user/profile/perf_profiler.c | 8 ++++---- agent/src/ebpf/user/socket.c | 5 +++-- agent/src/ebpf/user/tracer.c | 1 + agent/src/ebpf/user/tracer.h | 1 + 4 files changed, 9 insertions(+), 6 deletions(-) diff --git a/agent/src/ebpf/user/profile/perf_profiler.c b/agent/src/ebpf/user/profile/perf_profiler.c index de604830ec6..6be2a72f61e 100644 --- a/agent/src/ebpf/user/profile/perf_profiler.c +++ b/agent/src/ebpf/user/profile/perf_profiler.c @@ -292,16 +292,16 @@ static void set_stack_trace_msg(stack_trace_msg_t * msg, } } -static void reader_lost_cb_a(void *t, u64 lost) +static void reader_lost_cb_a(void *cookie, u64 lost) { - struct bpf_tracer *tracer = (struct bpf_tracer *)t; + struct bpf_tracer *tracer = profiler_tracer; atomic64_add(&tracer->lost, lost); perf_buf_lost_a_count++; } -static void reader_lost_cb_b(void *t, u64 lost) +static void reader_lost_cb_b(void *cookie, u64 lost) { - struct bpf_tracer *tracer = (struct bpf_tracer *)t; + struct bpf_tracer *tracer = profiler_tracer; atomic64_add(&tracer->lost, lost); perf_buf_lost_b_count++; } diff --git a/agent/src/ebpf/user/socket.c b/agent/src/ebpf/user/socket.c index af333513d0e..569d470d90e 100644 --- a/agent/src/ebpf/user/socket.c +++ b/agent/src/ebpf/user/socket.c @@ -942,9 +942,10 @@ static void reader_raw_cb(void *cookie, void *raw, int raw_size) atomic64_add(&q->enqueue_nr, nr); } -static void reader_lost_cb(void *t, uint64_t lost) +static void reader_lost_cb(void *cookie, uint64_t lost) { - struct bpf_tracer *tracer = (struct bpf_tracer *)t; + struct reader_forward_info *fwd_info = cookie; + struct bpf_tracer *tracer = fwd_info->tracer; atomic64_add(&tracer->lost, lost); } diff --git a/agent/src/ebpf/user/tracer.c b/agent/src/ebpf/user/tracer.c index a3fbbad059f..3e0fc0637b1 100644 --- a/agent/src/ebpf/user/tracer.c +++ b/agent/src/ebpf/user/tracer.c @@ -1177,6 +1177,7 @@ static int perf_reader_setup(struct bpf_perf_reader *perf_reader, int thread_nr) fwd_info->queue_id = spread_id; fwd_info->cpu_id = i; + fwd_info->tracer = perf_reader->tracer; ebpf_info("Perf buffer reader cpu(%d) -> queue(%d)\n", fwd_info->cpu_id, fwd_info->queue_id); diff --git a/agent/src/ebpf/user/tracer.h b/agent/src/ebpf/user/tracer.h index 6c4c9e2d871..3a1eff79ef8 100644 --- a/agent/src/ebpf/user/tracer.h +++ b/agent/src/ebpf/user/tracer.h @@ -454,6 +454,7 @@ struct bpf_tracer_param_array { struct reader_forward_info { uint64_t queue_id; int cpu_id; + struct bpf_tracer *tracer; }; extern volatile uint32_t *tracers_lock;