kfree_skb backtrace support

README.md

@@ -153,6 +153,7 @@ Usage:
     --diag-keep      don't quit when abnormal packet found
     --hooks          print netfilter hooks if dropping by netfilter
     --drop           skb drop monitor mode, for replace of 'droptrace'
+    --drop-stack     print the kernel function call stack of kfree_skb
 
     -v               show log information
     --debug          show debug information
@@ -171,6 +172,7 @@ Usage:
 - `diag-keep`：持续跟踪。`diag`模式下，默认在跟踪到异常报文后会停止跟踪，使用该参数后，会持续跟踪下去。
 - `hooks`：结合netfilter做的适配，详见下文
 - `drop`：进行系统丢包监控，取代原先的`droptrace`
+- `drop-stack`: 打印kfree_skb内核函数的调用堆栈
 
 下面我们首先来看一下默认模式下的工具使用方法。
 

component/parse_sym.c

@@ -32,6 +32,7 @@ static struct sym_result *lookup_sym_cache(__u64 pc, bool exact)
 				if (head->pc == pc)
 					return head;
 				sym = head;
+				break;
 			}
 		} else {
 			if (head->start == pc)

script/bash-completion.sh

@@ -1,2 +1,2 @@
-complete -W '-s --saddr --saddr6 -d --daddr --daddr6 --addr --addr6 -p -D --dport -S --sport -P --port --pid -t --trace -v --detail --debug --ret --basic --diag --diag-quiet --diag-keep --date --drop --hooks -h --help' nettrace
+complete -W '-s --saddr --saddr6 -d --daddr --daddr6 --addr --addr6 -p -D --dport -S --sport -P --port --pid -t --trace -v --detail --debug --ret --basic --diag --diag-quiet --diag-keep --date --drop --drop-stack --hooks -h --help' nettrace
 complete -W '-h -s -d --addr -p --dport --sport --port -t -v --detail --stack --stack-tracer --timeline -c --ret --skb-mode --force-stack --tcp-flags -o --output' nettrace-legacy

shared/bpf/skb_shared.h

@@ -79,4 +79,9 @@ typedef struct {
 
 #define CONFIG_MAP_SIZE	1024
 
+#ifndef PERF_MAX_STACK_DEPTH
+#define PERF_MAX_STACK_DEPTH 127
+#endif
+typedef __u64 stack_trace_t[PERF_MAX_STACK_DEPTH];
+
 #endif

src/Makefile

@@ -25,11 +25,15 @@ kprobe_gen_cmd = @make BPF_CFLAGS="$(BPF_CFLAGS) $(1)" \
 		mv progs/kprobe.skel.h progs/$(2).skel.h
 
 ifndef COMPAT
+	CFLAGS += -DSTACK_TRACE
 $(bpf_progs_ext):
 	rm -rf progs/kprobe*.skel.h
 	$(call kprobe_gen_cmd,-DCORE_FULL,kprobe_core)
 	make progs/kprobe.skel.h
 else
+ifneq ($(shell grep -c get_stackid $(HEADERS)/include/uapi/linux/bpf.h),0)
+	CFLAGS += -DSTACK_TRACE
+endif
 $(bpf_progs_ext): %: %.skel.h
 	@:
 endif

src/analysis.c

@@ -142,13 +142,14 @@ static void analy_entry_handle(analy_entry_t *entry)
 {
 	packet_t *pkt = &entry->event->pkt;
 	static char buf[1024], tinfo[128];
+	event_t *e = entry->event;
 	rule_t *rule;
 	trace_t *t;
 
 	t = get_trace_from_analy_entry(entry);
 	pr_debug("output entry(%llx)\n", PTR2X(entry));
 	if (trace_ctx.detail) {
-		detail_event_t *detail = (void *)entry->event;
+		detail_event_t *detail = (void *)e;
 		static char ifbuf[IF_NAMESIZE];
 		char *ifname = detail->ifname;
 
@@ -195,6 +196,8 @@ static void analy_entry_handle(analy_entry_t *entry)
 	}
 out:
 	pr_info("%s\n", buf);
+	if (trace_is_stack(t))
+		trace_ctx.ops->print_stack(e->stack_id);
 }
 
 static void analy_ctx_free(analy_ctx_t *ctx)

src/gen_trace.py

@@ -49,7 +49,7 @@ def gen_group_init(group, name):
 
 global_status = {}
 global_names = {}
-global_status['trace_index'] = 0
+global_status['trace_index'] = 1
 
 rule_levels = {
     'info': 'RULE_INFO',

src/nettrace.c

@@ -78,6 +78,13 @@ static void do_parse_args(int argc, char *argv[])
 			.type = OPTION_BOOL,
 			.desc = "skb drop monitor mode, for replace of 'droptrace'",
 		},
+#ifdef STACK_TRACE
+		{
+			.lname = "drop-stack", .dest = &trace_args->drop_stack,
+			.type = OPTION_BOOL,
+			.desc = "print the kernel function call stack of kfree_skb",
+		},
+#endif
 		{ .type = OPTION_BLANK },
 		{
 			.sname = 'v', .dest = &show_log,

src/progs/kprobe.c

@@ -19,6 +19,15 @@ struct {
 	__uint(max_entries, TRACE_MAX);
 } m_ret SEC(".maps");
 
+#ifdef STACK_TRACE
+struct {
+	__uint(type, BPF_MAP_TYPE_STACK_TRACE);
+	__uint(max_entries, 16384);
+	__uint(key_size, sizeof(__u32));
+	__uint(value_size, sizeof(stack_trace_t));
+} m_stack SEC(".maps");
+#endif
+
 #ifdef KERN_VER
 __u32 kern_ver SEC("version") = KERN_VER;
 #endif
@@ -64,8 +73,38 @@ static try_inline int put_ret(int func)
 	return 0;
 }
 
-static try_inline int handle_entry(void *regs, struct sk_buff *skb, event_t *e,
-			       int size, int func)
+#ifdef STACK_TRACE
+static try_inline void try_trace_stack(void *regs, bpf_args_t *bpf_args,
+				       event_t *e, int func)
+{
+	int i = 0, key;
+	u16 *funcs;
+
+	if (!ARGS_GET(stack))
+		return;
+
+	funcs = ARGS_GET(stack_funs);
+
+#pragma unroll
+	for (; i < MAX_FUNC_STACK; i++) {
+		if (!funcs[i])
+			break;
+		if (funcs[i] == func)
+			goto do_stack;
+	}
+	return;
+
+do_stack:
+	key = bpf_get_stackid(regs, &m_stack, 0);
+	e->stack_id = key;
+}
+#else
+static try_inline void try_trace_stack(void *regs, bpf_args_t *bpf_args,
+				       event_t *e, int func) { }
+#endif
+
+static try_inline int handle_entry(void *regs, struct sk_buff *skb,
+				   event_t *e, int size, int func)
 {
 	packet_t *pkt = &e->pkt;
 	bool *matched;
@@ -111,6 +150,7 @@ static try_inline int handle_entry(void *regs, struct sk_buff *skb, event_t *e,
 	}
 
 out:
+	try_trace_stack(regs, bpf_args, e, func);
 	pkt->ts = bpf_ktime_get_ns();
 	e->key = (u64)(void *)skb;
 
@@ -129,8 +169,8 @@ static try_inline int handle_destroy(struct sk_buff *skb)
 }
 
 static try_inline int default_handle_entry(struct pt_regs *ctx,
-				       struct sk_buff *skb,
-				       int func)
+					   struct sk_buff *skb,
+					   int func)
 {
 	if (ARGS_GET_CONFIG(detail)) {
 		detail_event_t e = { .func = func };

src/progs/shared.h

@@ -1,6 +1,8 @@
 #ifndef _H_PROGS_SHARED
 #define _H_PROGS_SHARED
 
+#define MAX_FUNC_STACK 16
+
 #define DEFINE_BPF_ARGS()	\
 	u32  trace_mode;	\
 	u32  pid;		\
@@ -10,14 +12,19 @@
 	bool detail;		\
 	bool hooks;		\
 	bool ready;		\
-	bool nft_high;
+	bool nft_high;		\
+	bool stack;		\
+	u16  stack_funs[MAX_FUNC_STACK];
 
 #include <skb_shared.h>
 
 typedef struct __attribute__((__packed__)) {
 	packet_t	pkt;
 	u64		key;
 	u32		func;
+#ifdef STACK_TRACE
+	u32		stack_id;
+#endif
 } event_t;
 
 typedef struct __attribute__((__packed__)) {

src/trace.c

@@ -198,6 +198,9 @@ static int trace_prepare_args()
 	if (args->drop) {
 		trace_ctx.mode = TRACE_MODE_DROP;
 		traces = "kfree_skb";
+	} else if (args->drop_stack) {
+		pr_err("--drop should be set!\n");
+		goto err;
 	}
 
 	if (!traces) {
@@ -244,11 +247,17 @@ static int trace_prepare_args()
 			trace_group_enable("life");
 		break;
 	case TRACE_MODE_BASIC:
-	case TRACE_MODE_DROP:
+	case TRACE_MODE_DROP: {
+		trace_t *drop_trace = search_trace_enabled(traces);
+
 		if (!trace_ctx.drop_reason)
 			pr_warn("skb drop reason is not support by your kernel"
 				", drop reason will not be printed\n");
+
+		if (args->drop_stack && trace_set_stack(drop_trace))
+			goto err;
 		break;
+	}
 	default:
 		goto err;
 	}

src/trace.h

@@ -23,6 +23,7 @@ struct analyzer;
 #define TRACE_ENABLE	(1 << 1)
 #define TRACE_INVALID	(1 << 2)
 #define TRACE_RET	(1 << 3)
+#define TRACE_STACK	(1 << 4)
 
 #define trace_for_each(pos) list_for_each_entry(pos, &trace_list, sibling)
 
@@ -63,6 +64,7 @@ typedef struct trace_args {
 	bool basic;
 	bool drop;
 	bool date;
+	bool drop_stack;
 	char *traces;
 } trace_args_t;
 
@@ -75,6 +77,7 @@ typedef struct {
 	int (*trace_anal)(event_t *e);
 	void (*trace_close)();
 	void (*trace_ready)();
+	void (*print_stack)(int key);
 	struct analyzer *analyzer;
 } trace_ops_t;
 
@@ -136,6 +139,31 @@ static inline bool trace_is_ret(trace_t *t)
 	return t->status & TRACE_RET;
 }
 
+static inline int trace_set_stack(trace_t *t)
+{
+	int i = 0;
+
+	for (; i < MAX_FUNC_STACK; i++) {
+		if (!trace_ctx.bpf_args.stack_funs[i]) {
+			trace_ctx.bpf_args.stack_funs[i] = t->index;
+			break;
+		}
+	}
+	if (i == MAX_FUNC_STACK) {
+		pr_err("stack trace is full!\n");
+		return -1;
+	}
+
+	trace_ctx.bpf_args.stack = true;
+	t->status |= TRACE_STACK;
+	return 0;
+}
+
+static inline bool trace_is_stack(trace_t *t)
+{
+	return t->status & TRACE_STACK;
+}
+
 static inline void trace_stop()
 {
 	trace_ctx.stop = true;

src/trace.yaml

@@ -200,20 +200,27 @@ children:
     visual: true
     children:
     - ip_rcv:0
+    - ipv6_rcv:0
     - ip_rcv_core:0
+    - ip6_rcv_core:0
     - ip_rcv_finish:2
+    - ip6_rcv_finish:2
     - ip_local_deliver:0
     - ip_local_deliver_finish:2
     - ip_forward:0
+    - ip6_forward:0
     - ip_forward_finish:0
   - name: ip-out
     desc: ip layer of packet out
     visual: true
     children:
     - ip_output:2
+    - ip6_output:2
     - ip_finish_output:2
+    - ip6_finish_output:2
     - ip_finish_output_gso:2
     - ip_finish_output2:2
+    - ip6_finish_output2:2
   - name: ip-route
     desc: ip route for packet in and out
     visual: true
@@ -238,10 +245,13 @@ children:
     visual: true
     children:
     - tcp_v4_rcv:0
+    - tcp_v6_rcv:0
     - tcp_filter:1
     - tcp_child_process:2
     - tcp_v4_send_reset:1
+    - tcp_v6_send_reset:1
     - tcp_v4_do_rcv:1
+    - tcp_v6_do_rcv:1
     - tcp_rcv_established:1
     - tcp_rcv_state_process:1
     - tcp_queue_rcv:1
@@ -252,6 +262,13 @@ children:
         level: warn
         msg: tcp port is not listened
         adv: check your target tcp port
+    - name: inet6_lookup_listener:2
+      analyzer: ret
+      rules:
+      - exp: eq 0
+        level: warn
+        msg: tcp port is not listened
+        adv: check your target tcp port
 - name: udp
   desc: udp protocol layer (L4) of the network stack
   children:

src/trace_probe.c

@@ -1,3 +1,5 @@
+#include <parse_sym.h>
+
 #include "trace.h"
 #include "progs/kprobe.skel.h"
 #ifndef COMPAT_MODE
@@ -210,6 +212,28 @@ static void probe_trace_ready()
 	bpf_set_config_field(skel, bss, ready, true);
 }
 
+static void probe_print_stack(int key)
+{
+	int map_fd = bpf_map__fd(skel->maps.m_stack);
+	__u64 ip[PERF_MAX_STACK_DEPTH] = {};
+	struct sym_result *sym;
+	int i = 0;
+
+	if (bpf_map_lookup_elem(map_fd, &key, ip)) {
+		pr_info("Call Stack Error!\n");
+		return;
+	}
+
+	pr_info("Call Stack:\n");
+	for (; i < PERF_MAX_STACK_DEPTH && ip[i]; i++) {
+		sym = parse_sym(ip[i]);
+		if (!sym)
+			break;
+		pr_info("    -> %s\n", sym->desc);
+	}
+	pr_info("\n");
+}
+
 analyzer_t probe_analyzer =  {
 	.mode = TRACE_MODE_INETL_MASK | TRACE_MODE_TIMELINE_MASK,
 	.analy_entry = probe_analy_entry,
@@ -221,5 +245,6 @@ trace_ops_t probe_ops = {
 	.trace_open = probe_trace_open,
 	.trace_close = probe_trace_close,
 	.trace_ready = probe_trace_ready,
+	.print_stack = probe_print_stack,
 	.analyzer = &probe_analyzer,
 };