retstack: Split extra return stack from function graph tracer

mhiramat · Oct 26, 2017 · 8804f76 · 8804f76
1 parent eb9b5a7
commit 8804f76
Show file tree

Hide file tree

Showing 10 changed files with 416 additions and 249 deletions.
diff --git a/arch/Kconfig b/arch/Kconfig
@@ -169,6 +169,13 @@ config ARCH_USE_BUILTIN_BSWAP
 	 instructions should set this. And it shouldn't hurt to set it
 	 on architectures that don't have such instructions.
 
+
+config TASK_RET_STACK
+	bool
+	help
+	  Provide per-thread in-kernel extra return stack for function
+	  return hooker (e.g. function graph tracer and kretprobes).
+
 config KRETPROBES
 	def_bool y
 	depends on KPROBES && HAVE_KRETPROBES

diff --git a/arch/x86/kernel/ftrace.c b/arch/x86/kernel/ftrace.c
@@ -1046,7 +1046,7 @@ void prepare_ftrace_return(unsigned long self_addr, unsigned long *parent,
 	}
 
 	trace.func = self_addr;
-	trace.depth = current->curr_ret_stack + 1;
+	trace.depth = task_ftrace_graph_depth(current) + 1;
 
 	/* Only trace if the calling function expects to */
 	if (!ftrace_graph_entry(&trace)) {

diff --git a/include/linux/ftrace.h b/include/linux/ftrace.h
@@ -853,9 +853,9 @@ extern void ftrace_graph_init_task(struct task_struct *t);
 extern void ftrace_graph_exit_task(struct task_struct *t);
 extern void ftrace_graph_init_idle_task(struct task_struct *t, int cpu);
 
-static inline int task_curr_ret_stack(struct task_struct *t)
+static inline int task_ftrace_graph_depth(struct task_struct *t)
 {
-	return t->curr_ret_stack;
+	return t->ftrace_graph_depth;
 }
 
 static inline void pause_graph_tracing(void)
@@ -883,7 +883,7 @@ static inline int register_ftrace_graph(trace_func_graph_ret_t retfunc,
 }
 static inline void unregister_ftrace_graph(void) { }
 
-static inline int task_curr_ret_stack(struct task_struct *tsk)
+static inline int task_ftrace_graph_depth(struct task_struct *tsk)
 {
 	return -1;
 }

diff --git a/include/linux/retstack.h b/include/linux/retstack.h
@@ -0,0 +1,27 @@
+#ifndef _LINUX_RETSTACK_H
+#define _LINUX_RETSTACK_H
+
+/*
+ * Extra return stack for function return hook
+ */
+
+#define RETSTACK_ALLOC_SIZE 32
+#define RETSTACK_MAX_DEPTH 50
+
+void retstack_abort(void);
+bool retstack_is_dead(void);
+
+int retstack_init(void);
+int retstack_exit(void);
+void retstack_init_idle(struct task_struct *t, int cpu);
+void retstack_init_task(struct task_struct *t);
+void retstack_exit_task(struct task_struct *t);
+
+int retstack_push(unsigned long ret, unsigned long func, unsigned long *retp,
+		  struct retstack **entry);
+int retstack_peek(struct retstack **entry);
+int retstack_pop(unsigned long *ret, unsigned long *func);
+unsigned long retstack_ret_addr(struct task_struct *task, int *idx,
+				unsigned long ret, unsigned long *retp);
+
+#endif
diff --git a/include/linux/sched.h b/include/linux/sched.h
@@ -1020,21 +1020,25 @@ struct task_struct {
 	unsigned int			kasan_depth;
 #endif
 
-#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+#ifdef CONFIG_TASK_RETURN_STACK
 	/* Index of current stored address in ret_stack: */
 	int				curr_ret_stack;
 
 	/* Stack of return addresses for return function tracing: */
 	struct ftrace_ret_stack		*ret_stack;
 
-	/* Timestamp for last schedule: */
-	unsigned long long		ftrace_timestamp;
-
 	/*
 	 * Number of functions that haven't been traced
 	 * because of depth overrun:
 	 */
 	atomic_t			trace_overrun;
+#endif
+#ifdef CONFIG_FUNCTION_GRAPH_TRACER
+	/* Index of current ftrace graph tracing ret_stack: */
+	int				ftrace_graph_depth;
+
+	/* Timestamp for last schedule: */
+	unsigned long long		ftrace_timestamp;
 
 	/* Pause tracing: */
 	atomic_t			tracing_graph_pause;

diff --git a/kernel/Makefile b/kernel/Makefile
@@ -100,6 +100,7 @@ obj-$(CONFIG_TRACEPOINTS) += trace/
 obj-$(CONFIG_IRQ_WORK) += irq_work.o
 obj-$(CONFIG_CPU_PM) += cpu_pm.o
 obj-$(CONFIG_BPF) += bpf/
+obj-$(CONFIG_TASK_RETURN_STACK) += retstack.o
 
 obj-$(CONFIG_PERF_EVENTS) += events/