diff --git a/kernel/trace/trace_eprobe.c b/kernel/trace/trace_eprobe.c index 5345705cf84fba..6a96e3ce66976e 100644 --- a/kernel/trace/trace_eprobe.c +++ b/kernel/trace/trace_eprobe.c @@ -320,64 +320,6 @@ print_eprobe_event(struct trace_iterator *iter, int flags, return trace_handle_return(s); } -static unsigned long get_event_field(struct fetch_insn *code, void *rec) -{ - struct ftrace_event_field *field = code->data; - unsigned long val; - void *addr; - - addr = rec + field->offset; - - if (is_string_field(field)) { - switch (field->filter_type) { - case FILTER_DYN_STRING: - val = (unsigned long)(rec + (*(unsigned int *)addr & 0xffff)); - break; - case FILTER_RDYN_STRING: - val = (unsigned long)(addr + (*(unsigned int *)addr & 0xffff)); - break; - case FILTER_STATIC_STRING: - val = (unsigned long)addr; - break; - case FILTER_PTR_STRING: - val = (unsigned long)(*(char *)addr); - break; - default: - WARN_ON_ONCE(1); - return 0; - } - return val; - } - - switch (field->size) { - case 1: - if (field->is_signed) - val = *(char *)addr; - else - val = *(unsigned char *)addr; - break; - case 2: - if (field->is_signed) - val = *(short *)addr; - else - val = *(unsigned short *)addr; - break; - case 4: - if (field->is_signed) - val = *(int *)addr; - else - val = *(unsigned int *)addr; - break; - default: - if (field->is_signed) - val = *(long *)addr; - else - val = *(unsigned long *)addr; - break; - } - return val; -} - static int get_eprobe_size(struct trace_probe *tp, void *rec) { struct fetch_insn *code; @@ -420,92 +362,6 @@ static int get_eprobe_size(struct trace_probe *tp, void *rec) return ret; } -/* Kprobe specific fetch functions */ - -/* Note that we don't verify it, since the code does not come from user space */ -static int -process_fetch_insn(struct fetch_insn *code, void *rec, void *dest, - void *base) -{ - unsigned long val; - - retry: - switch (code->op) { - case FETCH_OP_TP_ARG: - val = get_event_field(code, rec); - break; - case FETCH_OP_IMM: - val = code->immediate; - break; - case FETCH_OP_COMM: - val = (unsigned long)current->comm; - break; - case FETCH_OP_DATA: - val = (unsigned long)code->data; - break; - case FETCH_NOP_SYMBOL: /* Ignore a place holder */ - code++; - goto retry; - default: - return -EILSEQ; - } - code++; - return process_fetch_insn_bottom(code, val, dest, base); -} -NOKPROBE_SYMBOL(process_fetch_insn) - -/* Return the length of string -- including null terminal byte */ -static nokprobe_inline int -fetch_store_strlen_user(unsigned long addr) -{ - return kern_fetch_store_strlen_user(addr); -} - -/* Return the length of string -- including null terminal byte */ -static nokprobe_inline int -fetch_store_strlen(unsigned long addr) -{ - return kern_fetch_store_strlen(addr); -} - -/* - * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf - * with max length and relative data location. - */ -static nokprobe_inline int -fetch_store_string_user(unsigned long addr, void *dest, void *base) -{ - return kern_fetch_store_string_user(addr, dest, base); -} - -/* - * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max - * length and relative data location. - */ -static nokprobe_inline int -fetch_store_string(unsigned long addr, void *dest, void *base) -{ - return kern_fetch_store_string(addr, dest, base); -} - -static nokprobe_inline int -probe_mem_read_user(void *dest, void *src, size_t size) -{ - const void __user *uaddr = (__force const void __user *)src; - - return copy_from_user_nofault(dest, uaddr, size); -} - -static nokprobe_inline int -probe_mem_read(void *dest, void *src, size_t size) -{ -#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE - if ((unsigned long)src < TASK_SIZE) - return probe_mem_read_user(dest, src, size); -#endif - return copy_from_kernel_nofault(dest, src, size); -} - /* eprobe handler */ static inline void __eprobe_trace_func(struct eprobe_data *edata, void *rec) diff --git a/kernel/trace/trace_events_synth.c b/kernel/trace/trace_events_synth.c index 29fbfb27c2b2cc..b980965a62de4a 100644 --- a/kernel/trace/trace_events_synth.c +++ b/kernel/trace/trace_events_synth.c @@ -18,6 +18,7 @@ #include #include #include "trace_probe.h" +#include "trace_probe_tmpl.h" #include "trace_probe_kernel.h" #include "trace_synth.h" @@ -420,12 +421,12 @@ static unsigned int trace_string(struct synth_trace_event *entry, data_offset += event->n_u64 * sizeof(u64); data_offset += data_size; - len = kern_fetch_store_strlen((unsigned long)str_val); + len = fetch_store_strlen((unsigned long)str_val); data_offset |= len << 16; *(u32 *)&entry->fields[*n_u64] = data_offset; - ret = kern_fetch_store_string((unsigned long)str_val, &entry->fields[*n_u64], entry); + ret = fetch_store_string((unsigned long)str_val, &entry->fields[*n_u64], entry); (*n_u64)++; } else { @@ -473,7 +474,7 @@ static notrace void trace_event_raw_event_synth(void *__data, val_idx = var_ref_idx[field_pos]; str_val = (char *)(long)var_ref_vals[val_idx]; - len = kern_fetch_store_strlen((unsigned long)str_val); + len = fetch_store_strlen((unsigned long)str_val); fields_size += len; } diff --git a/kernel/trace/trace_kprobe.c b/kernel/trace/trace_kprobe.c index a4ffa864dbb7a0..c2e0b741ae8264 100644 --- a/kernel/trace/trace_kprobe.c +++ b/kernel/trace/trace_kprobe.c @@ -1218,108 +1218,6 @@ static const struct file_operations kprobe_profile_ops = { .release = seq_release, }; -/* Kprobe specific fetch functions */ - -/* Return the length of string -- including null terminal byte */ -static nokprobe_inline int -fetch_store_strlen_user(unsigned long addr) -{ - return kern_fetch_store_strlen_user(addr); -} - -/* Return the length of string -- including null terminal byte */ -static nokprobe_inline int -fetch_store_strlen(unsigned long addr) -{ - return kern_fetch_store_strlen(addr); -} - -/* - * Fetch a null-terminated string from user. Caller MUST set *(u32 *)buf - * with max length and relative data location. - */ -static nokprobe_inline int -fetch_store_string_user(unsigned long addr, void *dest, void *base) -{ - return kern_fetch_store_string_user(addr, dest, base); -} - -/* - * Fetch a null-terminated string. Caller MUST set *(u32 *)buf with max - * length and relative data location. - */ -static nokprobe_inline int -fetch_store_string(unsigned long addr, void *dest, void *base) -{ - return kern_fetch_store_string(addr, dest, base); -} - -static nokprobe_inline int -probe_mem_read_user(void *dest, void *src, size_t size) -{ - const void __user *uaddr = (__force const void __user *)src; - - return copy_from_user_nofault(dest, uaddr, size); -} - -static nokprobe_inline int -probe_mem_read(void *dest, void *src, size_t size) -{ -#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE - if ((unsigned long)src < TASK_SIZE) - return probe_mem_read_user(dest, src, size); -#endif - return copy_from_kernel_nofault(dest, src, size); -} - -/* Note that we don't verify it, since the code does not come from user space */ -static int -process_fetch_insn(struct fetch_insn *code, void *rec, void *dest, - void *base) -{ - struct pt_regs *regs = rec; - unsigned long val; - -retry: - /* 1st stage: get value from context */ - switch (code->op) { - case FETCH_OP_REG: - val = regs_get_register(regs, code->param); - break; - case FETCH_OP_STACK: - val = regs_get_kernel_stack_nth(regs, code->param); - break; - case FETCH_OP_STACKP: - val = kernel_stack_pointer(regs); - break; - case FETCH_OP_RETVAL: - val = regs_return_value(regs); - break; - case FETCH_OP_IMM: - val = code->immediate; - break; - case FETCH_OP_COMM: - val = (unsigned long)current->comm; - break; - case FETCH_OP_DATA: - val = (unsigned long)code->data; - break; -#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API - case FETCH_OP_ARG: - val = regs_get_kernel_argument(regs, code->param); - break; -#endif - case FETCH_NOP_SYMBOL: /* Ignore a place holder */ - code++; - goto retry; - default: - return -EILSEQ; - } - code++; - - return process_fetch_insn_bottom(code, val, dest, base); -} -NOKPROBE_SYMBOL(process_fetch_insn) /* Kprobe handler */ static nokprobe_inline void diff --git a/kernel/trace/trace_probe_kernel.h b/kernel/trace/trace_probe_kernel.h index 77dbd9ff978268..cee6078b104384 100644 --- a/kernel/trace/trace_probe_kernel.h +++ b/kernel/trace/trace_probe_kernel.h @@ -12,7 +12,7 @@ */ /* Return the length of string -- including null terminal byte */ static nokprobe_inline int -kern_fetch_store_strlen_user(unsigned long addr) +fetch_store_strlen_user(unsigned long addr) { const void __user *uaddr = (__force const void __user *)addr; int ret; @@ -29,14 +29,14 @@ kern_fetch_store_strlen_user(unsigned long addr) /* Return the length of string -- including null terminal byte */ static nokprobe_inline int -kern_fetch_store_strlen(unsigned long addr) +fetch_store_strlen(unsigned long addr) { int ret, len = 0; u8 c; #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE if (addr < TASK_SIZE) - return kern_fetch_store_strlen_user(addr); + return fetch_store_strlen_user(addr); #endif do { @@ -63,7 +63,7 @@ static nokprobe_inline void set_data_loc(int ret, void *dest, void *__dest, void * with max length and relative data location. */ static nokprobe_inline int -kern_fetch_store_string_user(unsigned long addr, void *dest, void *base) +fetch_store_string_user(unsigned long addr, void *dest, void *base) { const void __user *uaddr = (__force const void __user *)addr; int maxlen = get_loc_len(*(u32 *)dest); @@ -86,7 +86,7 @@ kern_fetch_store_string_user(unsigned long addr, void *dest, void *base) * length and relative data location. */ static nokprobe_inline int -kern_fetch_store_string(unsigned long addr, void *dest, void *base) +fetch_store_string(unsigned long addr, void *dest, void *base) { int maxlen = get_loc_len(*(u32 *)dest); void *__dest; @@ -94,7 +94,7 @@ kern_fetch_store_string(unsigned long addr, void *dest, void *base) #ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE if ((unsigned long)addr < TASK_SIZE) - return kern_fetch_store_string_user(addr, dest, base); + return fetch_store_string_user(addr, dest, base); #endif if (unlikely(!maxlen)) @@ -112,4 +112,132 @@ kern_fetch_store_string(unsigned long addr, void *dest, void *base) return ret; } +static nokprobe_inline int +probe_mem_read_user(void *dest, void *src, size_t size) +{ + const void __user *uaddr = (__force const void __user *)src; + + return copy_from_user_nofault(dest, uaddr, size); +} + +static nokprobe_inline int +probe_mem_read(void *dest, void *src, size_t size) +{ +#ifdef CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE + if ((unsigned long)src < TASK_SIZE) + return probe_mem_read_user(dest, src, size); +#endif + return copy_from_kernel_nofault(dest, src, size); +} + +static unsigned long get_event_field(struct fetch_insn *code, void *rec) +{ + struct ftrace_event_field *field = code->data; + unsigned long val; + void *addr; + + addr = rec + field->offset; + + if (is_string_field(field)) { + switch (field->filter_type) { + case FILTER_DYN_STRING: + val = (unsigned long)(rec + (*(unsigned int *)addr & 0xffff)); + break; + case FILTER_RDYN_STRING: + val = (unsigned long)(addr + (*(unsigned int *)addr & 0xffff)); + break; + case FILTER_STATIC_STRING: + val = (unsigned long)addr; + break; + case FILTER_PTR_STRING: + val = (unsigned long)(*(char *)addr); + break; + default: + WARN_ON_ONCE(1); + return 0; + } + return val; + } + + switch (field->size) { + case 1: + if (field->is_signed) + val = *(char *)addr; + else + val = *(unsigned char *)addr; + break; + case 2: + if (field->is_signed) + val = *(short *)addr; + else + val = *(unsigned short *)addr; + break; + case 4: + if (field->is_signed) + val = *(int *)addr; + else + val = *(unsigned int *)addr; + break; + default: + if (field->is_signed) + val = *(long *)addr; + else + val = *(unsigned long *)addr; + break; + } + return val; +} + +/* Note that we don't verify it, since the code does not come from user space */ +static int +process_fetch_insn(struct fetch_insn *code, void *rec, void *dest, + void *base) +{ + struct pt_regs *regs = rec; + unsigned long val; + +retry: + /* 1st stage: get value from context */ + switch (code->op) { + case FETCH_OP_REG: + val = regs_get_register(regs, code->param); + break; + case FETCH_OP_STACK: + val = regs_get_kernel_stack_nth(regs, code->param); + break; + case FETCH_OP_STACKP: + val = kernel_stack_pointer(regs); + break; + case FETCH_OP_RETVAL: + val = regs_return_value(regs); + break; + case FETCH_OP_IMM: + val = code->immediate; + break; + case FETCH_OP_COMM: + val = (unsigned long)current->comm; + break; + case FETCH_OP_DATA: + val = (unsigned long)code->data; + break; +#ifdef CONFIG_HAVE_FUNCTION_ARG_ACCESS_API + case FETCH_OP_ARG: + val = regs_get_kernel_argument(regs, code->param); + break; +#endif + case FETCH_NOP_SYMBOL: /* Ignore a place holder */ + code++; + goto retry; + case FETCH_OP_TP_ARG: + val = get_event_field(code, rec); + break; + default: + return -EILSEQ; + } + code++; + + return process_fetch_insn_bottom(code, val, dest, base); +} +NOKPROBE_SYMBOL(process_fetch_insn) + #endif /* __TRACE_PROBE_KERNEL_H_ */