diff --git a/pkg/report/linux.go b/pkg/report/linux.go index 0513ffcccbc..182cd645e83 100644 --- a/pkg/report/linux.go +++ b/pkg/report/linux.go @@ -504,6 +504,103 @@ func (ctx *linux) isCorrupted(title string, report []byte, format oopsFormat) (b return false, "" } +func linuxStallFrameExtractor(frames []string) (string, string) { + // During rcu stalls and cpu lockups kernel loops in some part of code, + // usually across several functions. When the stall is detected, traceback + // points to a random stack within the looping code. We generally take + // the top function in the stack (with few exceptions) as the bug identity. + // As the result stalls with the same root would produce multiple reports + // in different functions, which is bad. + // Instead we identify a representative function deeper in the stack. + // For most syscalls it can be the syscall entry function (e.g. SyS_timer_create). + // However, for highly discriminated functions syscalls like ioctl/read/write/connect + // we take the previous function (e.g. for connect the one that points to exact + // protocol, or for ioctl the one that is related to the device). + prev := frames[0] + for _, frame := range frames { + if matchesAny([]byte(frame), linuxStallAnchorFrames) { + for _, prefix := range []string{ + "__x64_", + "SYSC_", + "SyS_", + "compat_SYSC_", + "compat_SyS_", + } { + prev = strings.TrimPrefix(prev, prefix) + } + return prev, "" + } + prev = frame + } + return "", "did not find any anchor frame" +} + +var linuxStallAnchorFrames = []*regexp.Regexp{ + // Various generic functions that dispatch work. + // We also include some of their callers, so that if some names change + // we don't skip whole stacks and proceed parsing the next one. + compile("process_one_work"), // workqueue callback + compile("do_syscall_"), // syscall entry + compile("do_fast_syscall_"), // syscall entry + compile("netif_receive_skb"), // net receive entry point + compile("do_softirq"), + compile("call_timer_fn"), + compile("_run_timers"), + compile("run_timer_softirq"), + compile("run_ksoftirqd"), + compile("smpboot_thread_fn"), + compile("kthread"), + compile("start_secondary"), + compile("cpu_startup_entry"), + compile("ret_from_fork"), + // Important discriminated syscalls (file_operations callbacks, etc): + compile("vfs_write"), + compile("vfs_read"), + compile("vfs_iter_read"), + compile("vfs_iter_write"), + compile("do_iter_read"), + compile("do_iter_write"), + compile("vfs_ioctl"), + compile("blkdev_driver_ioctl"), + compile("blkdev_ioctl"), + compile("^call_read_iter"), + compile("^call_write_iter"), + compile("do_iter_readv_writev"), + compile("^call_mmap"), + compile("mmap_region"), + compile("do_mmap"), + compile("do_dentry_open"), + compile("vfs_open"), + // Socket operations: + compile("^sock_sendmsg"), + compile("^sock_recvmsg"), + compile("^sock_release"), + compile("^__sock_release"), + compile("__sys_setsockopt"), + compile("kernel_setsockopt"), + compile("sock_common_setsockopt"), + compile("__sys_listen"), + compile("kernel_listen"), + compile("sk_common_release"), + compile("^sock_mmap"), + compile("__sys_accept"), + compile("kernel_accept"), + compile("^sock_do_ioctl"), + compile("^sock_ioctl"), + compile("^compat_sock_ioctl"), + compile("^(compat_)?(SYSC|SyS|__sys|___sys|__do_sys|__se_sys|__x64_sys)_socketpair"), + compile("^(compat_)?(SYSC|SyS|__sys|___sys|__do_sys|__se_sys|__x64_sys)_connect"), + // Page fault entry points: + compile("__do_fault"), + compile("handle_mm_fault"), + compile("do_page_fault"), + compile("^page_fault$"), + // exit_to_usermode_loop callbacks: + compile("__fput"), + compile("task_work_run"), + compile("exit_to_usermode"), +} + var ( linuxSymbolizeRe = regexp.MustCompile(`(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`) stackFrameRe = regexp.MustCompile(`^ *(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`) @@ -697,6 +794,7 @@ var linuxOopses = []*oops{ compile("Call Trace:"), parseStackTrace, }, + extractor: linuxStallFrameExtractor, }, }, { @@ -963,18 +1061,8 @@ var linuxOopses = []*oops{ compile("apic_timer_interrupt"), parseStackTrace, }, - skip: []string{"apic_timer_interrupt", "rcu"}, - }, - }, - { - title: linuxRcuStall, - fmt: "INFO: rcu detected stall in %[1]v", - stack: &stackFmt{ - parts: []*regexp.Regexp{ - linuxRipFrame, - parseStackTrace, - }, - skip: []string{"apic_timer_interrupt", "rcu"}, + skip: []string{"apic_timer_interrupt", "rcu"}, + extractor: linuxStallFrameExtractor, }, }, { diff --git a/pkg/report/report.go b/pkg/report/report.go index ce953c8a16d..27a71e93753 100644 --- a/pkg/report/report.go +++ b/pkg/report/report.go @@ -234,8 +234,13 @@ type stackFmt struct { parts2 []*regexp.Regexp // Skip these functions in stack traces (matched as substring). skip []string + // Custom frame extractor (optional). + // Accepts set of all frames, returns guilty frame and corruption reason. + extractor frameExtractor } +type frameExtractor func(frames []string) (frame string, corrupted string) + var parseStackTrace *regexp.Regexp func compile(re string) *regexp.Regexp { @@ -367,24 +372,30 @@ func extractStackFrame(params *stackParams, stack *stackFmt, output []byte) (str if len(skip) != 0 { skipRe = regexp.MustCompile(strings.Join(skip, "|")) } - frame, corrupted := extractStackFrameImpl(params, output, skipRe, stack.parts) + extractor := stack.extractor + if extractor == nil { + extractor = func(frames []string) (string, string) { + return frames[0], "" + } + } + frame, corrupted := extractStackFrameImpl(params, output, skipRe, stack.parts, extractor) if frame != "" || len(stack.parts2) == 0 { return frame, corrupted } - return extractStackFrameImpl(params, output, skipRe, stack.parts2) + return extractStackFrameImpl(params, output, skipRe, stack.parts2, extractor) } func extractStackFrameImpl(params *stackParams, output []byte, skipRe *regexp.Regexp, - parts []*regexp.Regexp) (string, string) { - corrupted := "" + parts []*regexp.Regexp, extractor frameExtractor) (string, string) { s := bufio.NewScanner(bytes.NewReader(output)) + var frames []string nextPart: for _, part := range parts { if part == parseStackTrace { for s.Scan() { ln := bytes.Trim(s.Bytes(), "\r") - if corrupted == "" && matchesAny(ln, params.corruptedLines) { - corrupted = "corrupted line in report (1)" + if matchesAny(ln, params.corruptedLines) { + break nextPart } if matchesAny(ln, params.stackStartRes) { continue nextPart @@ -401,14 +412,14 @@ nextPart: } frame := ln[match[2]:match[3]] if skipRe == nil || !skipRe.Match(frame) { - return string(frame), corrupted + frames = append(frames, string(frame)) } } } else { for s.Scan() { ln := bytes.Trim(s.Bytes(), "\r") - if corrupted == "" && matchesAny(ln, params.corruptedLines) { - corrupted = "corrupted line in report (2)" + if matchesAny(ln, params.corruptedLines) { + break nextPart } match := part.FindSubmatchIndex(ln) if match == nil { @@ -417,14 +428,17 @@ nextPart: if len(match) == 4 && match[2] != -1 { frame := ln[match[2]:match[3]] if skipRe == nil || !skipRe.Match(frame) { - return string(frame), corrupted + frames = append(frames, string(frame)) } } break } } } - return "", corrupted + if len(frames) == 0 { + return "", "extracted no frames" + } + return extractor(frames) } func simpleLineParser(output []byte, oopses []*oops, params *stackParams, ignores []*regexp.Regexp) *Report { diff --git a/pkg/report/testdata/linux/report/145 b/pkg/report/testdata/linux/report/145 index 6a7348bf775..72f3df49926 100644 --- a/pkg/report/testdata/linux/report/145 +++ b/pkg/report/testdata/linux/report/145 @@ -1,4 +1,4 @@ -TITLE: INFO: rcu detected stall in __xfrm_decode_session +TITLE: INFO: rcu detected stall in ipv6_rcv [ 323.830017] INFO: rcu_sched detected stalls on CPUs/tasks: [ 323.835676] (detected by 0, t=125007 jiffies, g=66299, c=66298, q=40) @@ -7,74 +7,15 @@ TITLE: INFO: rcu detected stall in __xfrm_decode_session [ 323.861597] Call Trace: [ 323.864155] [ 323.866280] sched_show_task+0x4a3/0x5e0 -[ 323.870319] ? can_nice.part.83+0x20/0x20 -[ 323.874437] ? vprintk_func+0x5e/0xc0 -[ 323.878207] ? printk+0xaa/0xca -[ 323.881459] ? show_regs_print_info+0x18/0x18 -[ 323.885931] ? show_regs_print_info+0x18/0x18 [ 323.890409] print_other_cpu_stall+0x996/0x1090 -[ 323.895047] ? print_irqtrace_events+0x270/0x270 -[ 323.899771] ? print_irqtrace_events+0x270/0x270 -[ 323.904510] ? rcu_stall_kick_kthreads.part.48+0x770/0x770 -[ 323.910106] ? check_noncircular+0x20/0x20 -[ 323.914309] ? print_irqtrace_events+0x270/0x270 -[ 323.919041] ? print_irqtrace_events+0x270/0x270 -[ 323.923766] ? __lock_acquire+0x664/0x3e00 -[ 323.927980] ? check_noncircular+0x20/0x20 -[ 323.932190] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 323.937352] ? __lock_acquire+0x664/0x3e00 -[ 323.941558] ? rcu_read_lock_sched_held+0x108/0x120 -[ 323.946544] ? update_cfs_rq_load_avg.part.69+0x23d/0x2d0 -[ 323.952048] ? attach_entity_load_avg+0x7a0/0x7a0 [ 323.956870] check_cpu_stall.isra.61+0x6e6/0x15b0 -[ 323.961682] ? __lock_acquire+0x664/0x3e00 -[ 323.965892] ? rcu_lockdep_current_cpu_online+0x190/0x190 -[ 323.971398] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 323.976563] ? __lock_acquire+0x664/0x3e00 -[ 323.980767] ? __lock_acquire+0x664/0x3e00 -[ 323.984984] ? check_noncircular+0x20/0x20 -[ 323.989196] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 323.994352] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 323.999518] ? lock_downgrade+0x980/0x980 -[ 324.003646] ? trace_hardirqs_off+0xd/0x10 -[ 324.007852] ? check_noncircular+0x20/0x20 -[ 324.012063] ? check_noncircular+0x20/0x20 -[ 324.016277] ? cpuacct_account_field+0x1e4/0x3b0 -[ 324.021015] ? check_noncircular+0x20/0x20 -[ 324.025220] ? find_held_lock+0x35/0x1d0 -[ 324.029251] ? __acct_update_integrals+0x339/0x4d0 -[ 324.034154] ? __lock_is_held+0xb6/0x140 [ 324.038202] rcu_check_callbacks+0x256/0xd00 -[ 324.042585] ? rcu_cpu_stall_reset+0x260/0x260 -[ 324.047139] ? trace_hardirqs_off+0xd/0x10 -[ 324.051344] ? raise_softirq+0x325/0x490 -[ 324.055381] ? raise_softirq_irqoff+0x320/0x320 -[ 324.060042] ? run_local_timers+0x18d/0x200 -[ 324.064331] ? timer_clear_idle+0x50/0x50 -[ 324.068446] ? account_process_tick+0xd4/0x3e0 -[ 324.072999] ? thread_group_cputime+0xef0/0xef0 -[ 324.077641] ? ktime_get_resolution_ns+0x300/0x300 [ 324.082553] update_process_times+0x30/0x60 [ 324.086846] tick_sched_handle+0x85/0x160 [ 324.090968] tick_sched_timer+0x42/0x120 [ 324.094999] __hrtimer_run_queues+0x358/0xe20 -[ 324.099472] ? tick_sched_do_timer+0xe0/0xe0 -[ 324.103855] ? hrtimer_fixup_init+0x70/0x70 -[ 324.108155] ? pvclock_read_flags+0x160/0x160 -[ 324.112624] ? lock_acquire+0x1d5/0x580 -[ 324.116570] ? kvm_clock_get_cycles+0x25/0x30 -[ 324.121034] ? ktime_get_update_offsets_now+0x34a/0x520 -[ 324.126371] ? do_timer+0x50/0x50 -[ 324.129800] ? do_raw_spin_trylock+0x190/0x190 [ 324.134365] hrtimer_interrupt+0x1c2/0x5e0 -[ 324.172031] ? apic_timer_interrupt+0xa9/0xb0 [ 324.138588] smp_apic_timer_interrupt+0x14a/0x700 -[ 324.143401] ? smp_call_function_single_interrupt+0x640/0x640 -[ 324.149252] ? smp_call_function_single_interrupt+0x640/0x640 -[ 324.155108] ? _raw_spin_unlock+0x22/0x30 -[ 324.159231] ? mark_held_locks+0xaf/0x100 -[ 324.163349] ? retint_kernel+0x10/0x10 -[ 324.167210] ? trace_hardirqs_off_thunk+0x1a/0x1c [ 324.172031] apic_timer_interrupt+0xa9/0xb0 [ 324.176323] RIP: 0010:__sanitizer_cov_trace_pc+0x0/0x50 [ 324.181650] RSP: 0018:ffff8801db206738 EFLAGS: 00000206 ORIG_RAX: ffffffffffffff11 @@ -83,193 +24,34 @@ TITLE: INFO: rcu detected stall in __xfrm_decode_session [ 324.203803] RBP: ffff8801db2067c8 R08: ffffed0038bea003 R09: ffffed0038bea003 [ 324.211046] R10: 000000000000000b R11: ffffed0038bea002 R12: ffff8801c5f40580 [ 324.218304] R13: ffff8801db206aa0 R14: 00000000000079d0 R15: dffffc0000000000 -[ 324.225565] ? _decode_session6+0x519/0x1380 -[ 324.229954] ? _decode_session6+0x761/0x1380 [ 324.234345] __xfrm_decode_session+0x68/0x110 [ 324.238814] __xfrm_policy_check+0x18c/0x2350 -[ 324.243279] ? save_stack+0xa3/0xd0 -[ 324.246876] ? save_stack+0x43/0xd0 -[ 324.250469] ? kasan_kmalloc+0xad/0xe0 -[ 324.254322] ? kasan_slab_alloc+0x12/0x20 -[ 324.258435] ? kmem_cache_alloc+0x12e/0x760 -[ 324.262723] ? raw6_local_deliver+0x737/0xa80 -[ 324.267185] ? ip6_input_finish+0x3c7/0x17a0 -[ 324.271564] ? ipv6_rcv+0xf37/0x1fa0 -[ 324.275249] ? __netif_receive_skb_core+0x1a41/0x3460 -[ 324.280418] ? __xfrm_route_forward+0x670/0x670 -[ 324.285054] ? ip6_finish_output2+0xba0/0x23a0 -[ 324.289605] ? ip6_finish_output+0x6bb/0xaf0 -[ 324.293980] ? ip6_output+0x1eb/0x840 -[ 324.297748] ? ip6_local_out+0x95/0x160 -[ 324.301692] ? check_noncircular+0x20/0x20 -[ 324.305895] ? inet_sendmsg+0x11f/0x5e0 -[ 324.309836] ? sock_sendmsg+0xca/0x110 -[ 324.313694] ? SYSC_sendto+0x361/0x5c0 -[ 324.317547] ? SyS_sendto+0x40/0x50 -[ 324.321142] ? entry_SYSCALL_64_fastpath+0x23/0x9a -[ 324.326041] ? SyS_sendto+0x40/0x50 -[ 324.329636] ? entry_SYSCALL_64_fastpath+0x23/0x9a -[ 324.334536] ? free_obj_work+0x690/0x690 -[ 324.338567] ? debug_object_free+0x5a0/0x5a0 -[ 324.342948] ? memcpy+0x45/0x50 -[ 324.346199] ? __copy_skb_header+0x31f/0x540 -[ 324.350576] ? skb_may_tx_timestamp.part.62+0xc0/0xc0 -[ 324.355736] ? __lock_is_held+0xb6/0x140 -[ 324.359775] ? __skb_clone+0x2ac/0xa70 [ 324.363656] rawv6_rcv+0x8f6/0x1200 -[ 324.367262] ? raw6_icmp_error+0x740/0x740 -[ 324.371463] ? __raw_v6_lookup+0x29f/0x390 [ 324.375676] raw6_local_deliver+0x819/0xa80 -[ 324.379977] ? rawv6_rcv+0x1200/0x1200 -[ 324.383833] ? ip6_rcv_finish+0x7a0/0x7a0 -[ 324.387950] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 324.393106] ? lock_downgrade+0x980/0x980 -[ 324.397227] ? lock_release+0xa40/0xa40 -[ 324.401174] ? __lock_is_held+0xb6/0x140 [ 324.405216] ip6_input_finish+0x3c7/0x17a0 -[ 324.409418] ? ip6_input+0x3b4/0x560 -[ 324.413102] ? lock_downgrade+0x980/0x980 -[ 324.417231] ? ip6_rcv_finish+0x7a0/0x7a0 -[ 324.421351] ? nf_hook_slow+0xd3/0x1a0 [ 324.425219] ip6_input+0xe9/0x560 -[ 324.428645] ? ip6_input_finish+0x17a0/0x17a0 -[ 324.433114] ? find_held_lock+0x35/0x1d0 -[ 324.437149] ? ip6_rcv_finish+0x7a0/0x7a0 -[ 324.441267] ? ipv6_rcv+0x16cd/0x1fa0 [ 324.445039] ip6_rcv_finish+0x1a9/0x7a0 -[ 324.448985] ? ip6_make_skb+0x580/0x580 -[ 324.452931] ? __lock_is_held+0xb6/0x140 -[ 324.456967] ? nf_hook_slow+0xd3/0x1a0 [ 324.460829] ipv6_rcv+0xf37/0x1fa0 -[ 324.464348] ? ip6_input+0x560/0x560 -[ 324.468032] ? print_irqtrace_events+0x270/0x270 -[ 324.472762] ? rb_insert_color+0x1580/0x1580 -[ 324.477144] ? check_noncircular+0x20/0x20 -[ 324.481344] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 324.486500] ? __lock_acquire+0x664/0x3e00 -[ 324.490717] ? ip6_make_skb+0x580/0x580 -[ 324.494665] ? ip6_input+0x560/0x560 [ 324.498351] __netif_receive_skb_core+0x1a41/0x3460 -[ 324.503336] ? find_held_lock+0x35/0x1d0 -[ 324.507375] ? nf_ingress+0x9f0/0x9f0 -[ 324.511145] ? lock_downgrade+0x980/0x980 -[ 324.515285] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 324.520456] ? update_curr+0x47b/0xa60 -[ 324.524323] ? __lock_acquire+0x664/0x3e00 -[ 324.528527] ? numa_migrate_preferred+0x250/0x250 -[ 324.533347] ? _find_next_bit+0xee/0x120 -[ 324.537386] ? account_entity_enqueue+0x3c8/0x6e0 -[ 324.542205] ? check_noncircular+0x20/0x20 -[ 324.546411] ? check_noncircular+0x20/0x20 -[ 324.550619] ? enqueue_task_fair+0x3b7/0x2950 -[ 324.555083] ? find_held_lock+0x35/0x1d0 -[ 324.559118] ? check_preempt_wakeup+0xb20/0xb20 -[ 324.563765] ? find_held_lock+0x35/0x1d0 -[ 324.567803] ? lock_acquire+0x1d5/0x580 -[ 324.571756] ? process_backlog+0x45f/0x740 -[ 324.575958] ? lock_acquire+0x1d5/0x580 -[ 324.579901] ? process_backlog+0x1ab/0x740 -[ 324.584111] ? lock_release+0xa40/0xa40 [ 324.588071] __netif_receive_skb+0x2c/0x1b0 -[ 324.592361] ? __netif_receive_skb+0x2c/0x1b0 [ 324.596827] process_backlog+0x203/0x740 -[ 324.600854] ? mark_held_locks+0xaf/0x100 [ 324.604982] net_rx_action+0x792/0x1910 -[ 324.608940] ? napi_complete_done+0x6c0/0x6c0 -[ 324.613407] ? trace_hardirqs_off+0xd/0x10 -[ 324.617611] ? _raw_spin_unlock_irqrestore+0xa6/0xba -[ 324.622684] ? try_to_wake_up+0xf9/0x1600 -[ 324.626801] ? lock_downgrade+0x980/0x980 -[ 324.630925] ? migrate_swap_stop+0x970/0x970 -[ 324.635304] ? check_noncircular+0x20/0x20 -[ 324.639512] ? check_noncircular+0x20/0x20 -[ 324.643721] ? lock_acquire+0x1d5/0x580 -[ 324.647665] ? lock_acquire+0x1d5/0x580 -[ 324.651610] ? __hrtimer_run_queues+0x394/0xe20 -[ 324.656247] ? lock_downgrade+0x980/0x980 -[ 324.660367] ? lock_release+0xa40/0xa40 -[ 324.664322] ? find_held_lock+0x35/0x1d0 -[ 324.668363] ? clockevents_program_event+0x163/0x2e0 -[ 324.673435] ? lock_downgrade+0x980/0x980 -[ 324.677564] ? pvclock_read_flags+0x160/0x160 -[ 324.682041] ? check_noncircular+0x20/0x20 -[ 324.686245] ? print_irqtrace_events+0x270/0x270 -[ 324.690972] ? ktime_get_resolution_ns+0x300/0x300 -[ 324.695866] ? lock_downgrade+0x980/0x980 -[ 324.699984] ? check_noncircular+0x20/0x20 -[ 324.704187] ? do_timer+0x50/0x50 -[ 324.707615] ? __lock_is_held+0xb6/0x140 [ 324.711663] __do_softirq+0x2d7/0xb85 -[ 324.715432] ? task_prio+0x40/0x40 -[ 324.718949] ? __irqentry_text_end+0x4/0x4 -[ 324.723152] ? irq_exit+0xbb/0x200 -[ 324.726660] ? smp_apic_timer_interrupt+0x16b/0x700 -[ 324.731646] ? smp_reschedule_interrupt+0xe6/0x670 -[ 324.736543] ? smp_call_function_single_interrupt+0x640/0x640 -[ 324.742400] ? _raw_spin_lock+0x32/0x40 -[ 324.746356] ? _raw_spin_unlock+0x22/0x30 -[ 324.750474] ? handle_edge_irq+0x2b4/0x7c0 -[ 324.754681] ? task_prio+0x40/0x40 -[ 324.758199] ? trace_hardirqs_off_thunk+0x1a/0x1c [ 324.763025] do_softirq_own_stack+0x2a/0x40 [ 324.767312] [ 324.769522] do_softirq.part.21+0x14d/0x190 -[ 324.773813] ? ip6_finish_output2+0xb6d/0x23a0 [ 324.778364] __local_bh_enable_ip+0x1ee/0x230 [ 324.782830] ip6_finish_output2+0xba0/0x23a0 -[ 324.787213] ? print_irqtrace_events+0x270/0x270 -[ 324.791951] ? ip6_sk_dst_lookup_flow+0x7f0/0x7f0 -[ 324.796764] ? prandom_u32+0x24/0x30 -[ 324.800447] ? ip_idents_reserve+0x1e6/0x2a0 -[ 324.804828] ? ipv4_sysctl_rtcache_flush+0x70/0x70 -[ 324.809732] ? __lock_is_held+0xb6/0x140 -[ 324.813774] ? rcu_read_lock_sched_held+0x108/0x120 -[ 324.818758] ? ip6_copy_metadata+0x62a/0xad0 -[ 324.823132] ? __kmalloc_track_caller+0x46a/0x760 -[ 324.827944] ? dst_output+0x140/0x140 -[ 324.831715] ? memcpy+0x45/0x50 [ 324.834968] ip6_fragment+0x25f2/0x3470 -[ 324.838912] ? ip6_fragment+0x25f2/0x3470 -[ 324.843043] ? ip6_sk_dst_lookup_flow+0x7f0/0x7f0 -[ 324.847864] ? ip6_forward+0x33d0/0x33d0 -[ 324.851902] ? ip6_mtu+0x1c7/0x4d0 -[ 324.855410] ? ip6_dst_ifdown+0x3d0/0x3d0 [ 324.859537] ip6_finish_output+0x6bb/0xaf0 [ 324.863753] ip6_output+0x1eb/0x840 -[ 324.867362] ? ip6_finish_output+0xaf0/0xaf0 -[ 324.871740] ? lock_acquire+0x1d5/0x580 -[ 324.875683] ? rawv6_sendmsg+0x1d74/0x3e70 -[ 324.879892] ? ip6_fragment+0x3470/0x3470 [ 324.884025] ip6_local_out+0x95/0x160 [ 324.887802] ip6_send_skb+0xa1/0x330 [ 324.891487] ip6_push_pending_frames+0xb3/0xe0 [ 324.896041] rawv6_sendmsg+0x2ee9/0x3e70 -[ 324.900090] ? rawv6_bind+0x8c0/0x8c0 -[ 324.903870] ? avc_has_perm+0x35e/0x680 -[ 324.907814] ? lock_downgrade+0x980/0x980 -[ 324.911934] ? lock_release+0xa40/0xa40 -[ 324.915876] ? __handle_mm_fault+0x3ce0/0x3ce0 -[ 324.920436] ? up_read+0x1a/0x40 -[ 324.923776] ? avc_has_perm+0x43e/0x680 -[ 324.927724] ? avc_has_perm_noaudit+0x520/0x520 -[ 324.932372] ? do_page_fault+0xee/0x720 -[ 324.936313] ? iterate_fd+0x3f0/0x3f0 -[ 324.940091] ? find_held_lock+0x35/0x1d0 [ 324.944155] inet_sendmsg+0x11f/0x5e0 -[ 324.947922] ? inet_sendmsg+0x11f/0x5e0 -[ 324.951869] ? inet_recvmsg+0x5f0/0x5f0 -[ 324.955816] ? selinux_socket_sendmsg+0x36/0x40 -[ 324.960454] ? security_socket_sendmsg+0x89/0xb0 -[ 324.965181] ? inet_recvmsg+0x5f0/0x5f0 [ 324.969127] sock_sendmsg+0xca/0x110 [ 324.972812] SYSC_sendto+0x361/0x5c0 -[ 324.976502] ? SYSC_connect+0x4a0/0x4a0 -[ 324.980454] ? __might_sleep+0x95/0x190 -[ 324.984399] ? exit_to_usermode_loop+0x8c/0x310 -[ 324.989049] ? trace_event_raw_event_sys_exit+0x260/0x260 -[ 324.994566] ? syscall_return_slowpath+0x2ad/0x550 -[ 324.999464] ? prepare_exit_to_usermode+0x340/0x340 -[ 325.004450] ? entry_SYSCALL_64_fastpath+0x5/0x9a [ 325.009270] SyS_sendto+0x40/0x50 [ 325.012699] entry_SYSCALL_64_fastpath+0x23/0x9a [ 325.017423] RIP: 0033:0x452ac9 diff --git a/pkg/report/testdata/linux/report/171 b/pkg/report/testdata/linux/report/171 index cceb2a3f051..089735f0d8c 100644 --- a/pkg/report/testdata/linux/report/171 +++ b/pkg/report/testdata/linux/report/171 @@ -1,4 +1,4 @@ -TITLE: INFO: rcu detected stall in mulaw_decode +TITLE: INFO: rcu detected stall in snd_pcm_oss_write [ 196.114026] INFO: rcu_sched detected stalls on CPUs/tasks: [ 196.119690] (detected by 0, t=125002 jiffies, g=15654, c=15653, q=133) @@ -7,75 +7,15 @@ TITLE: INFO: rcu detected stall in mulaw_decode [ 196.145712] Call Trace: [ 196.148271] [ 196.150403] sched_show_task+0x4a3/0x5e0 -[ 196.154447] ? can_nice.part.83+0x20/0x20 -[ 196.158577] ? vprintk_func+0x5e/0xc0 -[ 196.162354] ? printk+0xaa/0xca -[ 196.165610] ? show_regs_print_info+0x18/0x18 -[ 196.170080] ? show_regs_print_info+0x18/0x18 [ 196.174578] print_other_cpu_stall+0x996/0x1090 -[ 196.179228] ? __lock_is_held+0xb6/0x140 -[ 196.183282] ? rcu_stall_kick_kthreads.part.48+0x770/0x770 -[ 196.188879] ? check_noncircular+0x20/0x20 -[ 196.193092] ? print_irqtrace_events+0x270/0x270 -[ 196.197837] ? print_irqtrace_events+0x270/0x270 -[ 196.202579] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 196.207744] ? check_noncircular+0x20/0x20 -[ 196.211962] ? print_irqtrace_events+0x270/0x270 -[ 196.216697] ? print_irqtrace_events+0x270/0x270 -[ 196.221440] ? __lock_acquire+0x664/0x3e00 -[ 196.225662] ? rcu_read_lock_sched_held+0x108/0x120 -[ 196.230657] ? update_cfs_rq_load_avg.part.69+0x23d/0x2d0 [ 196.236196] check_cpu_stall.isra.61+0x6e6/0x15b0 -[ 196.241015] ? check_noncircular+0x20/0x20 -[ 196.245241] ? rcu_lockdep_current_cpu_online+0x190/0x190 -[ 196.250756] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 196.255942] ? find_held_lock+0x35/0x1d0 -[ 196.259996] ? check_noncircular+0x20/0x20 -[ 196.264213] ? lock_downgrade+0x980/0x980 -[ 196.268340] ? debug_check_no_locks_freed+0x3c0/0x3c0 -[ 196.273513] ? lock_release+0xa40/0xa40 -[ 196.277475] ? check_noncircular+0x20/0x20 -[ 196.281699] ? check_noncircular+0x20/0x20 -[ 196.285922] ? cpuacct_account_field+0x1e4/0x3b0 -[ 196.290662] ? check_noncircular+0x20/0x20 -[ 196.294870] ? find_held_lock+0x35/0x1d0 -[ 196.298911] ? __acct_update_integrals+0x339/0x4d0 -[ 196.303825] ? __lock_is_held+0xb6/0x140 [ 196.307884] rcu_check_callbacks+0x256/0xd00 -[ 196.312279] ? rcu_cpu_stall_reset+0x260/0x260 -[ 196.316843] ? trace_hardirqs_off+0xd/0x10 -[ 196.321054] ? raise_softirq+0x325/0x490 -[ 196.325095] ? raise_softirq_irqoff+0x320/0x320 -[ 196.329745] ? read_boot_clock64+0x70/0x70 -[ 196.333961] ? lock_downgrade+0x980/0x980 -[ 196.338085] ? lock_downgrade+0x980/0x980 -[ 196.342218] ? run_local_timers+0x18d/0x200 -[ 196.346515] ? timer_clear_idle+0x50/0x50 -[ 196.350644] ? account_process_tick+0xd4/0x3e0 -[ 196.355210] ? thread_group_cputime+0xef0/0xef0 -[ 196.359859] ? ktime_get_resolution_ns+0x300/0x300 [ 196.364779] update_process_times+0x30/0x60 [ 196.369080] tick_sched_handle+0x85/0x160 [ 196.373206] tick_sched_timer+0x42/0x120 [ 196.377249] __hrtimer_run_queues+0x358/0xe20 -[ 196.381733] ? tick_sched_do_timer+0xe0/0xe0 -[ 196.386122] ? hrtimer_fixup_init+0x70/0x70 -[ 196.390435] ? pvclock_read_flags+0x160/0x160 -[ 196.394924] ? kvm_clock_get_cycles+0x25/0x30 -[ 196.399397] ? ktime_get_update_offsets_now+0x34a/0x520 -[ 196.404747] ? do_timer+0x50/0x50 -[ 196.408179] ? native_apic_msr_write+0x5c/0x80 -[ 196.412739] ? do_raw_spin_trylock+0x190/0x190 -[ 196.417301] ? lapic_next_event+0x54/0x80 -[ 196.421436] ? clockevents_program_event+0x108/0x2e0 [ 196.426526] hrtimer_interrupt+0x1c2/0x5e0 [ 196.430767] smp_apic_timer_interrupt+0x14a/0x700 -[ 196.435592] ? smp_call_function_single_interrupt+0x640/0x640 -[ 196.441455] ? _raw_spin_lock+0x32/0x40 -[ 196.445410] ? _raw_spin_unlock+0x22/0x30 -[ 196.449533] ? handle_edge_irq+0x2b4/0x7c0 -[ 196.453747] ? task_prio+0x40/0x40 -[ 196.457279] ? trace_hardirqs_off_thunk+0x1a/0x1c [ 196.462113] apic_timer_interrupt+0xa9/0xb0 [ 196.466405] [ 196.468619] RIP: 0010:check_memory_region+0x38/0x190 @@ -85,41 +25,15 @@ TITLE: INFO: rcu detected stall in mulaw_decode [ 196.495875] RBP: ffff8801bd2ff878 R08: ffffc9000160020a R09: dffffc0000000000 [ 196.503120] R10: 0000000000000001 R11: ffffed0037a5ff2e R12: ffffc9000160020a [ 196.510365] R13: ffff8801bd2ff970 R14: dffffc0000000000 R15: ffffc9000160020a -[ 196.517632] ? mulaw_decode+0x52f/0x770 -[ 196.521592] ? check_noncircular+0x20/0x20 [ 196.525806] memcpy+0x37/0x50 [ 196.528895] mulaw_decode+0x52f/0x770 -[ 196.532706] ? snd_pcm_plugin_build_linear+0x8a0/0x8a0 -[ 196.537960] ? snd_pcm_plugin_client_channels+0x17/0x80 -[ 196.543305] ? io_src_channels+0x10a/0x210 [ 196.547525] mulaw_transfer+0x222/0x270 -[ 196.551473] ? lock_release+0xa40/0xa40 -[ 196.555431] ? mulaw_encode+0x7c0/0x7c0 [ 196.559384] snd_pcm_plug_write_transfer+0x22d/0x420 -[ 196.564475] ? snd_pcm_plug_client_channels_buf+0x3f0/0x3f0 -[ 196.570164] ? snd_pcm_format_physical_width+0x5b/0x70 -[ 196.575419] ? snd_pcm_plug_client_channels_buf+0x1db/0x3f0 [ 196.581116] snd_pcm_oss_write2+0x260/0x420 -[ 196.585425] ? snd_pcm_oss_write3+0x1b0/0x1b0 [ 196.589913] snd_pcm_oss_write+0x5fe/0x830 -[ 196.594157] ? snd_pcm_oss_ioctl_compat+0x30/0x30 -[ 196.598991] ? snd_pcm_oss_ioctl_compat+0x30/0x30 [ 196.603813] __vfs_write+0xef/0x970 -[ 196.607418] ? rcu_note_context_switch+0x710/0x710 -[ 196.612327] ? kernel_read+0x120/0x120 -[ 196.616193] ? __might_sleep+0x95/0x190 -[ 196.620146] ? _cond_resched+0x14/0x30 -[ 196.624010] ? __inode_security_revalidate+0xd9/0x130 -[ 196.629180] ? avc_policy_seqno+0x9/0x20 -[ 196.633218] ? selinux_file_permission+0x82/0x460 -[ 196.638045] ? security_file_permission+0x89/0x1e0 -[ 196.642956] ? rw_verify_area+0xe5/0x2b0 -[ 196.646991] ? __fdget_raw+0x20/0x20 [ 196.650693] vfs_write+0x189/0x510 [ 196.654218] SyS_write+0xef/0x220 -[ 196.657656] ? SyS_read+0x220/0x220 -[ 196.661258] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 196.666256] ? trace_hardirqs_on_thunk+0x1a/0x1c [ 196.670999] entry_SYSCALL_64_fastpath+0x23/0x9a [ 196.675728] RIP: 0033:0x452ac9 [ 196.678895] RSP: 002b:00007fa354a13c58 EFLAGS: 00000212 ORIG_RAX: 0000000000000001 diff --git a/pkg/report/testdata/linux/report/200 b/pkg/report/testdata/linux/report/200 index d6c1f9deb40..d8e149eaf83 100644 --- a/pkg/report/testdata/linux/report/200 +++ b/pkg/report/testdata/linux/report/200 @@ -1,4 +1,4 @@ -TITLE: general protection fault in ip6t_do_table +TITLE: general protection fault in corrupted CORRUPTED: Y [ 73.452724] FAULT_INJECTION: forcing a failure. diff --git a/pkg/report/testdata/linux/report/207 b/pkg/report/testdata/linux/report/207 index 055c28d045a..168b6904949 100644 --- a/pkg/report/testdata/linux/report/207 +++ b/pkg/report/testdata/linux/report/207 @@ -1,4 +1,4 @@ -TITLE: INFO: rcu detected stall in free_percpu +TITLE: INFO: rcu detected stall in bpf_map_free_deferred isob2 login: [ 709.689951] INFO: rcu_sched detected stalls on CPUs/tasks: [ 709.695737] (detected by 0, t=125002 jiffies, g=6134, c=6133, q=11) diff --git a/pkg/report/testdata/linux/report/235 b/pkg/report/testdata/linux/report/235 index 2b62286241b..08599850f7e 100644 --- a/pkg/report/testdata/linux/report/235 +++ b/pkg/report/testdata/linux/report/235 @@ -1,4 +1,4 @@ -TITLE: INFO: task hung in should_fail +TITLE: INFO: task hung in corrupted CORRUPTED: Y [ 246.707981] FAULT_INJECTION: forcing a failure. diff --git a/pkg/report/testdata/linux/report/244 b/pkg/report/testdata/linux/report/244 index e8b1b55ceba..ba049d0ba51 100644 --- a/pkg/report/testdata/linux/report/244 +++ b/pkg/report/testdata/linux/report/244 @@ -1,4 +1,4 @@ -TITLE: WARNING: suspicious RCU usage in should_fail +TITLE: WARNING: suspicious RCU usage in corrupted CORRUPTED: Y [ 184.760966] ============================= diff --git a/pkg/report/testdata/linux/report/263 b/pkg/report/testdata/linux/report/263 new file mode 100644 index 00000000000..032cc08d0ac --- /dev/null +++ b/pkg/report/testdata/linux/report/263 @@ -0,0 +1,48 @@ +TITLE: INFO: rcu detected stall in snd_seq_write + +[ 599.875361] rcu: INFO: rcu_sched self-detected stall on CPU +[ 599.881199] rcu: 0-....: (105000 ticks this GP) idle=2f6/1/0x4000000000000002 softirq=23001/23001 fqs=26239 +[ 599.891361] rcu: (t=105008 jiffies g=49145 q=2382) +[ 599.896509] NMI backtrace for cpu 0 +[ 599.900150] CPU: 0 PID: 8551 Comm: syz-executor7 Not tainted 4.18.0-next-20180824+ #47 +[ 599.908198] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 599.917548] Call Trace: +[ 599.920132] +[ 599.922286] dump_stack+0x1c9/0x2b4 +[ 599.944682] nmi_cpu_backtrace.cold.3+0x48/0x88 +[ 599.954558] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 599.959858] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 599.965052] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 599.974486] print_cpu_stall.cold.78+0x2fb/0x59c +[ 599.995787] rcu_check_callbacks+0xd4a/0x15a0 +[ 600.093087] update_process_times+0x2d/0x70 +[ 600.097415] tick_sched_handle+0x9f/0x180 +[ 600.101564] tick_sched_timer+0x45/0x130 +[ 600.105627] __hrtimer_run_queues+0x3eb/0xff0 +[ 600.150990] hrtimer_interrupt+0x2f3/0x750 +[ 600.155241] smp_apic_timer_interrupt+0x16d/0x6a0 +[ 600.207857] apic_timer_interrupt+0xf/0x20 +[ 600.212089] +[ 600.214345] RIP: 0010:lock_release+0xa2/0x9f0 +[ 600.237744] RSP: 0018:ffff8801aa0078c8 EFLAGS: 00000292 ORIG_RAX: ffffffffffffff13 +[ 600.245455] RAX: 2ec5cd2493d34200 RBX: 1ffff10035400f1e RCX: 0000000000000000 +[ 600.252724] RDX: dffffc0000000000 RSI: 0000000000000000 RDI: ffff8801a9b56bbc +[ 600.259994] RBP: ffff8801aa0079f8 R08: 00000000000010df R09: 0000000000000001 +[ 600.267263] R10: ffff8801a9b56be8 R11: 737c3e5c87308161 R12: ffff8801aa0079d0 +[ 600.274530] R13: ffff8801c778d4c8 R14: ffff8801aa007ae0 R15: ffff8801a9b56380 +[ 600.307088] __might_fault+0x19e/0x1e0 +[ 600.310986] _copy_from_user+0x30/0x150 +[ 600.315053] snd_seq_write+0x472/0x8d0 +[ 600.332960] __vfs_write+0x117/0x9d0 +[ 600.379161] vfs_write+0x1fc/0x560 +[ 600.382717] ksys_write+0x101/0x260 +[ 600.390441] __x64_sys_write+0x73/0xb0 +[ 600.398306] do_syscall_64+0x1b9/0x820 +[ 600.436382] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 600.441569] RIP: 0033:0x457089 +[ 600.463760] RSP: 002b:00007f09af65dc78 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 600.471476] RAX: ffffffffffffffda RBX: 00007f09af65e6d4 RCX: 0000000000457089 +[ 600.478757] RDX: 00000000ffffff76 RSI: 0000000020000000 RDI: 0000000000000003 +[ 600.486032] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 600.493302] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 600.500573] R13: 00000000004d78a8 R14: 00000000004ca886 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/264 b/pkg/report/testdata/linux/report/264 new file mode 100644 index 00000000000..0ca42fafe54 --- /dev/null +++ b/pkg/report/testdata/linux/report/264 @@ -0,0 +1,105 @@ +TITLE: BUG: soft lockup in sys_rmdir + +[ 281.646907] watchdog: BUG: soft lockup - CPU#1 stuck for 134s! [syz-executor4:4588] +[ 281.654759] Modules linked in: +[ 281.657946] irq event stamp: 76609500 +[ 281.661752] hardirqs last enabled at (76609499): [] d_walk+0x80f/0xc80 +[ 281.670066] hardirqs last disabled at (76609500): [] interrupt_entry+0xb5/0xf0 +[ 281.679024] softirqs last enabled at (468244): [] __do_softirq+0x778/0xaf5 +[ 281.687696] softirqs last disabled at (468177): [] irq_exit+0x1d1/0x200 +[ 281.696008] CPU: 1 PID: 4588 Comm: syz-executor4 Not tainted 4.17.0-rc7+ #81 +[ 281.703881] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 281.713233] RIP: 0010:d_walk+0x840/0xc80 +[ 281.717286] RSP: 0018:ffff880183f5f9f8 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 281.724995] RAX: ffff8801b0d3c680 RBX: 0000000000000293 RCX: 1ffff100361a79e5 +[ 281.732263] RDX: 0000000000000000 RSI: ffffffff81c66cfb RDI: 0000000000000293 +[ 281.739552] RBP: ffff880183f5fb78 R08: ffff8801b0d3ceb8 R09: 0000000000000006 +[ 281.746815] R10: ffff8801b0d3c680 R11: 0000000000000000 R12: 0000000000000200 +[ 281.754080] R13: dffffc0000000000 R14: 1ffff100307ebf82 R15: ffff880183f5fb50 +[ 281.761359] FS: 000000000217c940(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 +[ 281.769577] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 281.775450] CR2: 00007ffca8309eac CR3: 0000000183eb2000 CR4: 00000000001406e0 +[ 281.782717] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 281.789980] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 281.797240] Call Trace: +[ 281.829726] shrink_dcache_parent+0x179/0x230 +[ 281.859894] vfs_rmdir+0x202/0x470 +[ 281.863431] do_rmdir+0x523/0x610 +[ 281.890027] __x64_sys_rmdir+0x36/0x40 +[ 281.893915] do_syscall_64+0x1b1/0x800 +[ 281.922815] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 281.927998] RIP: 0033:0x455777 +[ 281.931184] RSP: 002b:00007ffca830a5b8 EFLAGS: 00000206 ORIG_RAX: 0000000000000054 +[ 281.938889] RAX: ffffffffffffffda RBX: 0000000000000065 RCX: 0000000000455777 +[ 281.946179] RDX: 0000000000000000 RSI: 00007ffca830c360 RDI: 00007ffca830c360 +[ 281.953444] RBP: 00007ffca830c360 R08: 0000000000000000 R09: 0000000000000001 +[ 281.960792] R10: 0000000000000006 R11: 0000000000000206 R12: 000000000217d940 +[ 281.968057] R13: 0000000000000000 R14: 0000000000000198 R15: 000000000001f12d +[ 281.994904] Sending NMI from CPU 1 to CPUs 0: +[ 281.999398] NMI backtrace for cpu 0 +[ 281.999408] CPU: 0 PID: 4581 Comm: syz-executor6 Not tainted 4.17.0-rc7+ #81 +[ 281.999413] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 281.999428] RIP: 0010:select_collect+0x249/0x5f0 +[ 281.999433] RSP: 0018:ffff880189947908 EFLAGS: 00000286 +[ 281.999440] RAX: 0000000000000000 RBX: ffffed0031328f24 RCX: ffffffff81c5c3b4 +[ 281.999445] RDX: 0000000000000000 RSI: ffffffff81c5c4fa RDI: ffff880189947c28 +[ 281.999450] RBP: ffff8801899479e8 R08: ffff88018993e5c0 R09: ffffed0039dde7a4 +[ 281.999455] R10: ffffed0039dde7a4 R11: ffff8801ceef3d23 R12: 0000000000000000 +[ 281.999461] R13: 1ffff10031328f28 R14: ffff880189947c18 R15: dffffc0000000000 +[ 281.999468] FS: 0000000001790940(0000) GS:ffff8801dae00000(0000) knlGS:0000000000000000 +[ 281.999474] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 281.999479] CR2: 00007ffebea69e9c CR3: 000000018989f000 CR4: 00000000001406f0 +[ 281.999487] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 281.999491] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 281.999493] Call Trace: +[ 281.999541] d_walk+0x3c3/0xc80 +[ 281.999601] shrink_dcache_parent+0x179/0x230 +[ 281.999670] vfs_rmdir+0x202/0x470 +[ 281.999679] do_rmdir+0x523/0x610 +[ 281.999740] __x64_sys_rmdir+0x36/0x40 +[ 281.999748] do_syscall_64+0x1b1/0x800 +[ 281.999806] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 281.999812] RIP: 0033:0x455777 +[ 281.999816] RSP: 002b:00007ffebea6a5a8 EFLAGS: 00000202 ORIG_RAX: 0000000000000054 +[ 281.999824] RAX: ffffffffffffffda RBX: 0000000000000065 RCX: 0000000000455777 +[ 281.999829] RDX: 0000000000000000 RSI: 00007ffebea6c350 RDI: 00007ffebea6c350 +[ 281.999833] RBP: 00007ffebea6c350 R08: 0000000000000000 R09: 0000000000000001 +[ 281.999838] R10: 0000000000000006 R11: 0000000000000202 R12: 0000000001791940 +[ 281.999843] R13: 0000000000000000 R14: 0000000000000192 R15: 000000000001f12b +[ 282.000407] Kernel panic - not syncing: softlockup: hung tasks +[ 282.316419] CPU: 1 PID: 4588 Comm: syz-executor4 Tainted: G L 4.17.0-rc7+ #81 +[ 282.324992] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 282.334334] Call Trace: +[ 282.336997] +[ 282.339152] dump_stack+0x1b9/0x294 +[ 282.352760] panic+0x22f/0x4de +[ 282.375494] watchdog_timer_fn.cold.5+0xf3/0x126 +[ 282.380251] __hrtimer_run_queues+0x3e3/0x10a0 +[ 282.432572] hrtimer_interrupt+0x2f3/0x750 +[ 282.436818] smp_apic_timer_interrupt+0x15d/0x710 +[ 282.468292] apic_timer_interrupt+0xf/0x20 +[ 282.472527] +[ 282.474759] RIP: 0010:d_walk+0x840/0xc80 +[ 282.478811] RSP: 0018:ffff880183f5f9f8 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 282.486516] RAX: ffff8801b0d3c680 RBX: 0000000000000293 RCX: 1ffff100361a79e5 +[ 282.493778] RDX: 0000000000000000 RSI: ffffffff81c66cfb RDI: 0000000000000293 +[ 282.501043] RBP: ffff880183f5fb78 R08: ffff8801b0d3ceb8 R09: 0000000000000006 +[ 282.508310] R10: ffff8801b0d3c680 R11: 0000000000000000 R12: 0000000000000200 +[ 282.515587] R13: dffffc0000000000 R14: 1ffff100307ebf82 R15: ffff880183f5fb50 +[ 282.556091] shrink_dcache_parent+0x179/0x230 +[ 282.586289] vfs_rmdir+0x202/0x470 +[ 282.589830] do_rmdir+0x523/0x610 +[ 282.616379] __x64_sys_rmdir+0x36/0x40 +[ 282.620264] do_syscall_64+0x1b1/0x800 +[ 282.649064] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 282.654251] RIP: 0033:0x455777 +[ 282.657434] RSP: 002b:00007ffca830a5b8 EFLAGS: 00000206 ORIG_RAX: 0000000000000054 +[ 282.665140] RAX: ffffffffffffffda RBX: 0000000000000065 RCX: 0000000000455777 +[ 282.672404] RDX: 0000000000000000 RSI: 00007ffca830c360 RDI: 00007ffca830c360 +[ 282.679668] RBP: 00007ffca830c360 R08: 0000000000000000 R09: 0000000000000001 +[ 282.687015] R10: 0000000000000006 R11: 0000000000000206 R12: 000000000217d940 +[ 282.694282] R13: 0000000000000000 R14: 0000000000000198 R15: 000000000001f12d +[ 282.702335] Dumping ftrace buffer: +[ 282.706164] (ftrace buffer empty) +[ 282.709862] Kernel Offset: disabled +[ 282.713505] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/265 b/pkg/report/testdata/linux/report/265 new file mode 100644 index 00000000000..a21cdac9091 --- /dev/null +++ b/pkg/report/testdata/linux/report/265 @@ -0,0 +1,150 @@ +TITLE: BUG: soft lockup in snd_rawmidi_write + +[ 352.504666] watchdog: BUG: soft lockup - CPU#1 stuck for 134s! [syz-executor2:10431] +[ 352.512610] Modules linked in: +[ 352.515804] irq event stamp: 35856 +[ 352.519353] hardirqs last enabled at (35855): [] _raw_spin_unlock_irqrestore+0x74/0xc0 +[ 352.529060] hardirqs last disabled at (35856): [] interrupt_entry+0xb1/0xf0 +[ 352.537731] softirqs last enabled at (162): [] __do_softirq+0x778/0xaf5 +[ 352.546142] softirqs last disabled at (95): [] irq_exit+0x1d1/0x200 +[ 352.554114] CPU: 1 PID: 10431 Comm: syz-executor2 Not tainted 4.16.0+ #4 +[ 352.561560] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 352.570920] RIP: 0010:_raw_spin_unlock_irqrestore+0xa1/0xc0 +[ 352.576631] RSP: 0018:ffff880184db7780 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13 +[ 352.584344] RAX: dffffc0000000000 RBX: 0000000000000282 RCX: 0000000000000000 +[ 352.591613] RDX: 1ffffffff1162e55 RSI: 0000000000000001 RDI: 0000000000000282 +[ 352.598881] RBP: ffff880184db7790 R08: ffffed0035d21962 R09: 0000000000000000 +[ 352.606150] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801ae90cb08 +[ 352.613418] R13: ffff880184db7810 R14: 0000000000000001 R15: ffff8801cb9a5880 +[ 352.620696] FS: 00007fc943ad8700(0000) GS:ffff8801db100000(0000) knlGS:0000000000000000 +[ 352.628921] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 352.634800] CR2: 00007fc943ad7db8 CR3: 00000001b070d000 CR4: 00000000001406e0 +[ 352.642074] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 352.649346] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 352.656610] Call Trace: +[ 352.659210] snd_virmidi_output_trigger+0x522/0x6c0 +[ 352.674379] snd_rawmidi_kernel_write1+0x519/0x700 +[ 352.679328] snd_rawmidi_write+0x2e2/0xdc0 +[ 352.703684] __vfs_write+0x10b/0x880 +[ 352.751248] vfs_write+0x1f8/0x560 +[ 352.754805] ksys_write+0xf9/0x250 +[ 352.770317] SyS_write+0x24/0x30 +[ 352.777491] do_syscall_64+0x29e/0x9d0 +[ 352.810491] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 352.815683] RIP: 0033:0x455259 +[ 352.818870] RSP: 002b:00007fc943ad7c68 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 352.826582] RAX: ffffffffffffffda RBX: 00007fc943ad86d4 RCX: 0000000000455259 +[ 352.833855] RDX: 00000000e78e624c RSI: 0000000020000040 RDI: 0000000000000014 +[ 352.841123] RBP: 000000000072c010 R08: 0000000000000000 R09: 0000000000000000 +[ 352.848390] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 352.855661] R13: 00000000000006ca R14: 00000000006fd390 R15: 0000000000000002 +[ 352.882430] Kernel panic - not syncing: softlockup: hung tasks +[ 352.888409] CPU: 1 PID: 10431 Comm: syz-executor2 Tainted: G L 4.16.0+ #4 +[ 352.896551] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 352.905898] Call Trace: +[ 352.908475] +[ 352.910636] dump_stack+0x1b9/0x294 +[ 352.927344] panic+0x22f/0x4de +[ 352.953918] watchdog_timer_fn.cold.5+0xf3/0x126 +[ 352.958685] __hrtimer_run_queues+0x3e3/0x10a0 +[ 353.006124] hrtimer_interrupt+0x286/0x650 +[ 353.010381] smp_apic_timer_interrupt+0x15d/0x710 +[ 353.027757] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 353.029387] 1-....: (1 GPs behind) idle=7a2/1/4611686018427387906 softirq=35919/35920 fqs=20 +[ 353.034918] apic_timer_interrupt+0xf/0x20 +[ 353.034924] +[ 353.034937] RIP: 0010:_raw_spin_unlock_irqrestore+0xa1/0xc0 +[ 353.039231] +[ 353.047962] RSP: 0018:ffff880184db7780 EFLAGS: 00000282 +[ 353.051471] (detected by 0, t=125014 jiffies, g=19098, c=19097, q=977) +[ 353.051482] Sending NMI from CPU 0 to CPUs 1: +[ 353.056298] ORIG_RAX: ffffffffffffff13 +[ 353.056309] RAX: dffffc0000000000 RBX: 0000000000000282 RCX: 0000000000000000 +[ 353.062594] NMI backtrace for cpu 1 +[ 353.062600] CPU: 1 PID: 10431 Comm: syz-executor2 Tainted: G L 4.16.0+ #4 +[ 353.062606] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 353.062611] RIP: 0010:__sanitizer_cov_trace_const_cmp4+0x16/0x20 +[ 353.062616] RSP: 0018:ffff8801db107080 EFLAGS: 00000006 +[ 353.062623] RAX: 0000000000010003 RBX: ffff8801db1071aa RCX: ffffffff87462c86 +[ 353.062629] RDX: 0000000000000003 RSI: 0000000000000009 RDI: 0000000000000005 +[ 353.062634] RBP: ffff8801db107080 R08: ffff8801894c8440 R09: fffffbfff148ee7c +[ 353.062639] R10: fffffbfff148ee7c R11: ffffffff8a4773e0 R12: 0000000000000003 +[ 353.062644] R13: 0000000000003033 R14: 000000000000000a R15: ffffffff8a4773e1 +[ 353.062650] FS: 00007fc943ad8700(0000) GS:ffff8801db100000(0000) knlGS:0000000000000000 +[ 353.062654] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 353.062659] CR2: 00007fc943ad7db8 CR3: 00000001b070d000 CR4: 00000000001406e0 +[ 353.062665] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 353.062670] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 353.062673] Call Trace: +[ 353.062675] +[ 353.062679] put_dec_trunc8+0x266/0x300 +[ 353.062682] put_dec+0x3b/0xf0 +[ 353.062686] number+0xb94/0xc90 +[ 353.062704] vsnprintf+0x13a5/0x1b40 +[ 353.062711] sprintf+0xa7/0xd0 +[ 353.062718] print_prefix+0x26a/0x3f0 +[ 353.062721] msg_print_text+0xca/0x1c0 +[ 353.062725] console_unlock+0x4f5/0x1100 +[ 353.062751] vprintk_emit+0x6ad/0xdd0 +[ 353.062773] vprintk_default+0x28/0x30 +[ 353.062777] vprintk_func+0x7a/0xe7 +[ 353.062780] printk+0x9e/0xba +[ 353.062791] __show_regs.cold.7+0x86/0x54a +[ 353.062799] show_regs_if_on_stack.constprop.7+0x36/0x39 +[ 353.062803] show_trace_log_lvl+0x120/0x25c +[ 353.062811] show_stack+0x38/0x3a +[ 353.062814] dump_stack+0x1b9/0x294 +[ 353.062829] panic+0x22f/0x4de +[ 353.062853] watchdog_timer_fn.cold.5+0xf3/0x126 +[ 353.062857] __hrtimer_run_queues+0x3e3/0x10a0 +[ 353.062898] hrtimer_interrupt+0x286/0x650 +[ 353.062903] smp_apic_timer_interrupt+0x15d/0x710 +[ 353.062930] apic_timer_interrupt+0xf/0x20 +[ 353.062932] +[ 353.062937] RIP: 0010:_raw_spin_unlock_irqrestore+0xa1/0xc0 +[ 353.062941] RSP: 0018:ffff880184db7780 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13 +[ 353.062950] RAX: dffffc0000000000 RBX: 0000000000000282 RCX: 0000000000000000 +[ 353.062955] RDX: 1ffffffff1162e55 RSI: 0000000000000001 RDI: 0000000000000282 +[ 353.062960] RBP: ffff880184db7790 R08: ffffed0035d21962 R09: 0000000000000000 +[ 353.062965] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801ae90cb08 +[ 353.062971] R13: ffff880184db7810 R14: 0000000000000001 R15: ffff8801cb9a5880 +[ 353.062975] snd_virmidi_output_trigger+0x522/0x6c0 +[ 353.062987] snd_rawmidi_kernel_write1+0x519/0x700 +[ 353.062991] snd_rawmidi_write+0x2e2/0xdc0 +[ 353.063017] __vfs_write+0x10b/0x880 +[ 353.063059] vfs_write+0x1f8/0x560 +[ 353.063062] ksys_write+0xf9/0x250 +[ 353.063076] SyS_write+0x24/0x30 +[ 353.063084] do_syscall_64+0x29e/0x9d0 +[ 353.063112] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 353.063115] RIP: 0033:0x455259 +[ 353.063119] RSP: 002b:00007fc943ad7c68 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 353.063128] RAX: ffffffffffffffda RBX: 00007fc943ad86d4 RCX: 0000000000455259 +[ 353.063134] RDX: 00000000e78e624c RSI: 0000000020000040 RDI: 0000000000000014 +[ 353.063139] RBP: 000000000072c010 R08: 0000000000000000 R09: 0000000000000000 +[ 353.063144] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 353.063149] R13: 00000000000006ca R14: 00000000006fd390 R15: 0000000000000002 +[ 353.765584] RDX: 1ffffffff1162e55 RSI: 0000000000000001 RDI: 0000000000000282 +[ 353.772854] RBP: ffff880184db7790 R08: ffffed0035d21962 R09: 0000000000000000 +[ 353.780122] R10: 0000000000000000 R11: 0000000000000000 R12: ffff8801ae90cb08 +[ 353.787390] R13: ffff880184db7810 R14: 0000000000000001 R15: ffff8801cb9a5880 +[ 353.794699] snd_virmidi_output_trigger+0x522/0x6c0 +[ 353.809878] snd_rawmidi_kernel_write1+0x519/0x700 +[ 353.814825] snd_rawmidi_write+0x2e2/0xdc0 +[ 353.839166] __vfs_write+0x10b/0x880 +[ 353.886693] vfs_write+0x1f8/0x560 +[ 353.890241] ksys_write+0xf9/0x250 +[ 353.905722] SyS_write+0x24/0x30 +[ 353.912894] do_syscall_64+0x29e/0x9d0 +[ 353.945884] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 353.951072] RIP: 0033:0x455259 +[ 353.954261] RSP: 002b:00007fc943ad7c68 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 353.961976] RAX: ffffffffffffffda RBX: 00007fc943ad86d4 RCX: 0000000000455259 +[ 353.969244] RDX: 00000000e78e624c RSI: 0000000020000040 RDI: 0000000000000014 +[ 353.976513] RBP: 000000000072c010 R08: 0000000000000000 R09: 0000000000000000 +[ 353.983779] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 353.991051] R13: 00000000000006ca R14: 00000000006fd390 R15: 0000000000000002 +[ 354.000710] Dumping ftrace buffer: +[ 354.004343] (ftrace buffer empty) +[ 354.008034] Kernel Offset: disabled +[ 354.011650] Rebooting in 86400 seconds.. diff --git a/pkg/report/testdata/linux/report/266 b/pkg/report/testdata/linux/report/266 new file mode 100644 index 00000000000..3dc7fadae7c --- /dev/null +++ b/pkg/report/testdata/linux/report/266 @@ -0,0 +1,40 @@ +TITLE: BUG: soft lockup in sys_rmdir + +[ 1144.580971] watchdog: BUG: soft lockup - CPU#1 stuck for 134s! [syz-executor0:4569] +[ 1144.589091] Modules linked in: +[ 1144.592335] irq event stamp: 78766212 +[ 1144.596241] hardirqs last enabled at (78766211): [] d_walk+0x80f/0xc80 +[ 1144.604582] hardirqs last disabled at (78766212): [] interrupt_entry+0xb5/0xf0 +[ 1144.613524] softirqs last enabled at (13779542): [] __do_softirq+0x778/0xaf5 +[ 1144.622392] softirqs last disabled at (13779283): [] irq_exit+0x1d1/0x200 +[ 1144.630891] CPU: 1 PID: 4569 Comm: syz-executor0 Not tainted 4.17.0-rc7+ #38 +[ 1144.639205] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1144.648575] RIP: 0010:check_memory_region+0xc3/0x1b0 +[ 1144.653694] RSP: 0018:ffff8801a14cf930 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 1144.661412] RAX: ffffed0033825dd1 RBX: ffff88019c12ee80 RCX: ffffffff815e0b5e +[ 1144.668708] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff88019c12ee80 +[ 1144.676008] RBP: ffff8801a14cf938 R08: ffffed0033825dd1 R09: ffffed0033825dd0 +[ 1144.683284] R10: ffffed0033825dd0 R11: ffff88019c12ee83 R12: 1ffff10034299f29 +[ 1144.690568] R13: dffffc0000000000 R14: ffff8801a14cf9a8 R15: ffff8801a14cfb50 +[ 1144.697845] FS: 0000000000bbe940(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 +[ 1144.706087] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1144.711970] CR2: 0000000000930008 CR3: 00000001a1b76000 CR4: 00000000001406e0 +[ 1144.719244] DR0: 0000000020000000 DR1: 0000000020000000 DR2: 0000000000000000 +[ 1144.726528] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000600 +[ 1144.733967] Call Trace: +[ 1144.740725] do_raw_spin_unlock+0x9e/0x2e0 +[ 1144.758045] _raw_spin_unlock+0x22/0x30 +[ 1144.762022] d_walk+0x451/0xc80 +[ 1144.792735] shrink_dcache_parent+0x179/0x230 +[ 1144.823077] vfs_rmdir+0x202/0x470 +[ 1144.826642] do_rmdir+0x523/0x610 +[ 1144.853253] __x64_sys_rmdir+0x36/0x40 +[ 1144.857239] do_syscall_64+0x1b1/0x800 +[ 1144.886120] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1144.891437] RIP: 0033:0x455897 +[ 1144.894624] RSP: 002b:00007ffc25471ad8 EFLAGS: 00000206 ORIG_RAX: 0000000000000054 +[ 1144.902336] RAX: ffffffffffffffda RBX: 0000000000000065 RCX: 0000000000455897 +[ 1144.909605] RDX: 0000000000000000 RSI: 00007ffc25473880 RDI: 00007ffc25473880 +[ 1144.916875] RBP: 00007ffc25473880 R08: 0000000000000000 R09: 0000000000000001 +[ 1144.924148] R10: 000000000000000a R11: 0000000000000206 R12: 0000000000bbf940 +[ 1144.931450] R13: 0000000000000000 R14: 00000000000013f1 R15: 00000000000ef6b8 diff --git a/pkg/report/testdata/linux/report/267 b/pkg/report/testdata/linux/report/267 new file mode 100644 index 00000000000..05dffdc0208 --- /dev/null +++ b/pkg/report/testdata/linux/report/267 @@ -0,0 +1,53 @@ +TITLE: BUG: soft lockup in ipv6_rcv + +[ 1290.104807] watchdog: BUG: soft lockup - CPU#1 stuck for 134s! [syz-executor738:4553] +[ 1290.112830] Modules linked in: +[ 1290.116020] irq event stamp: 13885653 +[ 1290.120062] hardirqs last enabled at (13885652): [] restore_regs_and_return_to_kernel+0x0/0x2b +[ 1290.130479] hardirqs last disabled at (13885653): [] interrupt_entry+0xb5/0xf0 +[ 1290.139538] softirqs last enabled at (13614028): [] tun_get_user+0x1dd9/0x4290 +[ 1290.148551] softirqs last disabled at (13614032): [] tun_get_user+0x313f/0x4290 +[ 1290.158637] CPU: 1 PID: 4553 Comm: syz-executor738 Not tainted 4.17.0-rc3+ #40 +[ 1290.165983] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1290.175400] RIP: 0010:__sanitizer_cov_trace_pc+0x20/0x50 +[ 1290.180861] RSP: 0018:ffff8801d8cfe250 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 1290.188563] RAX: ffff8801d88a8080 RBX: ffff8801d7389e40 RCX: 0000000000000006 +[ 1290.195826] RDX: 0000000000000000 RSI: ffffffff868da4ad RDI: ffff8801c8a53277 +[ 1290.203081] RBP: ffff8801d8cfe250 R08: ffff8801d88a8080 R09: ffff8801d8cfe3e8 +[ 1290.210432] R10: ffffed003b19fc87 R11: ffff8801d8cfe43f R12: ffff8801c8a5327f +[ 1290.217775] R13: 0000000000000000 R14: ffff8801c8a4e5fe R15: ffff8801d8cfe3e8 +[ 1290.225035] FS: 0000000000d88940(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 +[ 1290.233337] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1290.239235] CR2: ffffffffff600400 CR3: 00000001acab3000 CR4: 00000000001406e0 +[ 1290.246677] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1290.253936] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 1290.261210] Call Trace: +[ 1290.263888] _decode_session6+0xc1d/0x14f0 +[ 1290.268154] __xfrm_decode_session+0x71/0x140 +[ 1290.272668] icmpv6_route_lookup+0x395/0x6e0 +[ 1290.294459] icmp6_send+0x1982/0x2da0 +[ 1290.398009] icmpv6_send+0x17a/0x300 +[ 1290.421143] ip6_input_finish+0x14e1/0x1a30 +[ 1290.444140] ip6_input+0xe1/0x5e0 +[ 1290.456233] ip6_rcv_finish+0x29c/0xa10 +[ 1290.482435] ipv6_rcv+0xeb8/0x2040 +[ 1290.510550] __netif_receive_skb_core+0x2468/0x3650 +[ 1290.597399] __netif_receive_skb+0x2c/0x1e0 +[ 1290.601729] netif_receive_skb_internal+0x126/0x7b0 +[ 1290.629906] napi_gro_frags+0x631/0xc40 +[ 1290.650727] tun_get_user+0x3168/0x4290 +[ 1290.691649] tun_chr_write_iter+0xb9/0x154 +[ 1290.695900] do_iter_readv_writev+0x859/0xa50 +[ 1290.709371] do_iter_write+0x185/0x5f0 +[ 1290.716890] vfs_writev+0x1c7/0x330 +[ 1290.759809] do_writev+0x112/0x2f0 +[ 1290.772027] __x64_sys_writev+0x75/0xb0 +[ 1290.776025] do_syscall_64+0x1b1/0x800 +[ 1290.804960] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1290.810147] RIP: 0033:0x442c30 +[ 1290.813340] RSP: 002b:00007ffe3fff4978 EFLAGS: 00000246 ORIG_RAX: 0000000000000014 +[ 1290.821081] RAX: ffffffffffffffda RBX: 0000000000000003 RCX: 0000000000442c30 +[ 1290.828385] RDX: 0000000000000001 RSI: 00007ffe3fff49c0 RDI: 00000000000000fc +[ 1290.835646] RBP: 0000000000d89940 R08: 0000000000000000 R09: 00007ffe000087fe +[ 1290.842901] R10: 0000000000000000 R11: 0000000000000246 R12: 585858582e72656c +[ 1290.850156] R13: 6c616b7a79732f2e R14: 0000000000000000 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/268 b/pkg/report/testdata/linux/report/268 new file mode 100644 index 00000000000..c6492ea7f40 --- /dev/null +++ b/pkg/report/testdata/linux/report/268 @@ -0,0 +1,59 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 328.823841] rcu: INFO: rcu_preempt self-detected stall on CPU +[ 328.830097] rcu: 1-...!: (3 GPs behind) idle=19a/1/0x4000000000000002 softirq=12356/12357 fqs=2 +[ 328.839607] rcu: (t=10500 jiffies g=12045 q=155) +[ 328.844567] rcu: rcu_preempt kthread starved for 10492 jiffies! g12045 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x0 ->cpu=0 +[ 328.855046] rcu: RCU grace-period kthread stack dump: +[ 328.860235] rcu_preempt R running task 22872 10 2 0x80000000 +[ 328.867450] Call Trace: +[ 328.870074] __schedule+0x86c/0x1ed0 +[ 328.919601] schedule+0xfe/0x460 +[ 328.950461] schedule_timeout+0x140/0x260 +[ 328.969038] rcu_gp_kthread+0x9d9/0x2310 +[ 329.236894] kthread+0x35a/0x420 +[ 329.248156] ret_from_fork+0x3a/0x50 +[ 329.251894] NMI backtrace for cpu 1 +[ 329.255529] CPU: 1 PID: 8314 Comm: syz-executor2 Not tainted 4.19.0-rc2+ #6 +[ 329.262634] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 329.271989] Call Trace: +[ 329.274578] +[ 329.276738] dump_stack+0x1c4/0x2b4 +[ 329.295554] nmi_cpu_backtrace.cold.3+0x63/0xa2 +[ 329.305441] nmi_trigger_cpumask_backtrace+0x1b3/0x1ed +[ 329.310726] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 329.315961] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 329.325408] print_cpu_stall.cold.78+0x2d3/0x524 +[ 329.374647] rcu_check_callbacks+0xfd9/0x1990 +[ 329.512616] update_process_times+0x2d/0x70 +[ 329.516985] tick_sched_handle+0x9f/0x180 +[ 329.521146] tick_sched_timer+0x45/0x130 +[ 329.525216] __hrtimer_run_queues+0x41c/0x10d0 +[ 329.584253] hrtimer_interrupt+0x313/0x780 +[ 329.588515] smp_apic_timer_interrupt+0x1a1/0x760 +[ 329.655276] apic_timer_interrupt+0xf/0x20 +[ 329.659508] +[ 329.661757] RIP: 0010:kvm_mmu_page_fault+0x2a3/0x1ad0 +[ 329.685879] RSP: 0018:ffff880191beeed0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 329.693625] RAX: 0000000000000000 RBX: ffff880191b70480 RCX: ffffc900022ce000 +[ 329.700898] RDX: 0000000000000007 RSI: ffffffff8112ec74 RDI: 0000000000000005 +[ 329.708171] RBP: ffff880191bef2b0 R08: ffff8801c44f83c0 R09: ffff880191beed98 +[ 329.715444] R10: ffff880191beeae8 R11: ffff88018d7cf38b R12: 0000000000000001 +[ 329.722721] R13: 0000000000000001 R14: 0000000000000001 R15: 0000000000000f7e +[ 329.842590] handle_ept_violation+0x29e/0x6a0 +[ 329.859146] vmx_handle_exit+0x2f7/0x17f0 +[ 329.884567] vcpu_enter_guest+0x14a9/0x62e0 +[ 329.992063] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 330.001951] kvm_vcpu_ioctl+0x72b/0x1150 +[ 330.061445] do_vfs_ioctl+0x1de/0x1720 +[ 330.102388] ksys_ioctl+0xa9/0xd0 +[ 330.105859] __x64_sys_ioctl+0x73/0xb0 +[ 330.109814] do_syscall_64+0x1b9/0x820 +[ 330.148822] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 330.154015] RIP: 0033:0x457099 +[ 330.176154] RSP: 002b:00007f9e42b44c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 330.183875] RAX: ffffffffffffffda RBX: 00007f9e42b456d4 RCX: 0000000000457099 +[ 330.191150] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 330.198426] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 330.205697] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 330.212969] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/269 b/pkg/report/testdata/linux/report/269 new file mode 100644 index 00000000000..18a6692ed1f --- /dev/null +++ b/pkg/report/testdata/linux/report/269 @@ -0,0 +1,51 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 351.337674] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 351.343969] rcu: (detected by 1, t=10502 jiffies, g=9981, q=283) +[ 351.350228] rcu: All QSes seen, last rcu_preempt kthread activity 10503 (4294972288-4294961785), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 351.363323] syz-executor0 R running task 21280 7720 5511 0x00000000 +[ 351.370540] Call Trace: +[ 351.373124] +[ 351.375304] sched_show_task.cold.83+0x2b6/0x30a +[ 351.411243] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 351.477062] rcu_check_callbacks+0xafc/0x1990 +[ 351.621883] update_process_times+0x2d/0x70 +[ 351.626218] tick_sched_handle+0x9f/0x180 +[ 351.630382] tick_sched_timer+0x45/0x130 +[ 351.634462] __hrtimer_run_queues+0x41c/0x10d0 +[ 351.693471] hrtimer_interrupt+0x313/0x780 +[ 351.697833] smp_apic_timer_interrupt+0x1a1/0x760 +[ 351.758839] apic_timer_interrupt+0xf/0x20 +[ 351.763076] +[ 351.765322] RIP: 0010:__virt_addr_valid+0xfe/0x230 +[ 351.789242] RSP: 0018:ffff88018a1d68b8 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 351.796963] RAX: dffffc0000000000 RBX: 0000000193be3c00 RCX: ffffc900060ed000 +[ 351.804244] RDX: 1ffffffff1679c04 RSI: ffffffff81386a72 RDI: ffffffff8b3ce020 +[ 351.811523] RBP: ffff88018a1d68d0 R08: ffff8801c071a040 R09: ffffed003277c780 +[ 351.818800] R10: ffffed003277c781 R11: ffff880193be3c0e R12: 0000000000000032 +[ 351.826073] R13: 0000000000000000 R14: 0000000000000000 R15: 000000000000000f +[ 351.842081] __check_object_size+0x15b/0x782 +[ 351.859038] __kvm_read_guest_page+0xda/0x130 +[ 351.863547] kvm_vcpu_read_guest_page+0x40/0x50 +[ 351.868231] kvm_fetch_guest_virt+0x149/0x1c0 +[ 351.876984] __do_insn_fetch_bytes+0x45b/0x950 +[ 351.890976] x86_decode_insn+0x1544/0x54c0 +[ 351.904190] x86_emulate_instruction+0x89a/0x1f90 +[ 351.928622] kvm_mmu_page_fault+0x342/0x1ad0 +[ 352.039266] handle_ept_violation+0x29e/0x6a0 +[ 352.055832] vmx_handle_exit+0x2f7/0x17f0 +[ 352.081257] vcpu_enter_guest+0x14a9/0x62e0 +[ 352.160089] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 352.169984] kvm_vcpu_ioctl+0x72b/0x1150 +[ 352.247217] do_vfs_ioctl+0x1de/0x1720 +[ 352.291956] ksys_ioctl+0xa9/0xd0 +[ 352.295448] __x64_sys_ioctl+0x73/0xb0 +[ 352.299350] do_syscall_64+0x1b9/0x820 +[ 352.343921] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 352.349120] RIP: 0033:0x457099 +[ 352.371250] RSP: 002b:00007f10ce81ac78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 352.378968] RAX: ffffffffffffffda RBX: 00007f10ce81b6d4 RCX: 0000000000457099 +[ 352.386246] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 352.393525] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 352.400800] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 352.408074] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/270 b/pkg/report/testdata/linux/report/270 new file mode 100644 index 00000000000..45c7f7ed259 --- /dev/null +++ b/pkg/report/testdata/linux/report/270 @@ -0,0 +1,46 @@ +TITLE: INFO: rcu detected stall in mousedev_write + +[ 489.536147] rcu: INFO: rcu_sched self-detected stall on CPU +[ 489.542259] rcu: 1-....: (104999 ticks this GP) idle=19e/1/0x4000000000000002 softirq=112269/112269 fqs=19579 +[ 489.552836] rcu: (t=105000 jiffies g=225813 q=1973) +[ 489.558034] NMI backtrace for cpu 1 +[ 489.561659] CPU: 1 PID: 21558 Comm: syz-executor5 Not tainted 4.18.0+ #203 +[ 489.568681] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 489.578024] Call Trace: +[ 489.580590] +[ 489.582742] dump_stack+0x1c9/0x2b4 +[ 489.605152] nmi_cpu_backtrace.cold.3+0x48/0x88 +[ 489.615031] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 489.620303] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 489.625482] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 489.634885] print_cpu_stall.cold.78+0x2fb/0x59c +[ 489.660436] rcu_check_callbacks+0xd4a/0x15a0 +[ 489.763624] update_process_times+0x2d/0x70 +[ 489.767934] tick_sched_handle+0x9f/0x180 +[ 489.772076] tick_sched_timer+0x45/0x130 +[ 489.776132] __hrtimer_run_queues+0x3eb/0xff0 +[ 489.825496] hrtimer_interrupt+0x2f3/0x750 +[ 489.829726] smp_apic_timer_interrupt+0x16d/0x6a0 +[ 489.882166] apic_timer_interrupt+0xf/0x20 +[ 489.886385] +[ 489.888622] RIP: 0010:_raw_spin_unlock_irq+0x56/0x70 +[ 489.912610] RSP: 0018:ffff88018eb8faf8 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 489.920323] RAX: dffffc0000000000 RBX: ffff8801bff42be8 RCX: 0000000000000000 +[ 489.927581] RDX: 1ffffffff0fe379f RSI: 0000000000000001 RDI: ffffffff87f1bcf8 +[ 489.934838] RBP: ffff88018eb8fb00 R08: ffff88018d902000 R09: 0000000000000000 +[ 489.942093] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000c6c5ef +[ 489.949350] R13: dffffc0000000000 R14: 0000000000c6c5ef R15: ffff8801bff42be8 +[ 489.961123] mousedev_write+0x19d/0x840 +[ 489.970770] __vfs_write+0x117/0x9d0 +[ 490.009169] vfs_write+0x1fc/0x560 +[ 490.012705] ksys_write+0x101/0x260 +[ 490.025464] __x64_sys_write+0x73/0xb0 +[ 490.029373] do_syscall_64+0x1b9/0x820 +[ 490.068264] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 490.073439] RIP: 0033:0x457089 +[ 490.095520] RSP: 002b:00007f0d92208c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000001 +[ 490.103219] RAX: ffffffffffffffda RBX: 00007f0d922096d4 RCX: 0000000000457089 +[ 490.110563] RDX: 00000000ffffffd1 RSI: 00000000200001c0 RDI: 0000000000000005 +[ 490.117820] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 490.125073] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 490.132328] R13: 00000000004d7088 R14: 00000000004ca379 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/271 b/pkg/report/testdata/linux/report/271 new file mode 100644 index 00000000000..b202f874005 --- /dev/null +++ b/pkg/report/testdata/linux/report/271 @@ -0,0 +1,57 @@ +TITLE: INFO: rcu detected stall in fuse_dev_release + +[ 629.392020] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 629.397716] (detected by 0, t=105007 jiffies, g=118924, c=118923, q=65) +[ 629.404568] All QSes seen, last rcu_sched kthread activity 105014 (4295296700-4295191686), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 629.416688] syz-executor7 R running task 23352 14084 4424 0x00000008 +[ 629.423888] Call Trace: +[ 629.426460] +[ 629.428611] sched_show_task.cold.84+0x27a/0x301 +[ 629.459938] print_other_cpu_stall.cold.77+0x92f/0x9d2 +[ 629.498729] check_cpu_stall.isra.60+0x721/0xf70 +[ 629.570316] rcu_check_callbacks+0x23f/0xcd0 +[ 629.647174] update_process_times+0x2d/0x70 +[ 629.651489] tick_sched_handle+0x9f/0x180 +[ 629.655627] tick_sched_timer+0x45/0x130 +[ 629.659679] __hrtimer_run_queues+0x3eb/0x10c0 +[ 629.708023] hrtimer_interrupt+0x2f3/0x750 +[ 629.712277] smp_apic_timer_interrupt+0x165/0x730 +[ 629.739746] apic_timer_interrupt+0xf/0x20 +[ 629.743963] +[ 629.746192] RIP: 0010:check_memory_region+0x121/0x1b0 +[ 629.770657] RSP: 0018:ffff88018fec7388 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 629.778358] RAX: ffffed0039557a49 RBX: 1ffff10039557a48 RCX: ffffffff815faf29 +[ 629.785618] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff8801caabd240 +[ 629.792874] RBP: ffff88018fec73a0 R08: ffffed0039557a49 R09: ffffed0039557a48 +[ 629.800131] R10: ffffed0039557a48 R11: ffff8801caabd243 R12: ffffed0039557a49 +[ 629.807389] R13: 0000000000000001 R14: dffffc0000000000 R15: ffff8801caabd240 +[ 629.820455] kasan_check_read+0x11/0x20 +[ 629.824417] native_queued_spin_lock_slowpath+0x189/0x1200 +[ 629.924285] do_raw_spin_lock+0x1a7/0x200 +[ 629.928436] _raw_spin_lock+0x32/0x40 +[ 629.936113] request_end+0x400/0xb20 +[ 629.948990] end_requests+0x296/0x460 +[ 629.983347] fuse_dev_release+0x18c/0x2e0 +[ 630.004675] __fput+0x355/0x8b0 +[ 630.015712] ____fput+0x15/0x20 +[ 630.018984] task_work_run+0x1ec/0x2a0 +[ 630.031840] exit_to_usermode_loop+0x313/0x370 +[ 630.044964] do_syscall_64+0x6be/0x820 +[ 630.073391] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 630.078571] RIP: 0033:0x456b29 +[ 630.101035] RSP: 002b:00007fae499f2c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000000 +[ 630.108738] RAX: 0000000000000030 RBX: 00007fae499f36d4 RCX: 0000000000456b29 +[ 630.116002] RDX: 0000000000001000 RSI: 0000000020001000 RDI: 0000000000000017 +[ 630.123265] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 630.130534] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 630.137793] R13: 00000000004d2b40 R14: 00000000004c7b8b R15: 0000000000000000 +[ 630.145073] rcu_sched kthread starved for 105755 jiffies! g118924 c118923 f0x2 RCU_GP_WAIT_FQS(3) ->state=0x0 ->cpu=1 +[ 630.155808] RCU grace-period kthread stack dump: +[ 630.160554] rcu_sched R running task 23688 10 2 0x80000000 +[ 630.167746] Call Trace: +[ 630.170332] __schedule+0x87c/0x1ec0 +[ 630.195223] schedule+0xfb/0x450 +[ 630.229166] schedule_timeout+0x140/0x260 +[ 630.242118] rcu_gp_kthread+0x717/0x1c90 +[ 630.389446] kthread+0x345/0x410 +[ 630.401684] ret_from_fork+0x3a/0x50 diff --git a/pkg/report/testdata/linux/report/272 b/pkg/report/testdata/linux/report/272 new file mode 100644 index 00000000000..c9c6b7b62a1 --- /dev/null +++ b/pkg/report/testdata/linux/report/272 @@ -0,0 +1,58 @@ +TITLE: INFO: rcu detected stall in netlink_sendmsg + +[ 480.173025] INFO: rcu_sched self-detected stall on CPU +[ 480.178536] 1-....: (104999 ticks this GP) idle=002/1/4611686018427387906 softirq=125049/125058 fqs=26224 +[ 480.188655] (t=105000 jiffies g=69343 c=69342 q=1302) +[ 480.194010] NMI backtrace for cpu 1 +[ 480.197631] CPU: 1 PID: 29490 Comm: syz-executor5 Not tainted 4.18.0-rc7+ #170 +[ 480.204969] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 480.214301] Call Trace: +[ 480.216863] +[ 480.219013] dump_stack+0x1c9/0x2b4 +[ 480.231770] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 480.241597] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 480.246859] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 480.252036] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 480.261519] check_cpu_stall.isra.60.cold.78+0x36c/0x5a6 +[ 480.326256] rcu_check_callbacks+0x23f/0xcd0 +[ 480.398507] update_process_times+0x2d/0x70 +[ 480.402812] tick_sched_handle+0x9f/0x180 +[ 480.406950] tick_sched_timer+0x45/0x130 +[ 480.410995] __hrtimer_run_queues+0x3eb/0x10c0 +[ 480.459331] hrtimer_interrupt+0x2f3/0x750 +[ 480.463559] smp_apic_timer_interrupt+0x165/0x730 +[ 480.490977] apic_timer_interrupt+0xf/0x20 +[ 480.495190] +[ 480.497429] RIP: 0010:lock_acquire+0x25f/0x540 +[ 480.521174] RSP: 0018:ffff8801999a67c0 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 480.528871] RAX: dffffc0000000000 RBX: 1ffff10033334cfd RCX: 0000000000000000 +[ 480.536121] RDX: 1ffffffff0fe3615 RSI: 0000000000000000 RDI: 0000000000000286 +[ 480.543371] RBP: ffff8801999a68b0 R08: 0000000000000008 R09: 0000000000000002 +[ 480.550624] R10: ffff880192d5c908 R11: 632564d1cb2a3fed R12: ffff880192d5c080 +[ 480.557874] R13: 0000000000000002 R14: 0000000000000000 R15: 0000000000000000 +[ 480.585903] tipc_sk_lookup+0x131/0xfa0 +[ 480.642175] tipc_nl_publ_dump+0x243/0xfa2 +[ 480.687542] __tipc_nl_compat_dumpit.isra.11+0x20b/0xad0 +[ 480.717699] tipc_nl_compat_sk_dump+0x87c/0xca0 +[ 480.769074] __tipc_nl_compat_dumpit.isra.11+0x337/0xad0 +[ 480.788328] tipc_nl_compat_dumpit+0x1f4/0x440 +[ 480.798089] tipc_nl_compat_recv+0x12b3/0x19a0 +[ 480.825649] genl_family_rcv_msg+0x8a3/0x1140 +[ 480.855652] genl_rcv_msg+0xc6/0x168 +[ 480.859351] netlink_rcv_skb+0x172/0x440 +[ 480.876417] genl_rcv+0x28/0x40 +[ 480.879678] netlink_unicast+0x5a0/0x760 +[ 480.898642] netlink_sendmsg+0xa18/0xfd0 +[ 480.921151] sock_sendmsg+0xd5/0x120 +[ 480.924856] ___sys_sendmsg+0x7fd/0x930 +[ 480.963667] __sys_sendmsg+0x11d/0x290 +[ 480.984618] __x64_sys_sendmsg+0x78/0xb0 +[ 480.988662] do_syscall_64+0x1b9/0x820 +[ 481.012549] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 481.017720] RIP: 0033:0x456a09 +[ 481.040077] RSP: 002b:00007f8665b39c78 EFLAGS: 00000246 ORIG_RAX: 000000000000002e +[ 481.047767] RAX: ffffffffffffffda RBX: 00007f8665b3a6d4 RCX: 0000000000456a09 +[ 481.055026] RDX: 0000000000000000 RSI: 0000000020000000 RDI: 0000000000000022 +[ 481.062281] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 481.076996] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 481.084255] R13: 00000000004d3058 R14: 00000000004c7d3e R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/273 b/pkg/report/testdata/linux/report/273 new file mode 100644 index 00000000000..5df0b7ab3df --- /dev/null +++ b/pkg/report/testdata/linux/report/273 @@ -0,0 +1,41 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 631.121467] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 631.127772] rcu: (detected by 0, t=10502 jiffies, g=6833, q=154) +[ 631.134056] rcu: All QSes seen, last rcu_preempt kthread activity 10503 (4295000262-4294989759), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 631.146731] syz-executor4 R running task 22104 7395 5354 0x00000000 +[ 631.153965] Call Trace: +[ 631.156563] +[ 631.158736] sched_show_task.cold.83+0x2b6/0x30a +[ 631.194724] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 631.261421] rcu_check_callbacks+0xafc/0x1990 +[ 631.401184] update_process_times+0x2d/0x70 +[ 631.405548] tick_sched_handle+0x9f/0x180 +[ 631.409740] tick_sched_timer+0x45/0x130 +[ 631.413842] __hrtimer_run_queues+0x41c/0x10d0 +[ 631.468709] hrtimer_interrupt+0x313/0x780 +[ 631.472986] smp_apic_timer_interrupt+0x1a1/0x760 +[ 631.539804] apic_timer_interrupt+0xf/0x20 +[ 631.544056] +[ 631.546310] RIP: 0010:__srcu_read_lock+0xec/0x160 +[ 631.570098] RSP: 0018:ffff8801bf2bf458 EFLAGS: 00000206 ORIG_RAX: ffffffffffffff13 +[ 631.577828] RAX: 0000000000000001 RBX: 1ffff10037e57e90 RCX: 0000000000000001 +[ 631.585112] RDX: 000060fe24eb2dc0 RSI: 0000000000000000 RDI: ffffc90001fd2c28 +[ 631.592397] RBP: ffff8801bf2bf4e0 R08: ffff8801c0b82080 R09: 0000000000000006 +[ 631.599676] R10: 0000000000000000 R11: ffff8801c0b82080 R12: 1ffff10037e57e8c +[ 631.606953] R13: ffffc90001fd27e0 R14: ffff8801bf2bf4c0 R15: 0000000000000000 +[ 631.622702] vcpu_enter_guest+0x1309/0x62e0 +[ 631.711508] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 631.721422] kvm_vcpu_ioctl+0x72b/0x1150 +[ 631.816167] do_vfs_ioctl+0x1de/0x1720 +[ 631.887384] ksys_ioctl+0xa9/0xd0 +[ 631.890862] __x64_sys_ioctl+0x73/0xb0 +[ 631.894796] do_syscall_64+0x1b9/0x820 +[ 631.933977] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 631.939197] RIP: 0033:0x457099 +[ 631.961342] RSP: 002b:00007ffeca5f9b58 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 631.969086] RAX: ffffffffffffffda RBX: 0000000002395914 RCX: 0000000000457099 +[ 631.976385] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 631.983681] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 631.990979] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 631.998286] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/274 b/pkg/report/testdata/linux/report/274 new file mode 100644 index 00000000000..d20c7e56a44 --- /dev/null +++ b/pkg/report/testdata/linux/report/274 @@ -0,0 +1,60 @@ +TITLE: INFO: rcu detected stall in ipv6_rcv + +[ 842.597024] INFO: rcu_sched self-detected stall on CPU +[ 842.602466] 0-....: (1 GPs behind) idle=222/1/4611686018427387906 softirq=196202/196203 fqs=31202 +[ 842.611869] (t=125000 jiffies g=95129 c=95128 q=2548) +[ 842.617218] NMI backtrace for cpu 0 +[ 842.620820] CPU: 0 PID: 6823 Comm: syz-executor6 Not tainted 4.16.0+ #288 +[ 842.627718] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 842.637043] Call Trace: +[ 842.639601] +[ 842.641730] dump_stack+0x1a7/0x27d +[ 842.662780] nmi_cpu_backtrace+0x1e0/0x220 +[ 842.671464] nmi_trigger_cpumask_backtrace+0x123/0x180 +[ 842.676716] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 842.681881] rcu_dump_cpu_stacks+0x186/0x1d3 +[ 842.686271] check_cpu_stall.isra.63+0xbb8/0x15b0 +[ 842.762673] rcu_check_callbacks+0x21b/0xae0 +[ 842.815807] update_process_times+0x30/0x60 +[ 842.820105] tick_sched_handle+0x85/0x160 +[ 842.824229] tick_sched_timer+0x42/0x120 +[ 842.828264] __hrtimer_run_queues+0x39c/0xec0 +[ 842.874887] hrtimer_interrupt+0x2a5/0x6f0 +[ 842.879110] smp_apic_timer_interrupt+0x14a/0x700 +[ 842.914536] apic_timer_interrupt+0xf/0x20 +[ 842.918740] +[ 842.920956] RIP: 0010:__sanitizer_cov_trace_pc+0x0/0x50 +[ 842.926292] RSP: 0018:ffff8801aabde340 EFLAGS: 00000287 ORIG_RAX: ffffffffffffff12 +[ 842.933978] RAX: 0000000000000007 RBX: ffff8801c81331c0 RCX: 000000000000fde1 +[ 842.941221] RDX: 0000000000000000 RSI: 0000000000004699 RDI: ffff8801c8133244 +[ 842.948465] RBP: ffff8801aabde3d0 R08: 0000000000000000 R09: ffff8801aabde6a8 +[ 842.955718] R10: 000000000000000b R11: ffffed003557bcdf R12: 000000000000fde1 +[ 842.962962] R13: ffff8801aabde6a8 R14: 00000000000046c8 R15: dffffc0000000000 +[ 842.979432] __xfrm_decode_session+0x68/0x110 +[ 842.983904] __xfrm_policy_check+0x19e/0x2300 +[ 843.040856] ip6_input_finish+0x1288/0x17b0 +[ 843.068947] ip6_input+0xdb/0x560 +[ 843.089410] ip6_mc_input+0x3a8/0xb10 +[ 843.096979] ip6_rcv_finish+0x297/0x8c0 +[ 843.121561] ipv6_rcv+0xf45/0x1fc0 +[ 843.145285] __netif_receive_skb_core+0x1a52/0x3480 +[ 843.236444] __netif_receive_skb+0x2c/0x1b0 +[ 843.245211] netif_receive_skb_internal+0x10b/0x670 +[ 843.266287] netif_receive_skb+0xae/0x390 +[ 843.288664] tun_rx_batched.isra.50+0x5ee/0x870 +[ 843.306989] tun_get_user+0x299b/0x3b50 +[ 843.363324] tun_chr_write_iter+0xb9/0x160 +[ 843.367536] do_iter_readv_writev+0x55c/0x830 +[ 843.380795] do_iter_write+0x154/0x540 +[ 843.392735] compat_writev+0x225/0x420 +[ 843.420428] do_compat_writev+0x115/0x220 +[ 843.440528] compat_SyS_writev+0x26/0x30 +[ 843.448867] do_fast_syscall_32+0x3ec/0xf9f +[ 843.486226] entry_SYSENTER_compat+0x70/0x7f +[ 843.490607] RIP: 0023:0xf7ff2c99 +[ 843.493944] RSP: 002b:00000000f5fee044 EFLAGS: 00000292 ORIG_RAX: 0000000000000092 +[ 843.501630] RAX: ffffffffffffffda RBX: 00000000000000fc RCX: 00000000f5fee094 +[ 843.508882] RDX: 0000000000000001 RSI: 000000000813af00 RDI: 00000000ffffffff +[ 843.516133] RBP: 00000000f5fee158 R08: 0000000000000000 R09: 0000000000000000 +[ 843.523377] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 +[ 843.530621] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/275 b/pkg/report/testdata/linux/report/275 new file mode 100644 index 00000000000..bdeb527fd9c --- /dev/null +++ b/pkg/report/testdata/linux/report/275 @@ -0,0 +1,45 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_release + +[ 401.753130] rcu: INFO: rcu_sched detected stalls on CPUs/tasks: +[ 401.759317] rcu: (detected by 0, t=105002 jiffies, g=89793, q=195) +[ 401.765737] rcu: All QSes seen, last rcu_sched kthread activity 105002 (4295067669-4294962667), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 401.778318] syz-executor5 R running task 24488 9936 4748 0x00000008 +[ 401.785560] Call Trace: +[ 401.788141] +[ 401.790301] sched_show_task.cold.85+0x273/0x2d5 +[ 401.821799] print_other_cpu_stall.cold.79+0x8e0/0x91d +[ 401.860568] rcu_check_callbacks+0x986/0x15a0 +[ 401.964031] update_process_times+0x2d/0x70 +[ 401.968355] tick_sched_handle+0x9f/0x180 +[ 401.972517] tick_sched_timer+0x45/0x130 +[ 401.976581] __hrtimer_run_queues+0x3eb/0xff0 +[ 402.026193] hrtimer_interrupt+0x2f3/0x750 +[ 402.030445] smp_apic_timer_interrupt+0x16d/0x6a0 +[ 402.083022] apic_timer_interrupt+0xf/0x20 +[ 402.087252] +[ 402.089510] RIP: 0010:smp_call_function_single+0x3f5/0x5c0 +[ 402.114059] RSP: 0018:ffff8801cfb8f3a0 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 402.121773] RAX: ffff8801cfb8f418 RBX: ffff8801cfb8f418 RCX: ffffffff8170865c +[ 402.129044] RDX: 0000000000000000 RSI: ffffffff8170860b RDI: 0000000000000005 +[ 402.136314] RBP: ffff8801cfb8f4f0 R08: ffff8801bcb92140 R09: ffffed003b625b98 +[ 402.143582] R10: ffffed003b625b98 R11: ffff8801db12dcc7 R12: ffffed0039f71e90 +[ 402.150856] R13: dffffc0000000000 R14: 0000000000000000 R15: 0000000000000001 +[ 402.191483] vmx_vcpu_load+0x873/0xfe0 +[ 402.235986] kvm_arch_vcpu_load+0x22b/0x940 +[ 402.244821] vcpu_load+0x32/0x40 +[ 402.248189] kvm_arch_destroy_vm+0x1f9/0x7c0 +[ 402.284984] kvm_put_kvm+0x73f/0x1060 +[ 402.317058] kvm_vcpu_release+0x7b/0xa0 +[ 402.321039] __fput+0x38a/0xa40 +[ 402.354473] ____fput+0x15/0x20 +[ 402.357755] task_work_run+0x1e8/0x2a0 +[ 402.374215] exit_to_usermode_loop+0x318/0x380 +[ 402.389202] do_syscall_64+0x6be/0x820 +[ 402.423222] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 402.428415] RIP: 0033:0x410c41 +[ 402.450537] RSP: 002b:00007ffc6fee9a70 EFLAGS: 00000293 ORIG_RAX: 0000000000000003 +[ 402.458282] RAX: 0000000000000000 RBX: 000000000000000b RCX: 0000000000410c41 +[ 402.465556] RDX: 0000000000000000 RSI: 00000000007305f0 RDI: 000000000000000a +[ 402.472826] RBP: 0000000000000000 R08: ffffffffffffffff R09: ffffffffffffffff +[ 402.480102] R10: 00007ffc6fee99a0 R11: 0000000000000293 R12: 0000000000000009 +[ 402.487376] R13: 00000000000485c6 R14: 0000000000000052 R15: badc0ffeebadface diff --git a/pkg/report/testdata/linux/report/276 b/pkg/report/testdata/linux/report/276 new file mode 100644 index 00000000000..037de1360c9 --- /dev/null +++ b/pkg/report/testdata/linux/report/276 @@ -0,0 +1,46 @@ +TITLE: INFO: rcu detected stall in __cleanup_mnt + +[ 396.319024] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 396.324708] (detected by 0, t=105007 jiffies, g=54385, c=54384, q=120) +[ 396.331479] All QSes seen, last rcu_sched kthread activity 105014 (4295063628-4294958614), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 396.343605] syz-executor5 R running task 25224 22239 4482 0x00000008 +[ 396.350812] Call Trace: +[ 396.353393] +[ 396.355560] sched_show_task.cold.84+0x27a/0x301 +[ 396.386938] print_other_cpu_stall.cold.77+0x92f/0x9d2 +[ 396.425896] check_cpu_stall.isra.60+0x721/0xf70 +[ 396.497958] rcu_check_callbacks+0x23f/0xcd0 +[ 396.575053] update_process_times+0x2d/0x70 +[ 396.579598] tick_sched_handle+0x9f/0x180 +[ 396.583746] tick_sched_timer+0x45/0x130 +[ 396.587826] __hrtimer_run_queues+0x3eb/0x10c0 +[ 396.636316] hrtimer_interrupt+0x2f3/0x750 +[ 396.640571] smp_apic_timer_interrupt+0x165/0x730 +[ 396.668106] apic_timer_interrupt+0xf/0x20 +[ 396.672346] +[ 396.674589] RIP: 0010:fuse_abort_conn+0xb1f/0x1580 +[ 396.698893] RSP: 0018:ffff880198357618 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 396.706602] RAX: 0000000000040000 RBX: dffffc0000000000 RCX: ffffc9000d4c7000 +[ 396.713886] RDX: 0000000000040000 RSI: ffffffff8267f418 RDI: ffff8801d7f27270 +[ 396.721154] RBP: ffff8801983579e0 R08: ffffed003afe4e50 R09: ffffed003afe4e4f +[ 396.728422] R10: ffffed003afe4e4f R11: ffff8801d7f2727b R12: ffff8801ae944108 +[ 396.735689] R13: ffff880198357738 R14: ffff8801983579b8 R15: dffffc0000000000 +[ 396.815744] fuse_put_super+0x123/0x440 +[ 396.841988] generic_shutdown_super+0x1bf/0x530 +[ 396.861913] kill_anon_super+0x3c/0x50 +[ 396.865801] fuse_kill_sb_anon+0x90/0xb0 +[ 396.869866] deactivate_locked_super+0x97/0x100 +[ 396.874546] deactivate_super+0x193/0x1c0 +[ 396.882779] cleanup_mnt+0xbf/0x160 +[ 396.886406] __cleanup_mnt+0x16/0x20 +[ 396.890123] task_work_run+0x1ec/0x2a0 +[ 396.907192] exit_to_usermode_loop+0x313/0x370 +[ 396.920274] do_syscall_64+0x6be/0x820 +[ 396.948792] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 396.953981] RIP: 0033:0x456cb9 +[ 396.976544] RSP: 002b:00007fc8f9ed9c78 EFLAGS: 00000246 ORIG_RAX: 00000000000000a6 +[ 396.984286] RAX: 0000000000000000 RBX: 00007fc8f9eda6d4 RCX: 0000000000456cb9 +[ 396.991568] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000020000100 +[ 396.998834] RBP: 0000000000930280 R08: 0000000000000000 R09: 0000000000000000 +[ 397.006100] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 397.013366] R13: 00000000004d6360 R14: 00000000004c9ab8 R15: 0000000000000003 diff --git a/pkg/report/testdata/linux/report/277 b/pkg/report/testdata/linux/report/277 new file mode 100644 index 00000000000..fa3d208d031 --- /dev/null +++ b/pkg/report/testdata/linux/report/277 @@ -0,0 +1,54 @@ +TITLE: INFO: rcu detected stall in ext4_file_write_iter + +[ 587.513972] rcu: INFO: rcu_sched self-detected stall on CPU +[ 587.520042] rcu: 0-....: (105000 ticks this GP) idle=76a/1/0x4000000000000002 softirq=144952/144952 fqs=26032 +[ 587.530723] rcu: (t=105010 jiffies g=334085 q=1885) +[ 587.535924] NMI backtrace for cpu 0 +[ 587.539556] CPU: 0 PID: 30128 Comm: syz-executor6 Not tainted 4.18.0+ #197 +[ 587.546568] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 587.555920] Call Trace: +[ 587.558509] +[ 587.560674] dump_stack+0x1c9/0x2b4 +[ 587.578675] nmi_cpu_backtrace.cold.3+0x48/0x88 +[ 587.588548] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 587.593830] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 587.599018] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 587.608455] print_cpu_stall.cold.78+0x2fb/0x59c +[ 587.638910] rcu_check_callbacks+0xd93/0x1660 +[ 587.732161] update_process_times+0x2d/0x70 +[ 587.736486] tick_sched_handle+0x9f/0x180 +[ 587.740649] tick_sched_timer+0x45/0x130 +[ 587.744711] __hrtimer_run_queues+0x3eb/0x10c0 +[ 587.785214] hrtimer_interrupt+0x2f3/0x750 +[ 587.789483] smp_apic_timer_interrupt+0x16d/0x730 +[ 587.817071] apic_timer_interrupt+0xf/0x20 +[ 587.821308] +[ 587.823563] RIP: 0010:lock_acquire+0x25f/0x540 +[ 587.847050] RSP: 0018:ffff8801912ee5e8 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13 +[ 587.854764] RAX: dffffc0000000000 RBX: 1ffff1003225dcc2 RCX: 0000000000000000 +[ 587.862032] RDX: 1ffffffff0fe374d RSI: 0000000000000000 RDI: 0000000000000282 +[ 587.869301] RBP: ffff8801912ee6d8 R08: 0000000000000008 R09: 0000000000000004 +[ 587.876569] R10: ffff8801bb228d58 R11: 5f54006f34535a0e R12: ffff8801bb228480 +[ 587.883836] R13: 0000000000000002 R14: 0000000000000000 R15: 0000000000000000 +[ 587.903344] balance_dirty_pages_ratelimited+0x187f/0x2200 +[ 587.991035] generic_perform_write+0x551/0x6c0 +[ 588.027011] __generic_file_write_iter+0x26e/0x630 +[ 588.031952] ext4_file_write_iter+0x390/0x1450 +[ 588.082118] __vfs_write+0x6af/0x9d0 +[ 588.102237] __kernel_write+0x10c/0x380 +[ 588.106225] do_acct_process+0x1148/0x1660 +[ 588.118181] acct_process+0x6b9/0x87d +[ 588.151604] do_exit+0x1afd/0x2760 +[ 588.255303] do_group_exit+0x177/0x440 +[ 588.277377] get_signal+0x88e/0x1970 +[ 588.310755] do_signal+0x9c/0x21c0 +[ 588.350143] exit_to_usermode_loop+0x2e5/0x380 +[ 588.365138] do_syscall_64+0x6be/0x820 +[ 588.397707] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 588.402900] RIP: 0033:0x457089 +[ 588.425533] RSP: 002b:00007fe66d9d4c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000003 +[ 588.433254] RAX: 0000000000000000 RBX: 00007fe66d9d56d4 RCX: 0000000000457089 +[ 588.440538] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000004 +[ 588.447815] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 588.455086] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 588.462371] R13: 00000000004cb460 R14: 00000000004ee072 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/278 b/pkg/report/testdata/linux/report/278 new file mode 100644 index 00000000000..9c27db5cfff --- /dev/null +++ b/pkg/report/testdata/linux/report/278 @@ -0,0 +1,59 @@ +TITLE: INFO: rcu detected stall in sys_futex + +[ 246.173515] rcu: INFO: rcu_sched self-detected stall on CPU +[ 246.179319] rcu: 1-...!: (1 ticks this GP) idle=2e6/1/0x4000000000000002 softirq=21091/21091 fqs=0 +[ 246.188580] rcu: (t=110325 jiffies g=45621 q=219) +[ 246.193603] rcu: rcu_sched kthread starved for 110325 jiffies! g45621 f0x0 RCU_GP_WAIT_FQS(5) ->state=0x402 ->cpu=1 +[ 246.204159] rcu: RCU grace-period kthread stack dump: +[ 246.209334] rcu_sched I23480 10 2 0x80000000 +[ 246.214997] Call Trace: +[ 246.217708] __schedule+0x87c/0x1ec0 +[ 246.247401] schedule+0xfb/0x450 +[ 246.277246] schedule_timeout+0x140/0x260 +[ 246.294402] rcu_gp_kthread+0x743/0x1d20 +[ 246.481639] kthread+0x35a/0x420 +[ 246.493143] ret_from_fork+0x3a/0x50 +[ 246.496878] NMI backtrace for cpu 1 +[ 246.500524] CPU: 1 PID: 4356 Comm: syz-fuzzer Not tainted 4.18.0-rc7-next-20180803+ #31 +[ 246.508667] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 246.518010] Call Trace: +[ 246.520578] +[ 246.522762] dump_stack+0x1c9/0x2b4 +[ 246.540779] nmi_cpu_backtrace.cold.3+0x48/0x88 +[ 246.550627] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 246.555897] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 246.561086] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 246.570531] print_cpu_stall.cold.78+0x2fb/0x59c +[ 246.596748] rcu_check_callbacks+0xd93/0x1660 +[ 246.719004] update_process_times+0x2d/0x70 +[ 246.723352] tick_sched_handle+0x9f/0x180 +[ 246.727495] tick_sched_timer+0x45/0x130 +[ 246.731565] __hrtimer_run_queues+0x3eb/0x10c0 +[ 246.776185] hrtimer_interrupt+0x2f3/0x750 +[ 246.780455] smp_apic_timer_interrupt+0x165/0x730 +[ 246.807990] apic_timer_interrupt+0xf/0x20 +[ 246.812208] +[ 246.814441] RIP: 0010:_raw_spin_unlock_irq+0x56/0x70 +[ 246.838429] RSP: 0018:ffff8801ad6ef090 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 246.846148] RAX: dffffc0000000000 RBX: ffff8801db12ca40 RCX: ffffffff81606487 +[ 246.853406] RDX: 1ffffffff0fe361f RSI: 0000000000000004 RDI: ffffffff87f1b0f8 +[ 246.860677] RBP: ffff8801ad6ef098 R08: ffffed003b625949 R09: ffffed003b625948 +[ 246.867934] R10: ffffed003b625948 R11: ffff8801db12ca43 R12: ffff8801ad502180 +[ 246.875212] R13: ffff8801d9f18380 R14: ffff88019c2e98c0 R15: dffffc0000000000 +[ 246.886921] finish_task_switch+0x1d3/0x870 +[ 246.965692] __schedule+0x884/0x1ec0 +[ 247.014854] schedule+0xfb/0x450 +[ 247.049482] futex_wait_queue_me+0x3f9/0x840 +[ 247.100651] futex_wait+0x45b/0xa20 +[ 247.125352] do_futex+0x336/0x27d0 +[ 247.229783] __x64_sys_futex+0x472/0x6a0 +[ 247.247689] do_syscall_64+0x1b9/0x820 +[ 247.280793] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 247.285980] RIP: 0033:0x45ddf3 +[ 247.308057] RSP: 002b:000000c420035e80 EFLAGS: 00000246 ORIG_RAX: 00000000000000ca +[ 247.315756] RAX: ffffffffffffffda RBX: 0000000000000000 RCX: 000000000045ddf3 +2018/08/03 14:36:29 Manager.Poll call failed: connection is shut down +[ 247.323013] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000001448298 +[ 247.330271] RBP: 000000c420035ec8 R08: 0000000000000000 R09: 0000000000000000 +[ 247.337529] R10: 000000c420035eb8 R11: 0000000000000246 R12: 0000000000430130 +[ 247.344791] R13: 00000000000000f1 R14: 0000000000000011 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/279 b/pkg/report/testdata/linux/report/279 new file mode 100644 index 00000000000..50d232b8e66 --- /dev/null +++ b/pkg/report/testdata/linux/report/279 @@ -0,0 +1,47 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 1057.417927] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 1057.424666] rcu: (detected by 1, t=10502 jiffies, g=7113, q=509) +[ 1057.431125] rcu: All QSes seen, last rcu_preempt kthread activity 10498 (4295042895-4295032397), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 1057.444299] syz-executor0 R running task 22808 7363 5346 0x00000000 +[ 1057.451771] Call Trace: +[ 1057.454712] +[ 1057.457168] sched_show_task.cold.83+0x2b6/0x30a +[ 1057.495301] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 1057.554036] rcu_check_callbacks+0xafc/0x1990 +[ 1057.687133] update_process_times+0x2d/0x70 +[ 1057.691573] tick_sched_handle+0x9f/0x180 +[ 1057.695766] tick_sched_timer+0x45/0x130 +[ 1057.699875] __hrtimer_run_queues+0x41c/0x10d0 +[ 1057.749858] hrtimer_interrupt+0x313/0x780 +[ 1057.754284] smp_apic_timer_interrupt+0x1a1/0x760 +[ 1057.821414] apic_timer_interrupt+0xf/0x20 +[ 1057.825660] +[ 1057.827917] RIP: 0010:lock_acquire+0x268/0x520 +[ 1057.851437] RSP: 0018:ffff8801c664eb20 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff13 +[ 1057.859184] RAX: dffffc0000000000 RBX: 1ffff10038cc9d69 RCX: 0000000000000000 +[ 1057.866490] RDX: 1ffffffff12a4535 RSI: 0000000000000000 RDI: 0000000000000282 +[ 1057.873788] RBP: ffff8801c664ec10 R08: ffff8801c10e6e70 R09: 0000000000000002 +[ 1057.881090] R10: ffff8801c10e6e50 R11: f333abdc872bc457 R12: ffff8801c10e6580 +[ 1057.888382] R13: 0000000000000000 R14: 0000000000000001 R15: 0000000000000000 +[ 1057.916107] _raw_spin_lock+0x2d/0x40 +[ 1057.925256] kvm_mmu_unprotect_page+0xf9/0x450 +[ 1057.938957] x86_emulate_instruction+0x635/0x1fc0 +[ 1057.970189] kvm_mmu_page_fault+0x396/0x1b30 +[ 1058.076393] handle_ept_violation+0x29e/0x6a0 +[ 1058.092996] vmx_handle_exit+0x2f7/0x17e0 +[ 1058.133773] vcpu_enter_guest+0x14a9/0x62e0 +[ 1058.232876] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 1058.242845] kvm_vcpu_ioctl+0x72b/0x1150 +[ 1058.347536] do_vfs_ioctl+0x1de/0x1720 +[ 1058.404376] ksys_ioctl+0xa9/0xd0 +[ 1058.407861] __x64_sys_ioctl+0x73/0xb0 +[ 1058.411768] do_syscall_64+0x1b9/0x820 +[ 1058.450911] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1058.456119] RIP: 0033:0x457099 +[ 1058.478267] RSP: 002b:00007fff8bfb2748 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 1058.486005] RAX: ffffffffffffffda RBX: 000000000212b914 RCX: 0000000000457099 +[ 1058.493299] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 1058.500585] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 1058.507875] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1058.515159] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/280 b/pkg/report/testdata/linux/report/280 new file mode 100644 index 00000000000..6900184cd5c --- /dev/null +++ b/pkg/report/testdata/linux/report/280 @@ -0,0 +1,49 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 885.637820] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 885.644147] rcu: (detected by 1, t=10502 jiffies, g=132945, q=115) +[ 885.650574] rcu: All QSes seen, last rcu_preempt kthread activity 10498 (4295025713-4295015215), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 885.663229] syz-executor7 R running task 21280 8024 5432 0x00000002 +[ 885.670447] Call Trace: +[ 885.673034] +[ 885.675201] sched_show_task.cold.83+0x2b6/0x30a +[ 885.711132] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 885.764802] rcu_check_callbacks+0xafc/0x1990 +[ 885.897413] update_process_times+0x2d/0x70 +[ 885.901759] tick_sched_handle+0x9f/0x180 +[ 885.905940] tick_sched_timer+0x45/0x130 +[ 885.910010] __hrtimer_run_queues+0x41c/0x10d0 +[ 885.966004] hrtimer_interrupt+0x313/0x780 +[ 885.970274] smp_apic_timer_interrupt+0x1a1/0x760 +[ 886.031595] apic_timer_interrupt+0xf/0x20 +[ 886.035836] +[ 886.038088] RIP: 0010:vmx_read_guest_seg_ar+0x127/0x270 +[ 886.062366] RSP: 0018:ffff8801a035ea40 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 +[ 886.070085] RAX: 0000000000000093 RBX: 0000000000000002 RCX: ffffc90004d15000 +[ 886.077363] RDX: 0000000000004818 RSI: ffffffff811c3301 RDI: ffffffff8803e9ac +[ 886.084637] RBP: ffff8801a035ea68 R08: ffff8801bfe0a140 R09: ffff8801c1f7c480 +[ 886.091910] R10: ffffed00383ef891 R11: ffff8801c1f7c48b R12: ffff8801c1f78a80 +[ 886.099184] R13: 0000000000004818 R14: 0000000000000000 R15: 0000000000100000 +[ 886.111255] vmx_get_cpl+0x5b/0x90 +[ 886.114813] kvm_fetch_guest_virt+0x65/0x1c0 +[ 886.123486] __do_insn_fetch_bytes+0x45b/0x950 +[ 886.136686] x86_decode_insn+0x1544/0x54c0 +[ 886.149892] x86_emulate_instruction+0x833/0x1fc0 +[ 886.171431] kvm_mmu_page_fault+0x396/0x1b30 +[ 886.299270] handle_ept_violation+0x29e/0x6a0 +[ 886.320472] vmx_handle_exit+0x2f7/0x17e0 +[ 886.345943] vcpu_enter_guest+0x14a9/0x62e0 +[ 886.447583] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 886.457476] kvm_vcpu_ioctl+0x72b/0x1150 +[ 886.510921] do_vfs_ioctl+0x1de/0x1720 +[ 886.567519] ksys_ioctl+0xa9/0xd0 +[ 886.570985] __x64_sys_ioctl+0x73/0xb0 +[ 886.574886] do_syscall_64+0x1b9/0x820 +[ 886.619519] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 886.624716] RIP: 0033:0x457099 +[ 886.646837] RSP: 002b:00007f84c55d1c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 886.654558] RAX: ffffffffffffffda RBX: 00007f84c55d26d4 RCX: 0000000000457099 +[ 886.661834] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000006 +[ 886.669108] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 886.676383] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 886.683661] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/281 b/pkg/report/testdata/linux/report/281 new file mode 100644 index 00000000000..6a6616ef729 --- /dev/null +++ b/pkg/report/testdata/linux/report/281 @@ -0,0 +1,55 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 557.780627] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 557.786921] rcu: (detected by 0, t=10502 jiffies, g=52381, q=77) +[ 557.793179] rcu: All QSes seen, last rcu_preempt kthread activity 10503 (4294992931-4294982428), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 557.805832] syz-executor1 R running task 21584 17898 5513 0x00000000 +[ 557.813042] Call Trace: +[ 557.815629] +[ 557.817796] sched_show_task.cold.83+0x2b6/0x30a +[ 557.853686] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 557.919480] rcu_check_callbacks+0xafc/0x1990 +[ 558.059280] update_process_times+0x2d/0x70 +[ 558.063624] tick_sched_handle+0x9f/0x180 +[ 558.067785] tick_sched_timer+0x45/0x130 +[ 558.071860] __hrtimer_run_queues+0x41c/0x10d0 +[ 558.126464] hrtimer_interrupt+0x313/0x780 +[ 558.130798] smp_apic_timer_interrupt+0x1a1/0x760 +[ 558.191678] apic_timer_interrupt+0xf/0x20 +[ 558.195920] +[ 558.198167] RIP: 0010:do_error_trap+0x270/0x4d0 +[ 558.221781] RSP: 0018:ffff88018c7f68e8 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 558.229505] RAX: dffffc0000000000 RBX: ffff88018c7f6a38 RCX: 1ffff10031358d4c +[ 558.236780] RDX: 1ffffffff12a4537 RSI: ffff880189ac6a68 RDI: ffffffff895229b8 +[ 558.244058] RBP: ffff88018c7f6a18 R08: ffff880189ac6a60 R09: 0000000000000006 +[ 558.251343] R10: 0000000000000000 R11: ffff880189ac61c0 R12: 0000000000000000 +[ 558.258622] R13: ffff88018c7f69f0 R14: 0000000000000008 R15: 1ffff100318fed22 +[ 558.311744] do_divide_error+0x18/0x20 +[ 558.315648] divide_error+0x14/0x20 +[ 558.319295] RIP: 0010:em_idiv_ex+0x0/0x8 +[ 558.342278] RSP: 0018:ffff88018c7f6ae0 EFLAGS: 00010202 +[ 558.347666] RAX: 0000000000000000 RBX: ffff88018ae6c960 RCX: 0000000000000000 +[ 558.354939] RDX: 0000000000000000 RSI: ffffffff811474e8 RDI: 0000000000000200 +[ 558.362215] RBP: ffff88018c7f6b18 R08: ffff880189ac61c0 R09: 1ffffffff12b43cd +[ 558.369489] R10: ffffed003b5c4732 R11: ffff8801dae23993 R12: 0000000000000000 +[ 558.376760] R13: ffffffff811474e8 R14: 0000000000000001 R15: ffffffff811474e8 +[ 558.401977] x86_emulate_insn+0x1efd/0x5020 +[ 558.437698] x86_emulate_instruction+0x665/0x1f90 +[ 558.460608] kvm_mmu_page_fault+0x342/0x1ad0 +[ 558.570767] handle_ept_violation+0x29e/0x6a0 +[ 558.587169] vmx_handle_exit+0x2f7/0x17f0 +[ 558.603946] vcpu_enter_guest+0x14a9/0x62e0 +[ 558.687489] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 558.697387] kvm_vcpu_ioctl+0x72b/0x1150 +[ 558.762259] do_vfs_ioctl+0x1de/0x1720 +[ 558.790202] ksys_ioctl+0xa9/0xd0 +[ 558.793668] __x64_sys_ioctl+0x73/0xb0 +[ 558.797566] do_syscall_64+0x1b9/0x820 +[ 558.842137] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 558.847341] RIP: 0033:0x457099 +[ 558.869448] RSP: 002b:00007fe96d7efc78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 558.877169] RAX: ffffffffffffffda RBX: 00007fe96d7f06d4 RCX: 0000000000457099 +[ 558.884442] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000009 +[ 558.891714] RBP: 0000000000930140 R08: 0000000000000000 R09: 0000000000000000 +[ 558.898989] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 558.906264] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000001 diff --git a/pkg/report/testdata/linux/report/282 b/pkg/report/testdata/linux/report/282 new file mode 100644 index 00000000000..35fefb6ba83 --- /dev/null +++ b/pkg/report/testdata/linux/report/282 @@ -0,0 +1,43 @@ +TITLE: INFO: rcu detected stall in kvm_vcpu_ioctl + +[ 618.102956] overlayfs: failed to resolve './file1': -2 +[ 724.054784] rcu: INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 724.061136] rcu: (detected by 1, t=10502 jiffies, g=71473, q=34) +[ 724.067416] rcu: All QSes seen, last rcu_preempt kthread activity 10503 (4295009460-4294998957), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 724.080087] syz-executor1 R running task 22672 24999 5368 0x00000000 +[ 724.087341] Call Trace: +[ 724.089931] +[ 724.092107] sched_show_task.cold.83+0x2b6/0x30a +[ 724.128077] print_other_cpu_stall.cold.79+0xa83/0xba5 +[ 724.199489] rcu_check_callbacks+0xafc/0x1990 +[ 724.345408] update_process_times+0x2d/0x70 +[ 724.349750] tick_sched_handle+0x9f/0x180 +[ 724.353919] tick_sched_timer+0x45/0x130 +[ 724.357996] __hrtimer_run_queues+0x41c/0x10d0 +[ 724.412730] hrtimer_interrupt+0x313/0x780 +[ 724.417102] smp_apic_timer_interrupt+0x1a1/0x760 +[ 724.477953] apic_timer_interrupt+0xf/0x20 +[ 724.482196] +[ 724.484449] RIP: 0010:debug_lockdep_rcu_enabled.part.3+0x0/0x60 +[ 724.509432] RSP: 0018:ffff88017575f3a0 EFLAGS: 00000202 ORIG_RAX: ffffffffffffff13 +[ 724.517157] RAX: 0000000000000003 RBX: ffffffff89936580 RCX: 0000000000000002 +[ 724.524434] RDX: 0000000000000001 RSI: ffffffff8120f03f RDI: ffffffff8a314a50 +[ 724.531735] RBP: ffff88017575f3b0 R08: ffff8801bbe52580 R09: 0000000000000001 +[ 724.539010] R10: ffff8801bbe52e28 R11: 473e358cd184502e R12: ffff88017575f4b8 +[ 724.546290] R13: 0000000000000000 R14: ffff880188148400 R15: 0000000000000000 +[ 724.562989] vmx_handle_exit+0x1144/0x17f0 +[ 724.579859] vcpu_enter_guest+0x14a9/0x62e0 +[ 724.673018] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 724.682933] kvm_vcpu_ioctl+0x72b/0x1150 +[ 724.742569] do_vfs_ioctl+0x1de/0x1720 +[ 724.793606] ksys_ioctl+0xa9/0xd0 +[ 724.797074] __x64_sys_ioctl+0x73/0xb0 +[ 724.800988] do_syscall_64+0x1b9/0x820 +[ 724.840073] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 724.845299] RIP: 0033:0x457099 +[ 724.867425] RSP: 002b:00007f0e6289ec78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 724.875170] RAX: ffffffffffffffda RBX: 00007f0e6289f6d4 RCX: 0000000000457099 +[ 724.882446] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000007 +[ 724.889728] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 724.897005] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 724.904285] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/283 b/pkg/report/testdata/linux/report/283 new file mode 100644 index 00000000000..81745919127 --- /dev/null +++ b/pkg/report/testdata/linux/report/283 @@ -0,0 +1,55 @@ +TITLE: INFO: rcu detected stall in snd_pcm_oss_release + +[ 362.024021] INFO: rcu_sched self-detected stall on CPU +[ 362.029432] 1-....: (124999 ticks this GP) idle=f9a/1/4611686018427387906 softirq=73140/73140 fqs=31238 +[ 362.039293] (t=125000 jiffies g=36742 c=36741 q=633) +[ 362.044549] NMI backtrace for cpu 1 +[ 362.048153] CPU: 1 PID: 17059 Comm: syz-executor4 Not tainted 4.16.0+ #4 +[ 362.054963] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 362.064292] Call Trace: +[ 362.066846] +[ 362.068978] dump_stack+0x1b9/0x294 +[ 362.081704] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 362.091512] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 362.096764] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 362.101928] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 362.111309] check_cpu_stall.isra.61.cold.80+0x36c/0x59a +[ 362.172363] rcu_check_callbacks+0x21b/0xad0 +[ 362.230142] update_process_times+0x2d/0x70 +[ 362.234439] tick_sched_handle+0xa0/0x180 +[ 362.238565] tick_sched_timer+0x42/0x130 +[ 362.242605] __hrtimer_run_queues+0x3e3/0x10a0 +[ 362.290776] hrtimer_interrupt+0x286/0x650 +[ 362.294994] smp_apic_timer_interrupt+0x15d/0x710 +[ 362.326302] apic_timer_interrupt+0xf/0x20 +[ 362.330507] +[ 362.332720] RIP: 0010:__sanitizer_cov_trace_const_cmp1+0x1a/0x20 +[ 362.338837] RSP: 0018:ffff8801c7b5ede0 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 362.346520] RAX: 0000000000000000 RBX: ffff8801ce12c8c0 RCX: ffffffff85a1dc9d +[ 362.353762] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 0000000000000001 +[ 362.361011] RBP: ffff8801c7b5ede0 R08: ffff8801c3d94300 R09: 0000000000000006 +[ 362.368261] R10: ffff8801c3d94300 R11: 0000000000000000 R12: 0000000000000000 +[ 362.375505] R13: 0000000000000000 R14: ffff8801ceaaed80 R15: ffff8801ce12cc18 +[ 362.387226] snd_pcm_oss_prepare+0x7d/0x1b0 +[ 362.391529] snd_pcm_oss_write3+0x1a7/0x220 +[ 362.395826] snd_pcm_oss_write2+0x34c/0x460 +[ 362.408634] snd_pcm_oss_sync1+0x332/0x5a0 +[ 362.448533] snd_pcm_oss_sync.isra.29+0x790/0x980 +[ 362.466072] snd_pcm_oss_release+0x214/0x290 +[ 362.475452] __fput+0x34d/0x890 +[ 362.490736] ____fput+0x15/0x20 +[ 362.493989] task_work_run+0x1e4/0x290 +[ 362.512311] do_exit+0x1aee/0x2730 +[ 362.635785] do_group_exit+0x16f/0x430 +[ 362.657106] get_signal+0x886/0x1960 +[ 362.717833] do_signal+0x98/0x2040 +[ 362.764335] exit_to_usermode_loop+0x28a/0x310 +[ 362.781265] do_syscall_64+0x792/0x9d0 +[ 362.813438] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 362.818600] RIP: 0033:0x455259 +[ 362.821763] RSP: 002b:00007fb730463ce8 EFLAGS: 00000246 ORIG_RAX: 00000000000000ca +[ 362.829446] RAX: fffffffffffffe00 RBX: 000000000072bf80 RCX: 0000000000455259 +[ 362.836692] RDX: 0000000000000000 RSI: 0000000000000000 RDI: 000000000072bf80 +[ 362.843936] RBP: 000000000072bf80 R08: 0000000000000000 R09: 000000000072bf58 +[ 362.851177] R10: 0000000000000000 R11: 0000000000000246 R12: 0000000000000000 +[ 362.858424] R13: 00007ffc071d643f R14: 00007fb7304649c0 R15: 0000000000000001 diff --git a/pkg/report/testdata/linux/report/284 b/pkg/report/testdata/linux/report/284 new file mode 100644 index 00000000000..9eec50d2ba8 --- /dev/null +++ b/pkg/report/testdata/linux/report/284 @@ -0,0 +1,51 @@ +TITLE: INFO: rcu detected stall in llcp_sock_sendmsg + +[ 1098.520844] INFO: rcu_sched self-detected stall on CPU +[ 1098.520919] 1-....: (20918 ticks this GP) idle=55a/1/4611686018427387906 softirq=11347/11347 fqs=20240 +[ 1098.520943] (t=125005 jiffies g=5572 c=5571 q=149) +[ 1098.520955] NMI backtrace for cpu 1 +[ 1098.520964] CPU: 1 PID: 4811 Comm: syz-executor0 Not tainted 4.18.0-rc1+ #115 +[ 1098.520969] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1098.521036] Call Trace: +[ 1098.521040] +[ 1098.521119] dump_stack+0x1c9/0x2b4 +[ 1098.521173] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 1098.521197] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 1098.521209] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 1098.521220] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 1098.521245] check_cpu_stall.isra.60.cold.78+0x36c/0x5a6 +[ 1098.521438] rcu_check_callbacks+0x23f/0xcd0 +[ 1098.521611] update_process_times+0x2d/0x70 +[ 1098.521623] tick_sched_handle+0x9f/0x180 +[ 1098.521635] tick_sched_timer+0x45/0x130 +[ 1098.521645] __hrtimer_run_queues+0x3eb/0x10c0 +[ 1098.521777] hrtimer_interrupt+0x2f3/0x750 +[ 1098.521808] smp_apic_timer_interrupt+0x165/0x730 +[ 1098.521901] apic_timer_interrupt+0xf/0x20 +[ 1098.521906] +[ 1098.521915] RIP: 0010:console_unlock+0xc84/0x10b0 +[ 1098.522104] RSP: 0018:ffff8801aab0f358 EFLAGS: 00000293 ORIG_RAX: ffffffffffffff13 +[ 1098.522115] RAX: ffff8801aa2802c0 RBX: 0000000000000200 RCX: 1ffff10035450163 +[ 1098.522121] RDX: 0000000000000000 RSI: ffffffff8162b8fb RDI: 0000000000000293 +[ 1098.522127] RBP: ffff8801aab0f4c0 R08: ffff8801aa280af8 R09: 0000000000000006 +[ 1098.522132] R10: ffff8801aa2802c0 R11: 0000000000000000 R12: 0000000000000000 +[ 1098.522138] R13: ffffffff84ea9880 R14: 0000000000000001 R15: dffffc0000000000 +[ 1098.522273] vprintk_emit+0x6c6/0xdf0 +[ 1098.522366] vprintk_default+0x28/0x30 +[ 1098.522375] vprintk_func+0x7a/0xe7 +[ 1098.522385] printk+0xa7/0xcf +[ 1098.522435] nfc_llcp_send_ui_frame.cold.9+0x18/0x1f +[ 1098.522507] llcp_sock_sendmsg+0x278/0x350 +[ 1098.522528] sock_sendmsg+0xd5/0x120 +[ 1098.522539] ___sys_sendmsg+0x51d/0x930 +[ 1098.522667] __sys_sendmmsg+0x240/0x6f0 +[ 1098.522794] __x64_sys_sendmmsg+0x9d/0x100 +[ 1098.522806] do_syscall_64+0x1b9/0x820 +[ 1098.522882] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1098.522891] RIP: 0033:0x455a99 +[ 1098.523077] RSP: 002b:00007ffec82fb2b8 EFLAGS: 00000246 ORIG_RAX: 0000000000000133 +[ 1098.523087] RAX: ffffffffffffffda RBX: 0000000001992914 RCX: 0000000000455a99 +[ 1098.523093] RDX: 04924924924926b2 RSI: 00000000200026c0 RDI: 0000000000000004 +[ 1098.523099] RBP: 000000000072bea0 R08: 0000000000000000 R09: 0000000000000000 +[ 1098.523104] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1098.523110] R13: 00000000004c0b44 R14: 00000000004d0980 R15: 0000000000000001 diff --git a/pkg/report/testdata/linux/report/285 b/pkg/report/testdata/linux/report/285 new file mode 100644 index 00000000000..213eccf7704 --- /dev/null +++ b/pkg/report/testdata/linux/report/285 @@ -0,0 +1,64 @@ +TITLE: INFO: rcu detected stall in sctp_generate_heartbeat_event + +[ 396.651023] INFO: rcu_sched self-detected stall on CPU +[ 396.656449] 0-...!: (120232 ticks this GP) idle=3da/1/4611686018427387906 softirq=85066/85067 fqs=87 +[ 396.666018] (t=125000 jiffies g=45941 c=45940 q=632561) +[ 396.671551] rcu_sched kthread starved for 96651 jiffies! g45941 c45940 f0x2 RCU_GP_WAIT_FQS(3) ->state=0x0 ->cpu=1 +[ 396.682013] RCU grace-period kthread stack dump: +[ 396.686747] rcu_sched R running task 22584 9 2 0x80000000 +[ 396.693925] Call Trace: +[ 396.696504] __schedule+0x801/0x1e30 +[ 396.721528] schedule+0xef/0x430 +[ 396.760095] schedule_timeout+0x138/0x240 +[ 396.773155] rcu_gp_kthread+0x6b5/0x1940 +[ 396.902194] kthread+0x345/0x410 +[ 396.914410] ret_from_fork+0x3a/0x50 +[ 396.918117] NMI backtrace for cpu 0 +[ 396.921726] CPU: 0 PID: 8 Comm: ksoftirqd/0 Not tainted 4.17.0-rc4+ #48 +[ 396.928465] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 396.937797] Call Trace: +[ 396.940361] +[ 396.942495] dump_stack+0x1b9/0x294 +[ 396.955257] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 396.965085] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 396.970349] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 396.975523] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 396.984915] check_cpu_stall.isra.61.cold.80+0x36c/0x59a +[ 397.045471] rcu_check_callbacks+0x21b/0xad0 +[ 397.099148] update_process_times+0x2d/0x70 +[ 397.103455] tick_sched_handle+0x9f/0x180 +[ 397.107586] tick_sched_timer+0x45/0x130 +[ 397.111633] __hrtimer_run_queues+0x3e3/0x10a0 +[ 397.164906] hrtimer_interrupt+0x2f3/0x750 +[ 397.169132] smp_apic_timer_interrupt+0x15d/0x710 +[ 397.200493] apic_timer_interrupt+0xf/0x20 +[ 397.204703] +[ 397.206926] RIP: 0010:unwind_next_frame.part.7+0x8b/0x9c0 +[ 397.212439] RSP: 0018:ffff8801d9aaebb0 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 397.220131] RAX: 1ffff1003b355da3 RBX: ffff8801d9aaecc8 RCX: 0000000000000007 +[ 397.227380] RDX: dffffc0000000000 RSI: ffffffff86e8b04f RDI: ffff8801d9aaecc8 +[ 397.234633] RBP: ffff8801d9aaeca0 R08: ffff8801d9aaed00 R09: ffff8801d9a9c200 +[ 397.241882] R10: ffffed003b355da3 R11: ffff8801d9aaed1f R12: 1ffff1003b355d7b +[ 397.249135] R13: 0000000000000000 R14: ffff8801d9a9c200 R15: ffff8801d9aaed18 +[ 397.284735] unwind_next_frame+0x3e/0x50 +[ 397.288784] __save_stack_trace+0x6e/0xd0 +[ 397.297403] save_stack_trace+0x1a/0x20 +[ 397.301360] save_stack+0x43/0xd0 +[ 397.415393] kasan_kmalloc+0xc4/0xe0 +[ 397.419094] kasan_slab_alloc+0x12/0x20 +[ 397.423094] kmem_cache_alloc+0x12e/0x760 +[ 397.427240] sctp_chunkify+0xce/0x400 +[ 397.455406] _sctp_make_chunk+0x157/0x280 +[ 397.459538] sctp_make_heartbeat+0x8f/0x430 +[ 397.483422] sctp_sf_heartbeat.isra.23+0x26/0x180 +[ 397.488250] sctp_sf_sendbeat_8_3+0x38e/0x550 +[ 397.492731] sctp_do_sm+0x1ab/0x7160 +[ 397.565731] sctp_generate_heartbeat_event+0x218/0x450 +[ 397.571003] call_timer_fn+0x230/0x940 +[ 397.617386] __run_timers+0x79e/0xc50 +[ 397.668620] run_timer_softirq+0x4c/0x70 +[ 397.672666] __do_softirq+0x2e0/0xaf5 +[ 397.707640] run_ksoftirqd+0x86/0x100 +[ 397.711424] smpboot_thread_fn+0x417/0x870 +[ 397.729086] kthread+0x345/0x410 +[ 397.739836] ret_from_fork+0x3a/0x50 diff --git a/pkg/report/testdata/linux/report/286 b/pkg/report/testdata/linux/report/286 new file mode 100644 index 00000000000..1e625a25d54 --- /dev/null +++ b/pkg/report/testdata/linux/report/286 @@ -0,0 +1,90 @@ +TITLE: INFO: rcu detected stall in sctp_generate_heartbeat_event + +[ 486.569033] INFO: rcu_sched self-detected stall on CPU +[ 486.574490] 0-...!: (124998 ticks this GP) idle=0be/1/4611686018427387908 softirq=15234/15234 fqs=59 +[ 486.584061] (t=125000 jiffies g=7610 c=7609 q=351640) +[ 486.589425] rcu_sched kthread starved for 124739 jiffies! g7610 c7609 f0x2 RCU_GP_WAIT_FQS(3) ->state=0x0 ->cpu=1 +[ 486.599806] RCU grace-period kthread stack dump: +[ 486.604550] rcu_sched R running task 23896 9 2 0x80000000 +[ 486.611759] Call Trace: +[ 486.614337] __schedule+0x801/0x1e30 +[ 486.639334] schedule+0xef/0x430 +[ 486.678051] schedule_timeout+0x138/0x240 +[ 486.690987] rcu_gp_kthread+0x6b5/0x1940 +[ 486.824473] kthread+0x345/0x410 +[ 486.836697] ret_from_fork+0x3a/0x50 +[ 486.840421] NMI backtrace for cpu 0 +[ 486.844038] CPU: 0 PID: 6381 Comm: sh Not tainted 4.17.0-rc5+ #58 +[ 486.850257] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 486.859596] Call Trace: +[ 486.862174] +[ 486.864315] dump_stack+0x1b9/0x294 +[ 486.877074] nmi_cpu_backtrace.cold.4+0x19/0xce +[ 486.886903] nmi_trigger_cpumask_backtrace+0x151/0x192 +[ 486.892167] arch_trigger_cpumask_backtrace+0x14/0x20 +[ 486.897347] rcu_dump_cpu_stacks+0x175/0x1c2 +[ 486.906765] check_cpu_stall.isra.61.cold.80+0x36c/0x59a +[ 486.967369] rcu_check_callbacks+0x21b/0xad0 +[ 487.021142] update_process_times+0x2d/0x70 +[ 487.025452] tick_sched_handle+0x9f/0x180 +[ 487.029583] tick_sched_timer+0x45/0x130 +[ 487.033630] __hrtimer_run_queues+0x3e3/0x10a0 +[ 487.074515] hrtimer_interrupt+0x2f3/0x750 +[ 487.078754] smp_apic_timer_interrupt+0x15d/0x710 +[ 487.098448] apic_timer_interrupt+0xf/0x20 +[ 487.102683] RIP: 0010:lock_acquire+0x257/0x520 +[ 487.107246] RSP: 0000:ffff8801dae06b60 EFLAGS: 00000286 ORIG_RAX: ffffffffffffff13 +[ 487.114937] RAX: dffffc0000000000 RBX: 1ffff1003b5c0d71 RCX: 0000000000000000 +[ 487.122190] RDX: 1ffffffff11a30e5 RSI: ffff8801c7b54c38 RDI: 0000000000000286 +[ 487.129440] RBP: ffff8801dae06c50 R08: 0000000000000008 R09: 0000000000000003 +[ 487.136692] R10: ffff8801c7b54cb0 R11: ffff8801c7b54400 R12: ffff8801c7b54400 +[ 487.143948] R13: 0000000000000002 R14: 0000000000000000 R15: 0000000000000000 +[ 487.173035] is_bpf_text_address+0x3b/0x170 +[ 487.181997] kernel_text_address+0x79/0xf0 +[ 487.190350] __kernel_text_address+0xd/0x40 +[ 487.194668] unwind_get_return_address+0x61/0xa0 +[ 487.199429] __save_stack_trace+0x7e/0xd0 +[ 487.203571] save_stack_trace+0x1a/0x20 +[ 487.207529] save_stack+0x43/0xd0 +[ 487.356403] kasan_kmalloc+0xc4/0xe0 +[ 487.360107] kasan_slab_alloc+0x12/0x20 +[ 487.364754] kmem_cache_alloc_node+0x131/0x780 +[ 487.383127] __alloc_skb+0x111/0x780 +[ 487.426126] _sctp_make_chunk+0x58/0x280 +[ 487.430174] sctp_make_heartbeat+0x8f/0x430 +[ 487.459807] sctp_sf_heartbeat.isra.23+0x26/0x180 +[ 487.464634] sctp_sf_sendbeat_8_3+0x38e/0x550 +[ 487.469117] sctp_do_sm+0x1ab/0x7160 +[ 487.544210] sctp_generate_heartbeat_event+0x218/0x450 +[ 487.549480] call_timer_fn+0x230/0x940 +[ 487.604501] __run_timers+0x79e/0xc50 +[ 487.647862] run_timer_softirq+0x4c/0x70 +[ 487.651907] __do_softirq+0x2e0/0xaf5 +[ 487.696662] irq_exit+0x1d1/0x200 +[ 487.700103] smp_apic_timer_interrupt+0x17e/0x710 +[ 487.731472] apic_timer_interrupt+0xf/0x20 +[ 487.735686] +[ 487.737908] RIP: 0010:rcu_is_watching+0xc4/0x140 +[ 487.742651] RSP: 0000:ffff8801c957f398 EFLAGS: 00000a06 ORIG_RAX: ffffffffffffff13 +[ 487.750347] RAX: 00000000000130be RBX: 1ffff100392afe74 RCX: 1ffff100392afe78 +[ 487.757598] RDX: 0000000000000004 RSI: 0000000000000004 RDI: ffff8801dae23610 +[ 487.764850] RBP: ffff8801c957f428 R08: ffffed003b5c46c3 R09: ffffed003b5c46c2 +[ 487.772101] R10: ffffed003b5c46c2 R11: ffff8801dae23613 R12: ffff8801c957f3c0 +[ 487.779353] R13: ffff8801dae23610 R14: ffff8801c957f400 R15: dffffc0000000000 +[ 487.804394] rcu_read_lock_sched_held+0x8d/0x120 +[ 487.809139] kmem_cache_alloc+0x5fa/0x760 +[ 487.821900] __anon_vma_prepare+0x3b3/0x700 +[ 487.838290] wp_page_copy+0xdf1/0x1440 +[ 487.873330] do_wp_page+0x425/0x1990 +[ 487.929184] __handle_mm_fault+0x2996/0x4310 +[ 487.958511] handle_mm_fault+0x53a/0xc70 +[ 487.970662] __do_page_fault+0x60b/0xe40 +[ 487.982996] do_page_fault+0xee/0x8a7 +[ 488.035994] page_fault+0x1e/0x30 +[ 488.039438] RIP: 0033:0x7f4232a6d1fb +[ 488.043133] RSP: 002b:00007fff2aeebd28 EFLAGS: 00010246 +[ 488.048483] RAX: 000000000000000d RBX: 00000000756e6547 RCX: 000000006c65746e +[ 488.055740] RDX: 0000000049656e69 RSI: 0000000000000025 RDI: 0000000000000002 +[ 488.063019] RBP: 00007fff2aeebe20 R08: 0000000000000001 R09: 000000000000002f +[ 488.070280] R10: 00007f4232a6caf0 R11: 00007fff2aeebb70 R12: 00007f4232a4e000 +[ 488.077543] R13: 00007f4232dd5038 R14: 00007f4232a6cac0 R15: 00007f4232dd5d98 diff --git a/pkg/report/testdata/linux/report/287 b/pkg/report/testdata/linux/report/287 new file mode 100644 index 00000000000..962845d9f7f --- /dev/null +++ b/pkg/report/testdata/linux/report/287 @@ -0,0 +1,53 @@ +TITLE: INFO: rcu detected stall in br_handle_frame + +[ 268.109997] bridge0: received packet on bridge_slave_0 with own address as source address +[ 268.110510] bridge0: received packet on bridge_slave_0 with own address as source address +[ 268.111039] bridge0: received packet on bridge_slave_0 with own address as source address +[ 268.638278] INFO: rcu_sched self-detected stall on CPU +[ 268.643724] 1-...: (10500 ticks this GP) idle=7fb/140000000000001/0 softirq=7363/7366 fqs=2223 +[ 268.652635] (t=10500 jiffies g=2314 c=2313 q=216) +[ 268.657826] Task dump for CPU 1: +[ 268.661186] kworker/u4:2 R running task 27384 2030 2 0x00000008 +[ 268.668655] Workqueue: netns cleanup_net +[ 268.672841] ffff8801ca698000 ffff8801db7072e0 ffffffff813eb3e6 1ffff1003b6e3ebf +[ 268.680895] ffff8801db71ecc0 0000000000000001 dffffc0000000000 ffffffff832dcec0 +[ 268.688951] ffffffff832dcf44 ffff8801db7072f0 ffffffff813eb402 ffff8801db707310 +[ 268.697003] Call Trace: +[ 268.699573] [] _sched_show_task+0x2d0/0x2dc +[ 268.706289] [] sched_show_task+0x10/0x12 +[ 268.711995] [] dump_cpu_task+0x7e/0x83 +[ 268.717534] [] rcu_dump_cpu_stacks+0x155/0x169 +[ 268.723767] [] rcu_check_callbacks.cold.83+0x5f5/0xc05 +[ 268.730700] [] update_process_times+0x3f/0x70 +[ 268.736853] [] tick_sched_handle.isra.17+0x5a/0xf0 +[ 268.743427] [] tick_sched_timer+0x77/0x130 +[ 268.755533] [] __hrtimer_run_queues+0x36c/0xda0 +[ 268.786608] [] hrtimer_interrupt+0x18b/0x3f0 +[ 268.792660] [] local_apic_timer_interrupt+0x7a/0xa0 +[ 268.799325] [] smp_apic_timer_interrupt+0x81/0xb0 +[ 268.805809] [] apic_timer_interrupt+0xa0/0xb0 +[ 268.823881] [] lockdep_rtnl_is_held+0x1a/0x30 +[ 268.830020] [] br_validate_ipv6+0x779/0xcf0 +[ 268.854769] [] br_nf_pre_routing_ipv6+0x85/0x520 +[ 268.884956] [] br_nf_pre_routing+0x8c2/0x1040 +[ 268.903028] [] nf_iterate+0x187/0x210 +[ 268.908474] [] nf_hook_slow+0x1b5/0x330 +[ 268.931204] [] br_handle_frame+0x732/0xca0 +[ 268.957406] [] __netif_receive_skb_core+0x7b4/0x27e0 +[ 268.982260] [] __netif_receive_skb+0x60/0x1c0 +[ 268.988398] [] process_backlog+0x21d/0x6a0 +[ 269.000323] [] net_rx_action+0x2d9/0xc50 +[ 269.024455] [] __do_softirq+0x231/0x9f2 +[ 269.036557] [] do_softirq_own_stack+0x1c/0x30 +[ 269.042690] [] do_softirq.part.17+0x59/0x60 +[ 269.049401] [] __local_bh_enable_ip+0xb9/0xc0 +[ 269.055544] [] nf_ct_iterate_cleanup+0x10d/0x520 +[ 269.068606] [] nf_ct_l3proto_pernet_unregister+0xb3/0xe0 +[ 269.081669] [] ipv4_net_exit+0x21/0x60 +[ 269.087203] [] ops_exit_list.isra.6+0xb5/0x160 +[ 269.093438] [] cleanup_net+0x31e/0x5e0 +[ 269.110339] [] process_one_work+0x7c9/0x14f0 +[ 269.128745] [] worker_thread+0xde/0xfc0 +[ 269.141289] [] kthread+0x272/0x310 +[ 269.171975] [] ret_from_fork+0x4e/0x80 +[ 273.109547] net_ratelimit: 10606 callbacks suppressed diff --git a/pkg/report/testdata/linux/report/288 b/pkg/report/testdata/linux/report/288 new file mode 100644 index 00000000000..cde967fb20c --- /dev/null +++ b/pkg/report/testdata/linux/report/288 @@ -0,0 +1,89 @@ +TITLE: BUG: soft lockup in kvm_vcpu_release + +[ 1342.342232] watchdog: BUG: soft lockup - CPU#0 stuck for 123s! [syz-executor0:8009] +[ 1342.350104] Modules linked in: +[ 1342.353311] irq event stamp: 32790 +[ 1342.356862] hardirqs last enabled at (32789): [] trace_hardirqs_on_thunk+0x1a/0x1c +[ 1342.366247] hardirqs last disabled at (32790): [] trace_hardirqs_off_thunk+0x1a/0x1c +[ 1342.375718] softirqs last enabled at (4938): [] __do_softirq+0x7ba/0xad8 +[ 1342.384222] softirqs last disabled at (4873): [] irq_exit+0x17f/0x1c0 +[ 1342.393426] CPU: 0 PID: 8009 Comm: syz-executor0 Not tainted 4.19.0-rc2+ #230 +[ 1342.400705] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1342.410083] RIP: 0010:__sanitizer_cov_trace_pc+0x20/0x50 +[ 1342.434452] RSP: 0018:ffff8801b9257390 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 1342.442171] RAX: ffff8801940f4580 RBX: ffff8801b9257418 RCX: ffffffff817284a9 +[ 1342.449440] RDX: 0000000000000000 RSI: ffffffff81728458 RDI: 0000000000000005 +[ 1342.456711] RBP: ffff8801b9257390 R08: ffff8801940f4580 R09: ffffed003b5e5ba0 +[ 1342.463980] R10: ffffed003b5e5ba0 R11: ffff8801daf2dd07 R12: ffffed003724ae90 +[ 1342.471269] R13: 1ffff1003724ae7c R14: dffffc0000000000 R15: 0000000000000001 +[ 1342.478545] FS: 000000000124c940(0000) GS:ffff8801dae00000(0000) knlGS:0000000000000000 +[ 1342.486771] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1342.492651] CR2: 0000000020fea800 CR3: 00000001c17e1000 CR4: 00000000001426f0 +[ 1342.499927] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1342.507203] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 1342.514471] Call Trace: +[ 1342.517069] smp_call_function_single+0x258/0x660 +[ 1342.546284] vmx_vcpu_load+0x8a7/0x1030 +[ 1342.587166] kvm_arch_vcpu_load+0x247/0x970 +[ 1342.606789] vcpu_load+0x35/0x70 +[ 1342.610160] kvm_arch_destroy_vm+0x1f9/0x7c0 +[ 1342.644514] kvm_put_kvm+0x6c8/0xff0 +[ 1342.676490] kvm_vcpu_release+0x7b/0xa0 +[ 1342.680470] __fput+0x385/0xa30 +[ 1342.714127] ____fput+0x15/0x20 +[ 1342.717411] task_work_run+0x1e8/0x2a0 +[ 1342.733866] exit_to_usermode_loop+0x318/0x380 +[ 1342.748873] do_syscall_64+0x6be/0x820 +[ 1342.782821] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1342.788012] RIP: 0033:0x410c51 +[ 1342.810143] RSP: 002b:0000000000a3fdc0 EFLAGS: 00000293 ORIG_RAX: 0000000000000003 +[ 1342.817870] RAX: 0000000000000000 RBX: 0000000000000006 RCX: 0000000000410c51 +[ 1342.825144] RDX: 0000000000000000 RSI: 0000000000730f10 RDI: 0000000000000005 +[ 1342.832419] RBP: 0000000000000000 R08: ffffffffffffffff R09: ffffffffffffffff +[ 1342.839694] R10: 0000000000a3fcf0 R11: 0000000000000293 R12: 0000000000000007 +[ 1342.846968] R13: 0000000000123610 R14: 0000000000000323 R15: badc0ffeebadface +[ 1342.854275] Sending NMI from CPU 0 to CPUs 1: +[ 1342.860797] NMI backtrace for cpu 1 +[ 1342.860804] CPU: 1 PID: 8029 Comm: syz-executor4 Not tainted 4.19.0-rc2+ #230 +[ 1342.860811] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 1342.860816] RIP: 0010:vmx_read_guest_seg_ar+0x1ff/0x270 +[ 1342.860834] RSP: 0018:ffff8801c020edf0 EFLAGS: 00000807 +[ 1342.860844] RAX: dffffc0000000000 RBX: 0000000000000002 RCX: ffffffff811c33b0 +[ 1342.860850] RDX: 0000000000000000 RSI: ffffffff811c33be RDI: ffff8801c757eafc +[ 1342.860856] RBP: ffff8801c020ee18 R08: ffff8801c88a04c0 R09: fffff520026ac047 +[ 1342.860862] R10: fffff520026ac047 R11: ffffc9001356023b R12: ffff8801c75790c0 +[ 1342.860868] R13: 0000000000000800 R14: 0000000000000800 R15: 0000000000100000 +[ 1342.860874] FS: 00007fae23f34700(0000) GS:ffff8801daf00000(0000) knlGS:0000000000000000 +[ 1342.860879] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 1342.860885] CR2: 0000001b31d25000 CR3: 00000001bf9be000 CR4: 00000000001426e0 +[ 1342.860891] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 1342.860897] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 1342.860901] Call Trace: +[ 1342.860905] vmx_get_cpl+0x5b/0x90 +[ 1342.860909] kvm_arch_vcpu_put+0x9b/0x420 +[ 1342.860913] kvm_sched_out+0x91/0xb0 +[ 1342.860917] __schedule+0xf8c/0x1ed0 +[ 1342.860958] preempt_schedule_notrace+0x70/0x130 +[ 1342.860962] ___preempt_schedule_notrace+0x16/0x31 +[ 1342.860967] rcu_is_watching+0x23/0x30 +[ 1342.860971] rcu_read_lock+0x43/0x70 +[ 1342.860975] get_mem_cgroup_from_mm+0x8c/0x440 +[ 1342.861014] memcg_kmem_get_cache+0x1fc/0x9d0 +[ 1342.861036] kmem_cache_alloc+0x193/0x730 +[ 1342.861050] mmu_topup_memory_caches+0x2ec/0x390 +[ 1342.861054] kvm_mmu_load+0x21/0xfa0 +[ 1342.861071] vcpu_enter_guest+0x3dee/0x62e0 +[ 1342.861143] kvm_arch_vcpu_ioctl_run+0x375/0x16e0 +[ 1342.861152] kvm_vcpu_ioctl+0x72b/0x1150 +[ 1342.861211] do_vfs_ioctl+0x1de/0x1720 +[ 1342.861272] ksys_ioctl+0xa9/0xd0 +[ 1342.861276] __x64_sys_ioctl+0x73/0xb0 +[ 1342.861280] do_syscall_64+0x1b9/0x820 +[ 1342.861318] entry_SYSCALL_64_after_hwframe+0x49/0xbe +[ 1342.861322] RIP: 0033:0x457099 +[ 1342.861339] RSP: 002b:00007fae23f33c78 EFLAGS: 00000246 ORIG_RAX: 0000000000000010 +[ 1342.861349] RAX: ffffffffffffffda RBX: 00007fae23f346d4 RCX: 0000000000457099 +[ 1342.861355] RDX: 0000000000000000 RSI: 000000000000ae80 RDI: 0000000000000005 +[ 1342.861361] RBP: 00000000009300a0 R08: 0000000000000000 R09: 0000000000000000 +[ 1342.861367] R10: 0000000000000000 R11: 0000000000000246 R12: 00000000ffffffff +[ 1342.861373] R13: 00000000004cf730 R14: 00000000004c59b9 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/289 b/pkg/report/testdata/linux/report/289 new file mode 100644 index 00000000000..febcd4d790f --- /dev/null +++ b/pkg/report/testdata/linux/report/289 @@ -0,0 +1,41 @@ +TITLE: INFO: rcu detected stall in ext4_filemap_fault + +[ 2355.201776] INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 2355.201784] Tasks blocked on level-0 rcu_node (CPUs 0-1): P2270 +[ 2355.201792] (detected by 0, t=10502 jiffies, g=98916, c=98915, q=146) +[ 2355.201808] syz-fuzzer R running task 25680 2270 2264 0x00000000 +[ 2355.201818] 00000000000008d8 16c3269289c67388 ffff8801db607c20 ffffffff8138d924 +[ 2355.201826] ffff8801caf9df00 ffffffff82ea86c0 0000000000000092 dffffc0000000000 +[ 2355.201835] ffff8801caf9e308 ffff8801db607c58 ffffffff8139811e 0000000000018263 +[ 2355.201836] Call Trace: +[ 2355.201852] [] sched_show_task+0x378/0x387 +[ 2355.201860] [] rcu_print_detail_task_stall_rnp+0xbd/0xf9 +[ 2355.201868] [] rcu_check_callbacks.cold.67+0xa85/0xd20 +[ 2355.201876] [] update_process_times+0x3a/0x70 +[ 2355.201885] [] tick_sched_handle.isra.6+0x4a/0xf0 +[ 2355.201892] [] tick_sched_timer+0x76/0x130 +[ 2355.201906] [] __hrtimer_run_queues+0x390/0xfc0 +[ 2355.201956] [] hrtimer_interrupt+0x1b1/0x430 +[ 2355.201964] [] local_apic_timer_interrupt+0x74/0xa0 +[ 2355.201971] [] smp_apic_timer_interrupt+0x7c/0xa0 +[ 2355.201978] [] apic_timer_interrupt+0x9d/0xb0 +[ 2355.201987] [] ? console_unlock+0x659/0xa10 +[ 2355.202000] [] vprintk_emit+0x3f5/0x830 +[ 2355.202006] [] vprintk+0x28/0x30 +[ 2355.202013] [] vprintk_default+0x1d/0x30 +[ 2355.202019] [] printk+0xaf/0xd7 +[ 2355.202042] [] dump_header.isra.4+0x617/0x6b5 +[ 2355.202050] [] oom_kill_process.cold.9+0x1b/0x9b6 +[ 2355.202065] [] out_of_memory+0x48e/0xab0 +[ 2355.202092] [] __alloc_pages_nodemask+0x122a/0x1430 +[ 2355.202166] [] filemap_fault+0x802/0xbe0 +[ 2355.202173] [] ext4_filemap_fault+0x71/0xa0 +[ 2355.202181] [] __do_fault+0x1d7/0x360 +[ 2355.202203] [] handle_mm_fault+0x1cd9/0x2f30 +[ 2355.202231] [] __do_page_fault+0x291/0x7e0 +[ 2355.202238] [] do_page_fault+0x27/0x30 +[ 2355.202244] [] page_fault+0x25/0x30 +[ 2355.202257] syz-fuzzer R running task 25680 2270 2264 0x00000000 +[ 2355.202266] 00000000000008d8 16c3269289c67388 ffff8801db607c20 ffffffff8138d924 +[ 2355.202274] ffff8801caf9df00 ffffffff82ea86c0 0000000000000092 dffffc0000000000 +[ 2355.202283] ffff8801caf9e308 ffff8801db607c58 ffffffff8139811e ffffffff82ea8940 diff --git a/pkg/report/testdata/linux/report/290 b/pkg/report/testdata/linux/report/290 new file mode 100644 index 00000000000..f5784c80675 --- /dev/null +++ b/pkg/report/testdata/linux/report/290 @@ -0,0 +1,47 @@ +TITLE: INFO: rcu detected stall in ipv6_rcv + +[ 757.882665] INFO: rcu_preempt self-detected stall on CPU +[ 757.888422] 0-...: (1 GPs behind) idle=c37/140000000000001/0 softirq=7357/7358 fqs=12499 +[ 757.896793] (t=12500 jiffies g=1208 c=1207 q=83) +[ 757.901825] Task dump for CPU 0: +[ 757.905161] syz-executor0 R running task 28096 4312 3955 0x2002000c +[ 757.912583] 0000000000000f73 f9a9a8cb61184120 ffff8801db206ff0 ffffffff8140c8fc +[ 757.920592] ffff8801db21f4c0 0000000000000000 dffffc0000000000 ffffffff844bef00 +[ 757.928622] ffffffff844bef84 ffff8801db207010 ffffffff8140cb87 ffffffff844bef48 +[ 757.936617] Call Trace: +[ 757.939184] [] sched_show_task+0x2cb/0x2d6 +[ 757.945782] [] dump_cpu_task+0x79/0x7e +[ 757.951308] [] rcu_dump_cpu_stacks+0x150/0x164 +[ 757.957513] [] rcu_check_callbacks.cold.75+0x5c3/0xd20 +[ 757.970200] [] update_process_times+0x3a/0x70 +[ 757.976320] [] tick_sched_handle.isra.15+0x55/0xf0 +[ 757.982881] [] tick_sched_timer+0x72/0x120 +[ 757.994948] [] __hrtimer_run_queues+0x3ad/0x1000 +[ 758.025989] [] hrtimer_interrupt+0x1b1/0x430 +[ 758.032028] [] local_apic_timer_interrupt+0x74/0xa0 +[ 758.038752] [] smp_apic_timer_interrupt+0x7c/0xa0 +[ 758.045218] [] apic_timer_interrupt+0xa0/0xb0 +[ 758.063861] [] __xfrm_decode_session+0x69/0x100 +[ 758.070171] [] icmpv6_route_lookup+0x2ce/0x440 +[ 758.088109] [] icmp6_send+0xee9/0x1b80 +[ 758.152555] [] icmpv6_send+0xb1/0x1b0 +[ 758.164964] [] ip6_pkt_drop+0x16d/0x430 +[ 758.170569] [] ip6_pkt_discard+0x1c/0x20 +[ 758.176265] [] ip6_rcv_finish+0x13d/0x640 +[ 758.187979] [] ipv6_rcv+0x10cb/0x1cd0 +[ 758.217187] [] __netif_receive_skb_core+0x12d6/0x2940 +[ 758.237196] [] __netif_receive_skb+0x5b/0x1b0 +[ 758.243331] [] process_backlog+0x216/0x6a0 +[ 758.255225] [] net_rx_action+0x3a2/0xdb0 +[ 758.274043] [] __do_softirq+0x22c/0xa1a +[ 758.286381] [] do_softirq_own_stack+0x1c/0x30 +[ 758.292494] [] do_softirq.part.16+0x54/0x60 +[ 758.299190] [] do_softirq+0x19/0x20 +[ 758.304442] [] netif_rx_ni+0xec/0x3a0 +[ 758.309892] [] tun_get_user+0xbe7/0x2410 +[ 758.326886] [] tun_chr_write_iter+0xd5/0x190 +[ 758.332918] [] __vfs_write+0x30d/0x3f0 +[ 758.380578] [] vfs_write+0x191/0x4e0 +[ 758.385913] [] SyS_write+0xd9/0x1c0 +[ 758.414182] [] do_fast_syscall_32+0x326/0x8b0 +[ 758.420301] [] sysenter_flags_fixed+0xd/0x17 diff --git a/pkg/report/testdata/linux/report/291 b/pkg/report/testdata/linux/report/291 new file mode 100644 index 00000000000..47506e9d2d0 --- /dev/null +++ b/pkg/report/testdata/linux/report/291 @@ -0,0 +1,30 @@ +TITLE: INFO: rcu detected stall in rmdir + +[ 539.960575] INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 539.960585] (detected by 0, t=10502 jiffies, g=3970, c=3969, q=379) +[ 539.960592] All QSes seen, last rcu_preempt kthread activity 10502 (4294991162-4294980660), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 539.960608] syz-executor7 R running task 26968 2301 2298 0x00000000 +[ 539.960623] ffff8801db607c98 ffffffff813f5a7f ffffffff813f5886 0000000100003434 +[ 539.960633] 0000000100005d3a dffffc0000000000 ffff8801db621d40 ffffffff82ecd580 +[ 539.960643] ffff8801db607d68 ffffffff81401aa5 0000000000000000 ffff8801db607cd8 +[ 539.960645] Call Trace: +[ 539.960662] +[ 539.960663] [] sched_show_task.cold.35+0x279/0x31f +[ 539.960679] [] rcu_check_callbacks.cold.69+0xc70/0xd27 +[ 539.960688] [] update_process_times+0x30/0x70 +[ 539.960697] [] tick_sched_handle.isra.5+0x4a/0xf0 +[ 539.960704] [] tick_sched_timer+0x76/0x130 +[ 539.960720] [] __hrtimer_run_queues+0x357/0xe30 +[ 539.960766] [] hrtimer_interrupt+0x1b1/0x430 +[ 539.960775] [] local_apic_timer_interrupt+0x74/0xa0 +[ 539.960784] [] smp_apic_timer_interrupt+0x7c/0xa0 +[ 539.960791] [] apic_timer_interrupt+0x9d/0xb0 +[ 539.960802] +[ 539.960812] [] _raw_spin_lock_nested+0x44/0x50 +[ 539.960828] [] d_walk.part.10+0x1b9/0x710 +[ 539.960849] [] shrink_dcache_parent+0xd7/0x130 +[ 539.960873] [] vfs_rmdir2+0x1c9/0x410 +[ 539.960880] [] do_rmdir+0x304/0x3b0 +[ 539.960916] [] SyS_rmdir+0x1a/0x20 +[ 539.960922] [] do_syscall_64+0x19f/0x480 +[ 539.960930] [] entry_SYSCALL_64_after_swapgs+0x5d/0xdb diff --git a/pkg/report/testdata/linux/report/292 b/pkg/report/testdata/linux/report/292 new file mode 100644 index 00000000000..f8e401066d8 --- /dev/null +++ b/pkg/report/testdata/linux/report/292 @@ -0,0 +1,46 @@ +TITLE: INFO: rcu detected stall in wb_workfn + +[ 420.341960] INFO: rcu_preempt detected stalls on CPUs/tasks: +[ 420.347923] (detected by 1, t=10502 jiffies, g=9244, c=9243, q=178) +[ 420.354558] All QSes seen, last rcu_preempt kthread activity 10502 (4294979208-4294968706), jiffies_till_next_fqs=1, root ->qsmask 0x0 +[ 420.366779] kworker/u4:2 R running task 25560 32 2 0x00000008 +[ 420.374291] Workqueue: writeback wb_workfn (flush-8:0) +[ 420.379821] ffff8801db307ca0 ffffffff814249f9 0000000100000582 0000000100002e88 +[ 420.387871] dffffc0000000000 ffff8801db322880 ffffffff846e0300 ffff8801db307d70 +[ 420.395936] ffffffff8143035a 0000000000000000 ffffffff846def20 ffffffff846e0a18 +[ 420.403962] Call Trace: +[ 420.406526] [ 420.408595] [] sched_show_task.cold.127+0x1c9/0x279 +[ 420.415289] [] rcu_check_callbacks.cold.79+0xc70/0xd27 +[ 420.422204] [] update_process_times+0x30/0x70 +[ 420.428339] [] tick_sched_handle.isra.14+0x55/0xf0 +[ 420.434904] [] tick_sched_timer+0x72/0x120 +[ 420.446992] [] __hrtimer_run_queues+0x375/0xe50 +[ 420.478026] [] hrtimer_interrupt+0x1b1/0x430 +[ 420.484071] [] local_apic_timer_interrupt+0x74/0xa0 +[ 420.490726] [] smp_apic_timer_interrupt+0x7c/0xa0 +[ 420.497210] [] apic_timer_interrupt+0xa0/0xb0 +[ 420.503332] [ 420.505397] [] ? smp_call_function_single+0x11b/0x360 +[ 420.518914] [] smp_call_function_single+0x11b/0x360 +[ 420.561741] [] smp_call_function_many+0x57a/0x6a0 +[ 420.579966] [] native_flush_tlb_others+0xd4/0x510 +[ 420.597680] [] flush_tlb_mm_range+0x10c/0x440 +[ 420.603832] [] ptep_clear_flush+0xec/0x110 +[ 420.609706] [] page_mkclean_one+0x100/0x220 +[ 420.622141] [] rmap_walk_file+0x2c2/0x780 +[ 420.634853] [] rmap_walk+0xed/0x180 +[ 420.640112] [] page_mkclean+0x196/0x1d0 +[ 420.672156] [] clear_page_dirty_for_io+0x200/0x4e0 +[ 420.678721] [] mpage_submit_page+0x77/0x240 +[ 420.684675] [] mpage_process_page_bufs+0x3b2/0x490 +[ 420.691270] [] mpage_prepare_extent_to_map+0x52f/0x9a0 +[ 420.717326] [] ext4_writepages+0xdb9/0x2e50 +[ 420.749218] [] do_writepages+0xef/0x1d0 +[ 420.767364] [] __writeback_single_inode+0xd9/0x1020 +[ 420.774030] [] writeback_sb_inodes+0x4ac/0xe70 +[ 420.798721] [] __writeback_inodes_wb+0xfb/0x1e0 +[ 420.805040] [] wb_writeback+0x512/0xbd0 +[ 420.824129] [] wb_workfn+0x20e/0xdb0 +[ 420.841940] [] process_one_work+0x7e1/0x1500 +[ 420.860701] [] worker_thread+0xd6/0x10a0 +[ 420.872096] [] kthread+0x26d/0x300 +[ 420.900433] [] ret_from_fork+0x5c/0x70 diff --git a/pkg/report/testdata/linux/report/293 b/pkg/report/testdata/linux/report/293 new file mode 100644 index 00000000000..d08dcca52c0 --- /dev/null +++ b/pkg/report/testdata/linux/report/293 @@ -0,0 +1,50 @@ +TITLE: INFO: rcu detected stall in perf_mmap + +[ 323.391020] INFO: rcu_sched self-detected stall on CPU +[ 323.396363] 1-...: (64999 ticks this GP) idle=97a/140000000000001/0 softirq=7370/7370 fqs=16241 +[ 323.405345] (t=65000 jiffies g=4163 c=4162 q=64) +[ 323.410286] NMI backtrace for cpu 1 +[ 323.413913] CPU: 1 PID: 1885 Comm: syz-executor921 Not tainted 4.14.49+ #8 +[ 323.420908] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 323.430246] Call Trace: +[ 323.432801] +[ 323.434933] dump_stack+0x114/0x1cf +[ 323.457051] nmi_cpu_backtrace.cold.2+0x19/0x95 +[ 323.471164] nmi_trigger_cpumask_backtrace+0x102/0x143 +[ 323.476417] arch_trigger_cpumask_backtrace+0x19/0x20 +[ 323.481585] rcu_dump_cpu_stacks+0x177/0x1c4 +[ 323.485974] check_cpu_stall.isra.56.cold.74+0x289/0x4c3 +[ 323.569083] rcu_check_callbacks+0x2e4/0xe20 +[ 323.620799] update_process_times+0x35/0x70 +[ 323.625119] tick_sched_handle+0x8a/0x150 +[ 323.629243] tick_sched_timer+0x47/0x120 +[ 323.637661] __hrtimer_run_queues+0x4ea/0x1150 +[ 323.715212] hrtimer_interrupt+0x1b1/0x5b0 +[ 323.719432] smp_apic_timer_interrupt+0x1f9/0x870 +[ 323.756898] apic_timer_interrupt+0x8e/0xa0 +[ 323.761194] +[ 323.763409] RIP: 0010:lock_is_held_type+0x115/0x160 +[ 323.768396] RSP: 0018:ffff8801c5abf228 EFLAGS: 00000282 ORIG_RAX: ffffffffffffff10 +[ 323.776082] RAX: 0000000000000000 RBX: 1ffff10038b57e46 RCX: 0000000000000000 +[ 323.783326] RDX: 0000000000000000 RSI: ffffffff83efc6c0 RDI: 0000000000000246 +[ 323.790578] RBP: ffff8801c5abf2b0 R08: ffffffff81566839 R09: ffffffff8518b320 +[ 323.797822] R10: ffff8801c5abf700 R11: 0000000000000000 R12: 0000000000000000 +[ 323.805074] R13: ffffffff8385a220 R14: 00000000000002eb R15: 0000000000000000 +[ 323.832368] __might_sleep+0x9a/0x190 +[ 323.848601] __mutex_lock+0x143/0x19c0 +[ 323.917528] mutex_lock_nested+0x1b/0x20 +[ 323.925769] perf_mmap+0x689/0x17f0 +[ 323.969686] mmap_region+0xba0/0x1660 +[ 323.998425] do_mmap+0x730/0xea0 +[ 324.018118] vm_mmap_pgoff+0x1e3/0x270 +[ 324.033674] SyS_mmap_pgoff+0x46d/0x630 +[ 324.055175] SyS_mmap+0x1b/0x30 +[ 324.058439] do_syscall_64+0x251/0x750 +[ 324.087114] entry_SYSCALL_64_after_hwframe+0x42/0xb7 +[ 324.092288] RIP: 0033:0x445e09 +[ 324.095451] RSP: 002b:00007eff4d398d98 EFLAGS: 00000212 ORIG_RAX: 0000000000000009 +[ 324.103133] RAX: ffffffffffffffda RBX: 00000000006dac5c RCX: 0000000000445e09 +[ 324.110376] RDX: 0000000000000000 RSI: 0000000000001000 RDI: 00000000201bf000 +[ 324.117619] RBP: 00000000006dac58 R08: 0000000000000301 R09: 0000000000000000 +[ 324.124872] R10: 0080000000004011 R11: 0000000000000212 R12: 7479625f6e695f65 +[ 324.132116] R13: 676173752e424d32 R14: 2e626c7465677568 R15: 0000000000000009 diff --git a/pkg/report/testdata/linux/report/294 b/pkg/report/testdata/linux/report/294 new file mode 100644 index 00000000000..bf34a6e0cf0 --- /dev/null +++ b/pkg/report/testdata/linux/report/294 @@ -0,0 +1,44 @@ +TITLE: INFO: rcu detected stall in lo_compat_ioctl + +[ 480.801028] INFO: rcu_sched self-detected stall on CPU +[ 480.806333] 0-...: (64962 ticks this GP) idle=ff6/140000000000001/0 softirq=6610/6610 fqs=16220 +[ 480.815311] (t=65000 jiffies g=3400 c=3399 q=180) +[ 480.820444] NMI backtrace for cpu 0 +[ 480.824066] CPU: 0 PID: 1932 Comm: syz-executor0 Not tainted 4.14.35+ #15 +[ 480.830976] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 480.840409] Call Trace: +[ 480.842967] +[ 480.845148] dump_stack+0x114/0x1cf +[ 480.867367] nmi_cpu_backtrace.cold.2+0x19/0x95 +[ 480.881500] nmi_trigger_cpumask_backtrace+0x102/0x143 +[ 480.886763] arch_trigger_cpumask_backtrace+0x19/0x20 +[ 480.891968] rcu_dump_cpu_stacks+0x177/0x1c4 +[ 480.896355] check_cpu_stall.isra.56.cold.75+0x289/0x4b7 +[ 480.975100] rcu_check_callbacks+0x2e4/0xe20 +[ 481.026890] update_process_times+0x35/0x70 +[ 481.031242] tick_sched_handle+0x8a/0x150 +[ 481.035364] tick_sched_timer+0x47/0x120 +[ 481.043793] __hrtimer_run_queues+0x4ea/0x1150 +[ 481.122814] hrtimer_interrupt+0x1b1/0x5b0 +[ 481.127041] smp_apic_timer_interrupt+0x1f9/0x870 +[ 481.169084] apic_timer_interrupt+0x8e/0xa0 +[ 481.173377] +[ 481.175588] RIP: 0010:lo_ioctl+0x15e4/0x1b50 +[ 481.179969] RSP: 0018:ffff8801cd6efb78 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff10 +[ 481.187652] RAX: 1ffff100395975ef RBX: ffff8801cacbadc0 RCX: dffffc0000000000 +[ 481.194907] RDX: 0000000000000000 RSI: 0000000000000002 RDI: ffff8801cacbaf78 +[ 481.202165] RBP: ffff8801cd6efbf0 R08: 0000000000000001 R09: 0000000000000000 +[ 481.209417] R10: 0000000000000003 R11: 0000000000000001 R12: ffff8801caf99c40 +[ 481.216664] R13: ffff8801db016320 R14: ffff8801d51e4b48 R15: ffff8801db016240 +[ 481.236297] lo_compat_ioctl+0xb3/0x140 +[ 481.244057] compat_blkdev_ioctl+0x39c/0x1950 +[ 481.261895] compat_SyS_ioctl+0x19d/0x3030 +[ 481.281601] do_fast_syscall_32+0x3c1/0xbf1 +[ 481.317738] entry_SYSENTER_compat+0x84/0x96 +[ 481.322121] RIP: 0023:0xf7f71c99 +[ 481.325470] RSP: 002b:00000000fff4d7ec EFLAGS: 00000286 ORIG_RAX: 0000000000000036 +[ 481.333156] RAX: ffffffffffffffda RBX: 0000000000000004 RCX: 0000000000004c00 +[ 481.340410] RDX: 0000000000000003 RSI: 0000000000000000 RDI: 0000000000000000 +[ 481.347665] RBP: 0000000000000000 R08: 0000000000000000 R09: 0000000000000000 +[ 481.354909] R10: 0000000000000000 R11: 0000000000000000 R12: 0000000000000000 +[ 481.362160] R13: 0000000000000000 R14: 0000000000000000 R15: 0000000000000000 diff --git a/pkg/report/testdata/linux/report/295 b/pkg/report/testdata/linux/report/295 new file mode 100644 index 00000000000..2c903954c85 --- /dev/null +++ b/pkg/report/testdata/linux/report/295 @@ -0,0 +1,58 @@ +TITLE: INFO: rcu detected stall in netlink_sendmsg + +[ 355.572806] INFO: rcu_sched self-detected stall on CPU +[ 355.578254] 0: (104999 ticks this GP) idle=ab9/140000000000001/0 softirq=76582/76582 fqs=34894 +[ 355.587151] (t=105000 jiffies g=36307 c=36306 q=2303) +[ 355.592663] Task dump for CPU 0: +[ 355.596007] syz-executor7 R running task on cpu 0 0 19712 4019 0x0000000c 250568154159 +[ 355.605206] 0000000000000fb3 00000000ccf50a4a ffff8801daa07bd8 ffffffff816f0dc7 +[ 355.613296] ffff8801daa1f7c0 0000000000000000 dffffc0000000000 ffffffff83fca380 +[ 355.621298] ffffffff83fca3f4 ffff8801daa07be8 ffffffff816f154c ffff8801daa07c08 +[ 355.629309] Call Trace: +[ 355.631878] [] _sched_show_task+0x31a/0x325 +[ 355.638595] [] sched_show_task+0x10/0x12 +[ 355.644287] [] dump_cpu_task+0x7e/0x83 +[ 355.649807] [] rcu_dump_cpu_stacks+0x155/0x169 +[ 355.656012] [] rcu_check_callbacks.cold.76+0x61e/0xc77 +[ 355.668696] [] update_process_times+0x3f/0x70 +[ 355.674817] [] tick_sched_handle.isra.16+0x5a/0x100 +[ 355.681474] [] tick_sched_timer+0x7a/0x130 +[ 355.687337] [] __hrtimer_run_queues+0x3a5/0xc50 +[ 355.725548] [] hrtimer_interrupt+0x18e/0x400 +[ 355.731583] [] local_apic_timer_interrupt+0x74/0x90 +[ 355.738224] [] smp_apic_timer_interrupt+0xdf/0x130 +[ 355.744785] [] smp_apic_timer_interrupt_entry_after_kaiser_bti+0x20/0x2e +[ 355.753246] [] ? check_noncircular+0x20/0x20 +[ 355.766683] [] is_ftrace_trampoline+0x37/0x120 +[ 355.772905] [] __kernel_text_address+0x88/0xc0 +[ 355.779116] [] print_context_stack+0x54/0xe0 +[ 355.785183] [] dump_trace+0x16d/0x320 +[ 355.790612] [] save_stack_trace+0x2b/0x50 +[ 355.796389] [] save_stack+0x43/0xd0 +[ 356.049567] [] kasan_kmalloc+0xc4/0xe0 +[ 356.055079] [] kasan_slab_alloc+0x12/0x20 +[ 356.060852] [] kmem_cache_alloc+0x14d/0x710 +[ 356.078645] [] ida_pre_get+0x12e/0x2b0 +[ 356.096113] [] ida_simple_get+0xd9/0x1e0 +[ 356.117746] [] __kernfs_new_node+0x97/0x2c0 +[ 356.123707] [] kernfs_new_node+0x85/0xf0 +[ 356.129409] [] kernfs_create_dir_ns+0x42/0x140 +[ 356.135620] [] sysfs_create_dir_ns+0xc3/0x1d0 +[ 356.141741] [] kobject_add_internal+0x284/0x990 +[ 356.148036] [] kobject_init_and_add+0xe7/0x140 +[ 356.160732] [] netdev_queue_update_kobjects+0xf4/0x2b0 +[ 356.167638] [] netdev_register_kobject+0x22f/0x2f0 +[ 356.174196] [] register_netdevice+0x7d1/0xf60 +[ 356.205338] [] ip_tunnel_newlink+0x378/0x770 +[ 356.217510] [] ipgre_newlink+0x102/0x160 +[ 356.247884] [] rtnl_newlink+0x1055/0x15c0 +[ 356.316833] [] rtnetlink_rcv_msg+0x4b4/0x700 +[ 356.353912] [] netlink_rcv_skb+0x143/0x370 +[ 356.365750] [] rtnetlink_rcv+0x2f/0x40 +[ 356.371288] [] netlink_unicast+0x517/0x740 +[ 356.395498] [] netlink_sendmsg+0x7e4/0xc90 +[ 356.420047] [] sock_sendmsg+0xe8/0x150 +[ 356.425579] [] ___sys_sendmsg+0x760/0x8b0 +[ 356.475821] [] __sys_sendmsg+0xdb/0x190 +[ 356.504375] [] SyS_sendmsg+0x32/0x50 +[ 356.509728] [] entry_SYSCALL_64_fastpath+0x12/0x17 diff --git a/pkg/report/testdata/linux/report/296 b/pkg/report/testdata/linux/report/296 new file mode 100644 index 00000000000..c5660be7a98 --- /dev/null +++ b/pkg/report/testdata/linux/report/296 @@ -0,0 +1,36 @@ +TITLE: BUG: soft lockup in mount + +[ 848.443720] NMI watchdog: BUG: soft lockup - CPU#1 stuck for 123s! [syz-executor694:3996] +[ 848.452125] Modules linked in: +[ 848.455412] irq event stamp: 933704 +[ 848.459142] hardirqs last enabled at (933703): [] restore_regs_and_iret+0x0/0x1d +[ 848.468415] hardirqs last disabled at (933704): [] apic_timer_interrupt+0x9b/0xb0 +[ 848.477698] softirqs last enabled at (921172): [] __do_softirq+0x46c/0x937 +[ 848.486386] softirqs last disabled at (921163): [] irq_exit+0x147/0x190 +[ 848.496037] CPU: 1 PID: 3996 Comm: syz-executor694 Not tainted 4.9.111-g03c70fe #10 +[ 848.503812] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 848.513416] task: ffff8801d8ac3000 task.stack: ffff8801b70a0000 +[ 848.519454] RIP: 0010:[] [] change_mnt_propagation+0xa9/0xbd0 +[ 848.528820] RSP: 0018:ffff8801b70a7af0 EFLAGS: 00000246 +[ 848.534258] RAX: ffff8801d8ac3000 RBX: ffff8801cda6e400 RCX: ffff8801cda6e468 +[ 848.541526] RDX: 1ffff1003723a2c4 RSI: ffffffff8160fbe8 RDI: ffff8801b91d1620 +[ 848.548806] RBP: ffff8801b70a7b50 R08: ffff8801cda6e468 R09: 0000000000000001 +[ 848.556086] R10: 0000000000000000 R11: 1ffff1003b15872c R12: dffffc0000000000 +[ 848.563348] R13: ffff8801b91d1600 R14: ffff8801d9c47340 R15: ffffed0039b4dc84 +[ 848.570605] FS: 0000000000000000(0000) GS:ffff8801db300000(0063) knlGS:00000000f7667b40 +[ 848.578828] CS: 0010 DS: 002b ES: 002b CR0: 0000000080050033 +[ 848.584692] CR2: 00000000f76cadb0 CR3: 00000001ca4fa000 CR4: 00000000001606f0 +[ 848.592068] DR0: 0000000000000000 DR1: 0000000000000000 DR2: 0000000000000000 +[ 848.599318] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000400 +[ 848.606567] Stack: +[ 848.608707] ffff8801b70a7b20 ffffffff815db8af 0000000000000000 ffff8801cda6e4f0 +[ 848.616764] ffff8801cda6e420 ffff8801cda6e4e0 00040000b70a7b50 ffff8801cda6e490 +[ 848.624800] dffffc0000000000 ffff8801cda6e400 ffff8801d9c469a0 ffff8801b70a7bf0 +[ 848.632818] Call Trace: +[ 848.641493] [] umount_tree+0x326/0x820 +[ 848.652638] [] attach_recursive_mnt+0x750/0x8f0 +[ 848.675872] [] graft_tree+0x16f/0x1f0 +[ 848.681308] [] do_mount+0x1f5f/0x2740 +[ 848.724324] [] compat_SyS_mount+0x4fc/0xff0 +[ 848.743149] [] do_fast_syscall_32+0x2f7/0x870 +[ 848.755940] [] entry_SYSENTER_compat+0x90/0xa2 diff --git a/pkg/report/testdata/linux/report/297 b/pkg/report/testdata/linux/report/297 new file mode 100644 index 00000000000..586be8c13ed --- /dev/null +++ b/pkg/report/testdata/linux/report/297 @@ -0,0 +1,39 @@ +TITLE: BUG: soft lockup in blkdev_close + +[ 601.962573] watchdog: BUG: soft lockup - CPU#1 stuck for 123s! [syz-executor1:24793] +[ 601.970515] Modules linked in: +[ 601.973739] CPU: 1 PID: 24793 Comm: syz-executor1 Not tainted 4.18.0-rc4+ #24 +[ 601.981015] Hardware name: Google Google Compute Engine/Google Compute Engine, BIOS Google 01/01/2011 +[ 601.990409] RIP: 0010:msan_get_shadow_origin_ptr+0x6f/0x300 +[ 601.996116] Code: 8b 48 0f 45 d8 80 3c 25 00 f0 a0 8b 00 0f 84 08 02 00 00 65 4c 8b 2c 25 80 fc 02 00 41 83 bd f4 08 00 00 00 0f 85 f1 01 00 00 <49> 81 fc 01 10 00 00 72 23 0f b6 ca 45 31 f6 48 c7 c7 ba 47 d7 8a +[ 602.016312] RSP: 0018:ffff8800b3a2f208 EFLAGS: 00000246 ORIG_RAX: ffffffffffffff13 +[ 602.024043] RAX: ffffffff8ba13000 RBX: ffffffff8ba11000 RCX: ffff880103d43b00 +[ 602.031323] RDX: 0000000000000000 RSI: 0000000000000004 RDI: ffff8800b3a2f2d8 +[ 602.038604] RBP: ffff8800b3a2f260 R08: 0000000000000000 R09: 0000000000000002 +[ 602.045887] R10: 0000000000000000 R11: ffffffff8117d090 R12: 0000000000000004 +[ 602.053171] R13: ffff880103d43b00 R14: ffffffff8ba10000 R15: ffff8800b3a2f2d8 +[ 602.060458] FS: 00007f4dc65dc700(0000) GS:ffff88021fd00000(0000) knlGS:0000000000000000 +[ 602.068697] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 +[ 602.074586] CR2: 00000000004544da CR3: 00000000b3c12000 CR4: 00000000001406e0 +[ 602.081868] DR0: 0000000020000000 DR1: 0000000020000000 DR2: 0000000000000000 +[ 602.089149] DR3: 0000000000000000 DR6: 00000000fffe0ff0 DR7: 0000000000000600 +[ 602.096425] Call Trace: +[ 602.104092] __msan_metadata_ptr_for_load_4+0x10/0x20 +[ 602.109301] smp_call_function_single+0x363/0x500 +[ 602.122402] smp_call_function_many+0x672/0xe30 +[ 602.135338] on_each_cpu_cond+0x335/0x420 +[ 602.148630] invalidate_bh_lrus+0x54/0x60 +[ 602.152805] __blkdev_put+0x44b/0xcc0 +[ 602.160518] blkdev_put+0x435/0x570 +[ 602.164201] blkdev_close+0xa6/0xf0 +[ 602.167872] __fput+0x458/0xa30 +[ 602.174520] ____fput+0x37/0x40 +[ 602.177819] task_work_run+0x22e/0x2b0 +[ 602.181766] do_exit+0x110e/0x3930 +[ 602.185376] do_group_exit+0x1a0/0x360 +[ 602.189287] get_signal+0x15c3/0x2190 +[ 602.216814] do_signal+0xca/0x2060 +[ 602.229420] prepare_exit_to_usermode+0x297/0x430 +[ 602.234312] syscall_return_slowpath+0x108/0x800 +[ 602.243411] do_syscall_64+0x1ad/0x230 +[ 602.247324] entry_SYSCALL_64_after_hwframe+0x63/0xe7 diff --git a/pkg/report/testdata/linux/report/298 b/pkg/report/testdata/linux/report/298 new file mode 100644 index 00000000000..1ecf5e7dc3c --- /dev/null +++ b/pkg/report/testdata/linux/report/298 @@ -0,0 +1,58 @@ +TITLE: INFO: rcu detected stall in netlink_sendmsg + +[ 287.048281] INFO: rcu_sched self-detected stall on CPU +[ 287.053694] 0: (104999 ticks this GP) idle=769/140000000000001/0 softirq=28286/28286 fqs=34911 +[ 287.062584] (t=105000 jiffies g=13648 c=13647 q=595) +[ 287.067988] Task dump for CPU 0: +[ 287.071369] syz-executor2 R running task on cpu 0 0 7831 4518 0x2002000c 181735472838 +[ 287.080530] 00000000000011a6 000000001a7b0e17 ffff8801daa07bd8 ffffffff816ecb37 +[ 287.088617] ffff8801daa1f780 0000000000000000 dffffc0000000000 ffffffff83fca0c0 +[ 287.096597] ffffffff83fca134 ffff8801daa07be8 ffffffff816ed2bc ffff8801daa07c08 +[ 287.104578] Call Trace: +[ 287.107132] [] _sched_show_task+0x31a/0x325 +[ 287.113816] [] sched_show_task+0x10/0x12 +[ 287.119500] [] dump_cpu_task+0x7e/0x83 +[ 287.125012] [] rcu_dump_cpu_stacks+0x155/0x169 +[ 287.131219] [] rcu_check_callbacks.cold.76+0x61e/0xc77 +[ 287.143892] [] update_process_times+0x3f/0x70 +[ 287.150010] [] tick_sched_handle.isra.16+0x5a/0x100 +[ 287.156649] [] tick_sched_timer+0x7a/0x130 +[ 287.162510] [] __hrtimer_run_queues+0x3a5/0xc50 +[ 287.200662] [] hrtimer_interrupt+0x18e/0x400 +[ 287.206695] [] local_apic_timer_interrupt+0x74/0x90 +[ 287.213352] [] smp_apic_timer_interrupt+0xdf/0x130 +[ 287.219929] [] smp_apic_timer_interrupt_entry_after_kaiser_bti+0x20/0x2e +[ 287.228401] [] ? alloc_netdev_mqs+0x97b/0xc50 +[ 287.241566] [] __kernel_text_address+0x88/0xc0 +[ 287.247777] [] print_context_stack+0x54/0xe0 +[ 287.253814] [] dump_trace+0x16d/0x320 +[ 287.259241] [] save_stack_trace+0x2b/0x50 +[ 287.265014] [] save_stack+0x43/0xd0 +[ 287.412837] [] kasan_kmalloc+0xc4/0xe0 +[ 287.418346] [] kasan_slab_alloc+0x12/0x20 +[ 287.424118] [] kmem_cache_alloc+0x5e3/0x740 +[ 287.436125] [] ida_pre_get+0x12e/0x2b0 +[ 287.453551] [] ida_simple_get+0xd9/0x1e0 +[ 287.471649] [] __kernfs_new_node+0x97/0x2c0 +[ 287.477598] [] kernfs_new_node+0x85/0xf0 +[ 287.483282] [] __kernfs_create_file+0x50/0x2a0 +[ 287.489487] [] sysfs_add_file_mode_ns+0x1f6/0x510 +[ 287.495955] [] sysfs_create_file_ns+0x90/0xb0 +[ 287.502085] [] kobject_add_internal+0x476/0x990 +[ 287.508396] [] kobject_init_and_add+0xe7/0x140 +[ 287.521073] [] netdev_queue_update_kobjects+0xf4/0x2b0 +[ 287.527973] [] netdev_register_kobject+0x22f/0x2f0 +[ 287.534545] [] register_netdevice+0x7d1/0xf60 +[ 287.565577] [] ip_tunnel_newlink+0x378/0x770 +[ 287.577761] [] ipgre_newlink+0x102/0x160 +[ 287.608141] [] rtnl_newlink+0x1055/0x15c0 +[ 287.699732] [] rtnetlink_rcv_msg+0x4b4/0x700 +[ 287.736738] [] netlink_rcv_skb+0x143/0x370 +[ 287.748552] [] rtnetlink_rcv+0x2f/0x40 +[ 287.754063] [] netlink_unicast+0x517/0x740 +[ 287.778219] [] netlink_sendmsg+0x7e4/0xc90 +[ 287.802726] [] sock_sendmsg+0xe8/0x150 +[ 287.808240] [] ___sys_sendmsg+0x760/0x8b0 +[ 287.858329] [] __sys_sendmsg+0xdb/0x190 +[ 287.888147] [] compat_SyS_sendmsg+0x2f/0x40 +[ 287.894111] [] sysenter_dispatch+0xf/0x32 diff --git a/pkg/report/testdata/linux/report/299 b/pkg/report/testdata/linux/report/299 new file mode 100644 index 00000000000..9da26909521 --- /dev/null +++ b/pkg/report/testdata/linux/report/299 @@ -0,0 +1,24 @@ +TITLE: INFO: rcu detected stall in sched_ttwu_pending + +[ 575.758025] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 575.764179] (detected by 1, t=300086 jiffies, g=27161, c=27160, q=118) +[ 575.771051] All QSes seen, last rcu_sched kthread activity 300086 (4295238516-4294938430), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 575.783183] swapper/1 R running task on cpu 1 0 0 1 0x00000008 0 +[ 575.791404] 0000000000000001 18186e061cc9dc83 ffff8801dab07c50 ffffffff816ecb37 +[ 575.799432] ffff8801dab20880 0000000100042374 00000000ffff8f3e ffffffff83fca0c0 +[ 575.807415] 0000000000000003 ffff8801dab07c60 ffffffff816ed2bc ffff8801dab07d30 +[ 575.815434] Call Trace: +[ 575.817998] [] _sched_show_task+0x31a/0x325 +[ 575.824729] [] sched_show_task+0x10/0x12 +[ 575.830434] [] rcu_check_callbacks.cold.76+0xbfa/0xc77 +[ 575.837357] [] update_process_times+0x3f/0x70 +[ 575.843494] [] tick_sched_handle.isra.16+0x5a/0x100 +[ 575.850145] [] tick_sched_timer+0x7a/0x130 +[ 575.856014] [] __hrtimer_run_queues+0x3a5/0xc50 +[ 575.894248] [] hrtimer_interrupt+0x18e/0x400 +[ 575.900294] [] local_apic_timer_interrupt+0x74/0x90 +[ 575.906949] [] smp_apic_timer_interrupt+0xdf/0x130 +[ 575.913516] [] smp_apic_timer_interrupt_entry_after_kaiser_bti+0x20/0x2e +[ 575.929641] [] sched_ttwu_pending+0x10a/0x200 +[ 575.941724] [] cpu_startup_entry+0x14e/0x990 +[ 575.974371] [] start_secondary+0x305/0x3e0 diff --git a/pkg/report/testdata/linux/report/300 b/pkg/report/testdata/linux/report/300 new file mode 100644 index 00000000000..8602edef52a --- /dev/null +++ b/pkg/report/testdata/linux/report/300 @@ -0,0 +1,44 @@ +TITLE: INFO: rcu detected stall in uhaul_release + +[ 3481.239960] INFO: rcu_sched detected stalls on CPUs/tasks: +[ 3481.239969] (detected by 1, t=105002 jiffies, g=687946, c=687945, q=520) +[ 3481.239975] All QSes seen, last rcu_sched kthread activity 103964 (4298143477-4298039513), jiffies_till_next_fqs=3, root ->qsmask 0x0 +[ 3481.239987] syz-executor0 R running task on cpu 1 0 315571 315308 0x0000000e 3375506638533 +[ 3481.240033] 000000000004cfac 000000007b396287 ffff8801dab07c50 ffffffff816efb87 +[ 3481.240042] ffff8801dab20880 00000001003076f5 00000001002ee0d9 ffffffff83fca0c0 +[ 3481.240050] 0000000000000003 ffff8801dab07c60 ffffffff816f030c ffff8801dab07d30 +[ 3481.240052] Call Trace: +[ 3481.240067] [] _sched_show_task+0x31a/0x325 +[ 3481.240075] [] sched_show_task+0x10/0x12 +[ 3481.240082] [] rcu_check_callbacks.cold.76+0xbfa/0xc77 +[ 3481.240092] [] update_process_times+0x3f/0x70 +[ 3481.240101] [] tick_sched_handle.isra.16+0x5a/0x100 +[ 3481.240108] [] tick_sched_timer+0x7a/0x130 +[ 3481.240115] [] __hrtimer_run_queues+0x3a5/0xc50 +[ 3481.240160] [] hrtimer_interrupt+0x18e/0x400 +[ 3481.240169] [] local_apic_timer_interrupt+0x74/0x90 +[ 3481.240177] [] smp_apic_timer_interrupt+0xdf/0x130 +[ 3481.240186] [] smp_apic_timer_interrupt_entry_after_kaiser_bti+0x20/0x2e +[ 3481.240196] [] ? __sanitizer_cov_trace_pc+0x21/0x60 +[ 3481.240204] [] smp_call_function_single+0x26c/0x330 +[ 3481.240241] [] smp_cfm_core+0x6a5/0x7e0 +[ 3481.240269] [] on_each_cpu+0x6b/0x120 +[ 3481.240278] [] flush_tlb_kernel_range+0x86/0xe0 +[ 3481.240294] [] __purge_vmap_area_lazy+0x3ec/0xf40 +[ 3481.240333] [] free_vmap_area_noflush+0x1bf/0x200 +[ 3481.240362] [] remove_vm_area+0x13e/0x1c0 +[ 3481.240368] [] __vunmap+0x49/0x330 +[ 3481.240374] [] vfree+0x55/0xe0 +[ 3481.240381] [] kvfree+0x3b/0x60 +[ 3481.240389] [] uhaul_rx_release+0x268/0x5a0 +[ 3481.240412] [] uhaul_release+0xce/0x3a0 +[ 3481.240432] [] sock_release+0x92/0x1b0 +[ 3481.240438] [] sock_close+0x1b/0x20 +[ 3481.240446] [] __fput+0x22e/0x6d0 +[ 3481.240453] [] ____fput+0x1a/0x20 +[ 3481.240462] [] task_work_run+0x1a0/0x240 +[ 3481.240470] [] get_signal+0x1098/0x14a0 +[ 3481.240486] [] do_signal+0x7e/0x400 +[ 3481.240524] [] prepare_exit_to_usermode+0x170/0x390 +[ 3481.240531] [] syscall_return_slowpath+0xc7/0x5c0 +[ 3481.240538] [] int_ret_from_sys_call+0x25/0xba diff --git a/pkg/report/testdata/linux/report/65 b/pkg/report/testdata/linux/report/65 index d6cc73b3573..e8b0d8381a0 100644 --- a/pkg/report/testdata/linux/report/65 +++ b/pkg/report/testdata/linux/report/65 @@ -1,4 +1,4 @@ -TITLE: BUG: soft lockup in smp_call_function_many +TITLE: BUG: soft lockup in jump_label_update_timeout [ 247.938942] watchdog: BUG: soft lockup - CPU#0 stuck for 134s! [kworker/0:2:1400] [ 247.946595] Modules linked in: @@ -22,86 +22,18 @@ TITLE: BUG: soft lockup in smp_call_function_many [ 248.078408] CS: 0010 DS: 0000 ES: 0000 CR0: 0000000080050033 [ 248.084262] CR2: 0000000020335ffc CR3: 00000001d0869000 CR4: 00000000001426f0 [ 248.091506] Call Trace: -[ 248.094075] ? add_nops+0x90/0x90 -[ 248.097504] ? generic_exec_single+0x5b0/0x5b0 -[ 248.102062] ? __mutex_lock+0x16f/0x1a80 -[ 248.106093] ? arch_jump_label_transform+0x1b/0x40 -[ 248.110995] ? print_usage_bug+0x3f0/0x3f0 -[ 248.115213] ? _find_next_bit+0xee/0x120 -[ 248.119250] ? add_nops+0x90/0x90 -[ 248.122676] ? find_next_bit+0x27/0x30 [ 248.126547] smp_call_function_many+0x773/0x930 -[ 248.131192] ? smp_call_function_many+0x773/0x930 -[ 248.136013] ? add_nops+0x90/0x90 -[ 248.139449] ? smp_call_function_single_async+0x2d0/0x2d0 -[ 248.144960] ? mark_held_locks+0xb2/0x100 -[ 248.149082] ? text_poke+0x336/0x530 -[ 248.152772] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.157763] ? apic_has_pending_timer+0x80/0x2a0 -[ 248.162488] ? trace_hardirqs_on+0xd/0x10 -[ 248.166606] ? add_nops+0x90/0x90 -[ 248.170033] ? apic_has_pending_timer+0x80/0x2a0 [ 248.174760] on_each_cpu+0x3d/0x1b0 -[ 248.178359] ? apic_has_pending_timer+0x7f/0x2a0 [ 248.183088] text_poke_bp+0xe4/0x170 -[ 248.186774] ? __mutex_unlock_slowpath+0xe9/0xac0 -[ 248.191590] ? poke_int3_handler+0x90/0x90 -[ 248.195800] ? wait_for_completion+0x7b0/0x7b0 -[ 248.200361] ? apic_has_pending_timer+0x7f/0x2a0 [ 248.205094] __jump_label_transform.isra.0+0x6a5/0x8a0 -[ 248.210352] ? bug_at+0x20/0x20 -[ 248.213606] ? check_noncircular+0x20/0x20 -[ 248.217814] ? perf_trace_lock_acquire+0xe3/0x980 -[ 248.222633] ? modules_open+0xa0/0xa0 [ 248.226421] arch_jump_label_transform+0x2f/0x40 [ 248.231152] __jump_label_update+0x207/0x2d0 -[ 248.235539] ? static_key_count+0x110/0x110 -[ 248.239838] ? atomic_dec_and_mutex_lock+0x112/0x150 -[ 248.244917] ? mutex_spin_on_owner+0xb50/0xb50 [ 248.249480] jump_label_update+0x22c/0x2b0 [ 248.253695] static_key_slow_dec_cpuslocked+0x176/0x1d0 -[ 248.259034] ? static_key_slow_inc+0x3c0/0x3c0 -[ 248.263596] ? rcu_read_lock_sched_held+0x108/0x120 [ 248.268594] jump_label_update_timeout+0x1f/0x30 [ 248.273326] process_one_work+0xbfd/0x1be0 -[ 248.277556] ? pwq_dec_nr_in_flight+0x450/0x450 -[ 248.282197] ? finish_task_switch+0x1d3/0x740 -[ 248.286664] ? finish_task_switch+0x1aa/0x740 -[ 248.291159] ? perf_trace_lock_acquire+0xe3/0x980 -[ 248.295987] ? perf_trace_lock+0x900/0x900 -[ 248.300191] ? __sched_text_start+0x8/0x8 -[ 248.304316] ? lock_downgrade+0x980/0x980 -[ 248.308440] ? __wake_up_common_lock+0x190/0x310 -[ 248.313189] ? find_held_lock+0x39/0x1d0 -[ 248.317242] ? lock_acquire+0x1d5/0x580 -[ 248.321189] ? worker_thread+0x4a3/0x1990 -[ 248.325321] ? lock_release+0xda0/0xda0 -[ 248.329271] ? worker_pool_assign_id+0x1b0/0x1b0 -[ 248.334004] ? do_raw_spin_trylock+0x190/0x190 [ 248.338580] worker_thread+0x223/0x1990 -[ 248.342560] ? process_one_work+0x1be0/0x1be0 -[ 248.347034] ? _raw_spin_unlock_irq+0x27/0x70 -[ 248.351505] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.356494] ? trace_hardirqs_on+0xd/0x10 -[ 248.360617] ? finish_task_switch+0x1d3/0x740 -[ 248.365083] ? finish_task_switch+0x1aa/0x740 -[ 248.369558] ? copy_overflow+0x20/0x20 -[ 248.373437] ? __schedule+0x8f3/0x2060 -[ 248.377296] ? check_noncircular+0x20/0x20 -[ 248.381527] ? find_held_lock+0x39/0x1d0 -[ 248.385573] ? find_held_lock+0x39/0x1d0 -[ 248.389625] ? lock_downgrade+0x980/0x980 -[ 248.393751] ? default_wake_function+0x30/0x50 -[ 248.398320] ? __schedule+0x2060/0x2060 -[ 248.402263] ? do_wait_intr+0x3e0/0x3e0 -[ 248.406215] ? do_raw_spin_trylock+0x190/0x190 -[ 248.410772] ? _raw_spin_unlock_irqrestore+0x31/0xba -[ 248.415852] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.420842] ? trace_hardirqs_on+0xd/0x10 -[ 248.424963] ? __kthread_parkme+0x175/0x240 [ 248.429263] kthread+0x37a/0x440 -[ 248.432602] ? process_one_work+0x1be0/0x1be0 -[ 248.437067] ? kthread_stop+0x7b0/0x7b0 [ 248.441018] ret_from_fork+0x24/0x30 [ 248.444727] Code: 00 00 00 fc ff df 44 89 bd 50 ff ff ff 48 c1 e8 03 4c 01 e8 41 83 e7 01 c6 00 f8 74 4e 49 89 c7 49 83 c4 18 e8 4e 25 0b 00 f3 90 <4c> 89 e2 41 c6 07 04 48 c1 ea 03 42 0f b6 14 2a 84 d2 74 09 80 [ 248.463942] Kernel panic - not syncing: softlockup: hung tasks @@ -111,33 +43,11 @@ TITLE: BUG: soft lockup in smp_call_function_many [ 248.492919] Call Trace: [ 248.495472] [ 248.497597] dump_stack+0x194/0x257 -[ 248.501202] ? arch_local_irq_restore+0x53/0x53 -[ 248.505843] ? vprintk_default+0x28/0x30 -[ 248.509878] ? vprintk_func+0x5e/0xc0 -[ 248.513650] ? vsnprintf+0x1ed/0x1900 [ 248.517432] panic+0x1e4/0x41c -[ 248.520597] ? refcount_error_report+0x214/0x214 -[ 248.525340] ? watchdog_timer_fn+0x303/0x320 [ 248.529731] watchdog_timer_fn+0x314/0x320 [ 248.533947] __hrtimer_run_queues+0x349/0xe10 -[ 248.538418] ? lock_downgrade+0x980/0x980 -[ 248.542540] ? watchdog+0x30/0x30 -[ 248.545971] ? hrtimer_cancel+0x40/0x40 -[ 248.549931] ? pvclock_read_flags+0x160/0x160 -[ 248.554417] ? kvm_clock_get_cycles+0x25/0x30 -[ 248.558883] ? ktime_get_update_offsets_now+0x34a/0x520 -[ 248.564229] ? do_timer+0x50/0x50 -[ 248.567661] ? native_apic_msr_write+0x5c/0x80 -[ 248.572213] ? do_raw_spin_trylock+0x190/0x190 -[ 248.576767] ? lapic_next_event+0x54/0x80 -[ 248.580891] ? clockevents_program_event+0x108/0x2e0 [ 248.585978] hrtimer_interrupt+0x1d4/0x5f0 [ 248.590213] smp_apic_timer_interrupt+0x14a/0x700 -[ 248.595032] ? smp_call_function_single_interrupt+0x640/0x640 -[ 248.600889] ? _raw_spin_lock+0x32/0x40 -[ 248.604847] ? handle_edge_irq+0x2b4/0x7c0 -[ 248.609054] ? task_prio+0x40/0x40 -[ 248.612583] ? trace_hardirqs_off_thunk+0x1a/0x1c [ 248.617409] apic_timer_interrupt+0x9d/0xb0 [ 248.621700] [ 248.623909] RIP: 0010:smp_call_function_single+0x364/0x560 @@ -147,87 +57,18 @@ TITLE: BUG: soft lockup in smp_call_function_many [ 248.651662] RBP: ffff8801d2ac7050 R08: 1ffff1003a558dff R09: 0000000000000000 [ 248.658903] R10: ffff8801d2ac7078 R11: 0000000000000000 R12: ffff8801d2ac6ff8 [ 248.666144] R13: dffffc0000000000 R14: 0000000000000000 R15: ffffed003a558df4 -[ 248.673405] ? smp_call_function_single+0x362/0x560 -[ 248.678409] ? add_nops+0x90/0x90 -[ 248.681839] ? generic_exec_single+0x5b0/0x5b0 -[ 248.686393] ? __mutex_lock+0x16f/0x1a80 -[ 248.690427] ? arch_jump_label_transform+0x1b/0x40 -[ 248.695329] ? print_usage_bug+0x3f0/0x3f0 -[ 248.699548] ? _find_next_bit+0xee/0x120 -[ 248.703586] ? add_nops+0x90/0x90 -[ 248.707013] ? find_next_bit+0x27/0x30 [ 248.710884] smp_call_function_many+0x773/0x930 -[ 248.715539] ? smp_call_function_many+0x773/0x930 -[ 248.720362] ? add_nops+0x90/0x90 -[ 248.723796] ? smp_call_function_single_async+0x2d0/0x2d0 -[ 248.729309] ? mark_held_locks+0xb2/0x100 -[ 248.733431] ? text_poke+0x336/0x530 -[ 248.737120] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.742109] ? apic_has_pending_timer+0x80/0x2a0 -[ 248.746838] ? trace_hardirqs_on+0xd/0x10 -[ 248.750957] ? add_nops+0x90/0x90 -[ 248.754383] ? apic_has_pending_timer+0x80/0x2a0 [ 248.759110] on_each_cpu+0x3d/0x1b0 -[ 248.762710] ? apic_has_pending_timer+0x7f/0x2a0 [ 248.767443] text_poke_bp+0xe4/0x170 -[ 248.771131] ? __mutex_unlock_slowpath+0xe9/0xac0 -[ 248.775946] ? poke_int3_handler+0x90/0x90 -[ 248.780158] ? wait_for_completion+0x7b0/0x7b0 -[ 248.784721] ? apic_has_pending_timer+0x7f/0x2a0 [ 248.789452] __jump_label_transform.isra.0+0x6a5/0x8a0 -[ 248.794707] ? bug_at+0x20/0x20 -[ 248.797964] ? check_noncircular+0x20/0x20 -[ 248.802170] ? perf_trace_lock_acquire+0xe3/0x980 -[ 248.806990] ? modules_open+0xa0/0xa0 [ 248.810776] arch_jump_label_transform+0x2f/0x40 [ 248.815507] __jump_label_update+0x207/0x2d0 -[ 248.819895] ? static_key_count+0x110/0x110 -[ 248.824192] ? atomic_dec_and_mutex_lock+0x112/0x150 -[ 248.829269] ? mutex_spin_on_owner+0xb50/0xb50 [ 248.833832] jump_label_update+0x22c/0x2b0 [ 248.838048] static_key_slow_dec_cpuslocked+0x176/0x1d0 -[ 248.843387] ? static_key_slow_inc+0x3c0/0x3c0 -[ 248.847951] ? rcu_read_lock_sched_held+0x108/0x120 [ 248.853695] jump_label_update_timeout+0x1f/0x30 [ 248.858424] process_one_work+0xbfd/0x1be0 -[ 248.862653] ? pwq_dec_nr_in_flight+0x450/0x450 -[ 248.867292] ? finish_task_switch+0x1d3/0x740 -[ 248.871756] ? finish_task_switch+0x1aa/0x740 -[ 248.876251] ? perf_trace_lock_acquire+0xe3/0x980 -[ 248.881080] ? perf_trace_lock+0x900/0x900 -[ 248.885287] ? __sched_text_start+0x8/0x8 -[ 248.889411] ? lock_downgrade+0x980/0x980 -[ 248.893533] ? __wake_up_common_lock+0x190/0x310 -[ 248.898281] ? find_held_lock+0x39/0x1d0 -[ 248.902335] ? lock_acquire+0x1d5/0x580 -[ 248.906283] ? worker_thread+0x4a3/0x1990 -[ 248.910414] ? lock_release+0xda0/0xda0 -[ 248.914363] ? worker_pool_assign_id+0x1b0/0x1b0 -[ 248.919094] ? do_raw_spin_trylock+0x190/0x190 [ 248.923671] worker_thread+0x223/0x1990 -[ 248.927650] ? process_one_work+0x1be0/0x1be0 -[ 248.932122] ? _raw_spin_unlock_irq+0x27/0x70 -[ 248.936593] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 248.941582] ? trace_hardirqs_on+0xd/0x10 -[ 248.945708] ? finish_task_switch+0x1d3/0x740 -[ 248.950177] ? finish_task_switch+0x1aa/0x740 -[ 248.954652] ? copy_overflow+0x20/0x20 -[ 248.958530] ? __schedule+0x8f3/0x2060 -[ 248.962386] ? check_noncircular+0x20/0x20 -[ 248.966615] ? find_held_lock+0x39/0x1d0 -[ 248.970661] ? find_held_lock+0x39/0x1d0 -[ 248.974716] ? lock_downgrade+0x980/0x980 -[ 248.978841] ? default_wake_function+0x30/0x50 -[ 248.983411] ? __schedule+0x2060/0x2060 -[ 248.987355] ? do_wait_intr+0x3e0/0x3e0 -[ 248.991308] ? do_raw_spin_trylock+0x190/0x190 -[ 248.995865] ? _raw_spin_unlock_irqrestore+0x31/0xba -[ 249.000944] ? trace_hardirqs_on_caller+0x421/0x5c0 -[ 249.005933] ? trace_hardirqs_on+0xd/0x10 -[ 249.010055] ? __kthread_parkme+0x175/0x240 [ 249.014357] kthread+0x37a/0x440 -[ 249.017695] ? process_one_work+0x1be0/0x1be0 -[ 249.022163] ? kthread_stop+0x7b0/0x7b0 [ 249.026113] ret_from_fork+0x24/0x30 [ 249.030488] Dumping ftrace buffer: [ 249.034111] (ftrace buffer empty) diff --git a/pkg/report/testdata/linux/report/67 b/pkg/report/testdata/linux/report/67 index 2eef4bbe57d..0976cb34ec0 100644 --- a/pkg/report/testdata/linux/report/67 +++ b/pkg/report/testdata/linux/report/67 @@ -1,4 +1,4 @@ -TITLE: BUG: soft lockup in mulaw_decode +TITLE: BUG: soft lockup in snd_pcm_oss_write [ 562.725743] watchdog: BUG: soft lockup - CPU#0 stuck for 135s! [syzkaller670324:3527] [ 562.733767] Modules linked in: