From bad4246bf2c72fa97640fbf3cf5c092b5173f581 Mon Sep 17 00:00:00 2001 From: Dmitry Vyukov Date: Mon, 16 Jul 2018 17:03:14 +0200 Subject: [PATCH] pkg/report: improve akaros reporter and implement symbolization --- pkg/report/akaros.go | 123 +++++++++++++++++++++++++--- pkg/report/freebsd.go | 2 +- pkg/report/gvisor.go | 2 +- pkg/report/linux.go | 9 +- pkg/report/report.go | 17 ++-- pkg/report/report_test.go | 2 +- pkg/report/testdata/akaros/report/0 | 1 + pkg/report/testdata/akaros/report/1 | 1 + pkg/report/testdata/akaros/report/2 | 31 ++++--- pkg/report/testdata/akaros/report/3 | 1 + pkg/report/testdata/akaros/report/4 | 1 + pkg/report/testdata/akaros/report/5 | 44 ++++++++++ pkg/report/testdata/akaros/report/6 | 83 +++++++++++++++++++ tools/syz-symbolize/symbolize.go | 2 +- 14 files changed, 279 insertions(+), 40 deletions(-) create mode 100644 pkg/report/testdata/akaros/report/5 create mode 100644 pkg/report/testdata/akaros/report/6 diff --git a/pkg/report/akaros.go b/pkg/report/akaros.go index 289a85016cf..126538cfe58 100644 --- a/pkg/report/akaros.go +++ b/pkg/report/akaros.go @@ -4,17 +4,26 @@ package report import ( + "bufio" "bytes" + "fmt" + "path/filepath" "regexp" + "strconv" + "strings" + + "github.com/google/syzkaller/pkg/symbolizer" ) type akaros struct { ignores []*regexp.Regexp + objfile string } func ctorAkaros(kernelSrc, kernelObj string, ignores []*regexp.Regexp) (Reporter, []string, error) { ctx := &akaros{ ignores: ignores, + objfile: filepath.Join(kernelObj, "akaros-kernel-64b"), } return ctx, nil, nil } @@ -52,34 +61,126 @@ func (ctx *akaros) Parse(output []byte) *Report { if oops == nil { return nil } - title, corrupted, _ := extractDescription(output[rep.StartPos:], oops, nil) + title, corrupted, _ := extractDescription(output[rep.StartPos:], oops, akarosStackParams) rep.Title = title - rep.Report = output[rep.StartPos:] + rep.Report = ctx.minimizeReport(output[rep.StartPos:]) rep.Corrupted = corrupted != "" - rep.corruptedReason = corrupted + rep.CorruptedReason = corrupted return rep } func (ctx *akaros) Symbolize(rep *Report) error { + symb := symbolizer.NewSymbolizer() + defer symb.Close() + var symbolized []byte + s := bufio.NewScanner(bytes.NewReader(rep.Report)) + for s.Scan() { + line := bytes.Trim(s.Bytes(), "\r") + line = ctx.symbolizeLine(symb.Symbolize, ctx.objfile, line) + symbolized = append(symbolized, line...) + symbolized = append(symbolized, '\n') + } + rep.Report = symbolized return nil } -// kernel panic at kern/src/vfs.c:1359, from core 1: assertion failed: buf == buf_end -// kernel panic at kern/src/ns/sysfile.c:719, from core 1: assertion failed: n >= sizeof(struct kdirent) -/// $ kernel panic at kern/src/slab.c:518, from core 1: [German Accent]: OOM for a small slab growth!!! +func (ctx *akaros) symbolizeLine(symbFunc func(bin string, pc uint64) ([]symbolizer.Frame, error), + objfile string, line []byte) []byte { + match := akarosSymbolizeRe.FindSubmatchIndex(line) + if match == nil { + return line + } + addr, err := strconv.ParseUint(string(line[match[2]:match[3]]), 0, 64) + if err != nil { + return line + } + frames, err := symbFunc(objfile, addr-1) + if err != nil || len(frames) == 0 { + return line + } + var symbolized []byte + for i, frame := range frames { + if i != 0 { + symbolized = append(symbolized, '\n') + } + file := frame.File + if pos := strings.LastIndex(file, "/kern/"); pos != -1 { + file = file[pos+6:] + } + modified := append([]byte{}, line...) + modified = append(modified, fmt.Sprintf(" at %v:%v", file, frame.Line)...) + if frame.Inline { + modified = replace(modified, match[4], match[5], []byte(frame.Func)) + modified = replace(modified, match[2], match[3], []byte(" [inline] ")) + } + symbolized = append(symbolized, modified...) + } + return symbolized +} + +func (ctx *akaros) minimizeReport(report []byte) []byte { + out := new(bytes.Buffer) + for s := bufio.NewScanner(bytes.NewReader(report)); s.Scan(); { + line := bytes.Trim(s.Bytes(), "\r") + if len(line) == 0 || + bytes.Contains(line, []byte("Entering Nanwan's Dungeon")) || + bytes.Contains(line, []byte("Type 'help' for a list of commands")) { + continue + } + out.Write(line) + out.WriteByte('\n') + } + return out.Bytes() +} + +var ( + akarosSymbolizeRe = compile(`^#[0-9]+ \[\<(0x[0-9a-f]+)\>\] in ([a-zA-Z0-9_]+)`) +) + +var akarosStackParams = &stackParams{ + stackStartRes: []*regexp.Regexp{ + regexp.MustCompile(`Stack Backtrace on Core [0-9]+:`), + }, + frameRes: []*regexp.Regexp{ + compile(`^#[0-9]+ {{PC}} in ([a-zA-Z0-9_]+)`), + }, + skipPatterns: []string{ + "backtrace", + "mon_backtrace", + "monitor", + "_panic", + }, +} + var akarosOopses = []*oops{ &oops{ []byte("kernel panic"), []oopsFormat{ { - title: compile("kernel panic .* assertion failed: (.*)"), - fmt: "assertion failed: %[1]v", - noStackTrace: true, + title: compile("kernel panic at {{SRC}}, from core [0-9]+: assertion failed: (.*)"), + fmt: "assertion failed: %[2]v", + stack: &stackFmt{ + parts: []*regexp.Regexp{ + compile(`Stack Backtrace on Core [0-9]+:`), + parseStackTrace, + }, + }, + }, + { + title: compile("kernel panic at {{SRC}}, from core [0-9]+: (.*)"), + fmt: "kernel panic: %[2]v", + stack: &stackFmt{ + parts: []*regexp.Regexp{ + compile(`Stack Backtrace on Core [0-9]+:`), + parseStackTrace, + }, + }, }, { - title: compile("kernel panic .* from core [0-9]+: (.*)"), - fmt: "kernel panic: %[1]v", + title: compile("kernel panic"), + fmt: "kernel panic", noStackTrace: true, + corrupted: true, }, }, []*regexp.Regexp{}, diff --git a/pkg/report/freebsd.go b/pkg/report/freebsd.go index 6e3bbb7e5f9..1b332eabf21 100644 --- a/pkg/report/freebsd.go +++ b/pkg/report/freebsd.go @@ -69,7 +69,7 @@ func (ctx *freebsd) Parse(output []byte) *Report { title, corrupted, _ := extractDescription(output[rep.StartPos:], oops, freebsdStackParams) rep.Title = title rep.Corrupted = corrupted != "" - rep.corruptedReason = corrupted + rep.CorruptedReason = corrupted return rep } diff --git a/pkg/report/gvisor.go b/pkg/report/gvisor.go index 05a73114276..5f4e854c6f0 100644 --- a/pkg/report/gvisor.go +++ b/pkg/report/gvisor.go @@ -67,7 +67,7 @@ func (ctx *gvisor) Parse(output []byte) *Report { rep.Title = replaceTable(gvisorTitleReplacement, title) rep.Report = ctx.shortenReport(output[rep.StartPos:]) rep.Corrupted = corrupted != "" - rep.corruptedReason = corrupted + rep.CorruptedReason = corrupted return rep } diff --git a/pkg/report/linux.go b/pkg/report/linux.go index 1ba3079140a..7e85102d24a 100644 --- a/pkg/report/linux.go +++ b/pkg/report/linux.go @@ -246,7 +246,7 @@ func (ctx *linux) Parse(output []byte) *Report { } rep.Title = title rep.Corrupted = corrupted != "" - rep.corruptedReason = corrupted + rep.CorruptedReason = corrupted // Prepend 5 lines preceding start of the report, // they can contain additional info related to the report. for _, prefix := range reportPrefix { @@ -255,7 +255,7 @@ func (ctx *linux) Parse(output []byte) *Report { } rep.Report = append(rep.Report, report...) if !rep.Corrupted { - rep.Corrupted, rep.corruptedReason = ctx.isCorrupted(title, report, format) + rep.Corrupted, rep.CorruptedReason = ctx.isCorrupted(title, report, format) } return rep } @@ -445,10 +445,6 @@ func (ctx *linux) extractFiles(report []byte) []string { } func (ctx *linux) isCorrupted(title string, report []byte, format oopsFormat) (bool, string) { - // Check if this crash format is marked as corrupted. - if format.corrupted { - return true, "report format is marked as corrupted" - } // Check if the report contains stack trace. if !format.noStackTrace && !bytes.Contains(report, []byte("Call Trace")) && !bytes.Contains(report, []byte("backtrace")) { @@ -496,7 +492,6 @@ func (ctx *linux) isCorrupted(title string, report []byte, format oopsFormat) (b } var ( - filenameRe = regexp.MustCompile(`[a-zA-Z0-9_\-\./]*[a-zA-Z0-9_\-]+\.(c|h):[0-9]+`) linuxSymbolizeRe = regexp.MustCompile(`(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`) stackFrameRe = regexp.MustCompile(`^ *(?:\[\<(?:[0-9a-f]+)\>\])?[ \t]+(?:[0-9]+:)?([a-zA-Z0-9_.]+)\+0x([0-9a-f]+)/0x([0-9a-f]+)`) linuxRcuStall = compile("INFO: rcu_(?:preempt|sched|bh) (?:self-)?detected(?: expedited)? stall") diff --git a/pkg/report/report.go b/pkg/report/report.go index d49f5cd36e3..934adc8988b 100644 --- a/pkg/report/report.go +++ b/pkg/report/report.go @@ -41,8 +41,8 @@ type Report struct { Suppressed bool // Corrupted indicates whether the report is truncated of corrupted in some other way. Corrupted bool - // corruptedReason contains reason why the report is marked as corrupted. - corruptedReason string + // CorruptedReason contains reason why the report is marked as corrupted. + CorruptedReason string // Maintainers is list of maintainer emails. Maintainers []string } @@ -239,7 +239,7 @@ var parseStackTrace *regexp.Regexp func compile(re string) *regexp.Regexp { re = strings.Replace(re, "{{ADDR}}", "0x[0-9a-f]+", -1) - re = strings.Replace(re, "{{PC}}", "\\[\\<[0-9a-f]+\\>\\]", -1) + re = strings.Replace(re, "{{PC}}", "\\[\\<(?:0x)?[0-9a-f]+\\>\\]", -1) re = strings.Replace(re, "{{FUNC}}", "([a-zA-Z0-9_]+)(?:\\.|\\+)", -1) re = strings.Replace(re, "{{SRC}}", "([a-zA-Z0-9-_/.]+\\.[a-z]+:[0-9]+)", -1) return regexp.MustCompile(re) @@ -341,6 +341,9 @@ func extractDescription(output []byte, oops *oops, params *stackParams) ( } desc = string(output[pos:end]) } + if corrupted == "" && format.corrupted { + corrupted = "report format is marked as corrupted" + } return } @@ -378,7 +381,7 @@ nextPart: for _, part := range parts { if part == parseStackTrace { for s.Scan() { - ln := s.Bytes() + ln := bytes.Trim(s.Bytes(), "\r") if corrupted == "" && matchesAny(ln, params.corruptedLines) { corrupted = "corrupted line in report (1)" } @@ -402,7 +405,7 @@ nextPart: } } else { for s.Scan() { - ln := s.Bytes() + ln := bytes.Trim(s.Bytes(), "\r") if corrupted == "" && matchesAny(ln, params.corruptedLines) { corrupted = "corrupted line in report (2)" } @@ -445,3 +448,7 @@ func replace(where []byte, start, end int, what []byte) []byte { } return where } + +var ( + filenameRe = regexp.MustCompile(`[a-zA-Z0-9_\-\./]*[a-zA-Z0-9_\-]+\.(c|h):[0-9]+`) +) diff --git a/pkg/report/report_test.go b/pkg/report/report_test.go index f3d53245905..1597c3f3655 100644 --- a/pkg/report/report_test.go +++ b/pkg/report/report_test.go @@ -139,7 +139,7 @@ func testParseImpl(t *testing.T, reporter Reporter, test *ParseTest) { if rep != nil { title = rep.Title corrupted = rep.Corrupted - corruptedReason = rep.corruptedReason + corruptedReason = rep.CorruptedReason suppressed = rep.Suppressed } if title != test.Title || corrupted != test.Corrupted || suppressed != test.Suppressed { diff --git a/pkg/report/testdata/akaros/report/0 b/pkg/report/testdata/akaros/report/0 index a016ea92658..d650a059a39 100644 --- a/pkg/report/testdata/akaros/report/0 +++ b/pkg/report/testdata/akaros/report/0 @@ -1,4 +1,5 @@ TITLE: kernel panic: [German Accent]: OOM for a small slab growth!!! +CORRUPTED: Y / $ kernel panic at kern/src/slab.c:518, from core 1: [German Accent]: OOM for a small slab growth!!! diff --git a/pkg/report/testdata/akaros/report/1 b/pkg/report/testdata/akaros/report/1 index 361a4dc9fcc..131a1ac68a8 100644 --- a/pkg/report/testdata/akaros/report/1 +++ b/pkg/report/testdata/akaros/report/1 @@ -1,4 +1,5 @@ TITLE: kernel panic: Proc-ful Page Fault in the Kernel at ADDR! +CORRUPTED: Y HW TRAP frame at 0xfffffff00006ab90 on core 0 rax 0xffff8000044a0b60 diff --git a/pkg/report/testdata/akaros/report/2 b/pkg/report/testdata/akaros/report/2 index 818497af7b7..500200672f7 100644 --- a/pkg/report/testdata/akaros/report/2 +++ b/pkg/report/testdata/akaros/report/2 @@ -1,15 +1,20 @@ -TITLE: assertion failed: buf == buf_end +TITLE: assertion failed: rpi->gp_acked + 1 == READ_ONCE(rsp->gpnum) + +kernel panic at kern/src/rcu.c:325, from core 3: assertion failed: rpi->gp_acked + 1 == READ_ONCE(rsp->gpnum) + +Stack Backtrace on Core 3: + +#01 [<0xffffffffc200a3b7>] in backtrace + +#02 [<0xffffffffc2009b7c>] in _panic + +#03 [<0xffffffffc20502d9>] in rcu_report_qs_rpi + +#04 [<0xffffffffc2050c2c>] in rcu_report_qs + +#05 [<0xffffffffc2054343>] in __smp_idle + + +Entering Nanwan's Dungeon on Core 3 (Ints off): -kernel panic at kern/src/vfs.c:1359, from core 1: assertion failed: buf == buf_end -Entering Nanwan's Dungeon on Core 1 (Ints on): Type 'help' for a list of commands. -ROS(Core 1)> bash-4.3$ bt -Stack Backtrace on Core 1: -#01 [<0xffffffffc2016074>] in mon_backtrace -#02 [<0xffffffffc2017177>] in monitor -#03 [<0xffffffffc200cbfc>] in _panic -#04 [<0xffffffffc205c51c>] in generic_file_write -#05 [<0xffffffffc2053890>] in sys_write -#06 [<0xffffffffc2056919>] in syscall -#07 [<0xffffffffc2056ad4>] in run_local_syscall -#08 [<0xffffffffc20a28aa>] in sysenter_callwrapper diff --git a/pkg/report/testdata/akaros/report/3 b/pkg/report/testdata/akaros/report/3 index 104c05126b3..afa8d9d39e5 100644 --- a/pkg/report/testdata/akaros/report/3 +++ b/pkg/report/testdata/akaros/report/3 @@ -1,4 +1,5 @@ TITLE: assertion failed: page && pm_slot_check_refcnt(*page->pg_tree_slot) +CORRUPTED: Y Unhandled user trap in vcore context from VC 1 HW TRAP frame (partial) at 0xffffffffc82cbd20 on core 1 diff --git a/pkg/report/testdata/akaros/report/4 b/pkg/report/testdata/akaros/report/4 index 99e36f87ce0..2c260f01a2a 100644 --- a/pkg/report/testdata/akaros/report/4 +++ b/pkg/report/testdata/akaros/report/4 @@ -1,4 +1,5 @@ TITLE: assertion failed: n >= sizeof(struct kdirent) +CORRUPTED: Y kernel panic at kern/src/ns/sysfile.c:719, from core 1: assertion failed: n >= sizeof(struct kdirent) diff --git a/pkg/report/testdata/akaros/report/5 b/pkg/report/testdata/akaros/report/5 new file mode 100644 index 00000000000..704a23ad585 --- /dev/null +++ b/pkg/report/testdata/akaros/report/5 @@ -0,0 +1,44 @@ +TITLE: assertion failed: !pte_is_mapped(pte) + +kernel panic at kern/src/mm.c:737, from core 1: assertion failed: !pte_is_mapped(pte) + +Stack Backtrace on Core 1: + +#01 [<0xffffffffc200a3b7>] in backtrace + +#02 [<0xffffffffc2009b7c>] in _panic + +#03 [<0xffffffffc200db41>] in map_page_at_addr + +#04 [<0xffffffffc200db92>] in populate_anon_va + +#05 [<0xffffffffc200fbf2>] in populate_va + +#06 [<0xffffffffc20560f0>] in sys_populate_va + +#07 [<0xffffffffc2059129>] in syscall + +#08 [<0xffffffffc20592e4>] in run_local_syscall + +#09 [<0xffffffffc2059819>] in prep_syscalls + +#10 [<0xffffffffc20aaefa>] in sysenter_callwrapper + + +Entering Nanwan's Dungeon on Core 1 (Ints on): + +Type 'help' for a list of commands. + +REPORT: +kernel panic at kern/src/mm.c:737, from core 1: assertion failed: !pte_is_mapped(pte) +Stack Backtrace on Core 1: +#01 [<0xffffffffc200a3b7>] in backtrace +#02 [<0xffffffffc2009b7c>] in _panic +#03 [<0xffffffffc200db41>] in map_page_at_addr +#04 [<0xffffffffc200db92>] in populate_anon_va +#05 [<0xffffffffc200fbf2>] in populate_va +#06 [<0xffffffffc20560f0>] in sys_populate_va +#07 [<0xffffffffc2059129>] in syscall +#08 [<0xffffffffc20592e4>] in run_local_syscall +#09 [<0xffffffffc2059819>] in prep_syscalls +#10 [<0xffffffffc20aaefa>] in sysenter_callwrapper diff --git a/pkg/report/testdata/akaros/report/6 b/pkg/report/testdata/akaros/report/6 new file mode 100644 index 00000000000..61423b170bf --- /dev/null +++ b/pkg/report/testdata/akaros/report/6 @@ -0,0 +1,83 @@ +TITLE: kernel panic: Proc-ful Page Fault in the Kernel at ADDR! + +HW TRAP frame at 0xfffffff00006dd00 on core 3 + + rax 0x0000000000000001 + + rbx 0x00000002deadbabe + + rcx 0x0000000000000080 + + rdx 0x00000002deadbac6 + + rbp 0xfffffff00006ddc8 + + rsi 0x0000000000000003 + + rdi 0x00000002deadbac6 + + r8 0x0000000000000000 + + r9 0x0000000000000000 + + r10 0x000010000000a4c0 + + r11 0x0000000000000206 + + r12 0xffff800002182e60 + + r13 0x00000000ffffffff + + r14 0x0000000000000004 + + r15 0xffff800002182ac0 + + trap 0x0000000e Page Fault + + gsbs 0xffffffffc8704140 + + fsbs 0x0000000000000000 + + err 0x--------00000000 + + rip 0xffffffffc2007ab7 + + cs 0x------------0008 + + flag 0x0000000000010206 + + rsp 0xfffffff00006ddc8 + + ss 0x------------0010 + + +Backtrace of kernel context on Core 3: + +#01 [<0xffffffffc2007ab7>] in kref_put + +#02 [<0xffffffffc2007e36>] in remove_fd_tap + +#03 [<0xffffffffc205846c>] in sys_tap_fds + +#04 [<0xffffffffc2059129>] in syscall + +#05 [<0xffffffffc20592e4>] in run_local_syscall + +#06 [<0xffffffffc2059819>] in prep_syscalls + +#07 [<0xffffffffc20aaefa>] in sysenter_callwrapper + +kernel panic at kern/arch/x86/trap.c:311, from core 3: Proc-ful Page Fault in the Kernel at 0x00000002deadbac6! + +Stack Backtrace on Core 3: + +#01 [<0xffffffffc200a3b7>] in backtrace + +#02 [<0xffffffffc2009b7c>] in _panic + +#03 [<0xffffffffc20aa629>] in trap + + +Entering Nanwan's Dungeon on Core 3 (Ints off): + +Type 'help' for a list of commands. diff --git a/tools/syz-symbolize/symbolize.go b/tools/syz-symbolize/symbolize.go index 9020d2fcc91..169dad84013 100644 --- a/tools/syz-symbolize/symbolize.go +++ b/tools/syz-symbolize/symbolize.go @@ -51,7 +51,7 @@ func main() { os.Exit(1) } fmt.Printf("TITLE: %v\n", rep.Title) - fmt.Printf("CORRUPTED: %v\n", rep.Corrupted) + fmt.Printf("CORRUPTED: %v (%v)\n", rep.Corrupted, rep.CorruptedReason) fmt.Printf("\n") os.Stdout.Write(rep.Report) }