From bddbd303478c09cf72c3cc09231c35015c225f78 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:05:01 -0700 Subject: [PATCH 01/28] Use Go implementation of stackcollapse-perf to eliminate Perl dependency. Signed-off-by: Harper, Jason M --- internal/script/script_defs.go | 2 +- tools/Makefile | 20 +- .../stackcollapse-perf/stackcollapse-perf.go | 269 ++++++++++++++++++ 3 files changed, 278 insertions(+), 13 deletions(-) create mode 100644 tools/stackcollapse-perf/stackcollapse-perf.go diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 89c6e111..99e8000c 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1410,7 +1410,7 @@ fi rm -f "$perf_fp_data" "$perf_dwarf_data" "$perf_dwarf_folded" "$perf_fp_folded" `, Superuser: true, - Depends: []string{"perf", "stackcollapse-perf.pl"}, + Depends: []string{"perf", "stackcollapse-perf"}, }, // lock analysis scripts ProfileKernelLockScriptName: { diff --git a/tools/Makefile b/tools/Makefile index 49c23486..9095c7fc 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -5,9 +5,9 @@ # default: tools -.PHONY: default tools async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool lshw lspci msr-tools pcm perf processwatch spectre-meltdown-checker sshpass stress-ng sysstat tsc turbostat +.PHONY: default tools async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm perf processwatch spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat -tools: async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stress-ng sysstat tsc turbostat +tools: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat mkdir -p bin cp -R async-profiler bin/ cp avx-turbo/avx-turbo bin/ @@ -15,7 +15,7 @@ tools: async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool cp dmidecode/dmidecode bin/ cp ethtool/ethtool bin/ cp fio/fio bin/ - cp flamegraph/stackcollapse-perf.pl bin/ + cp stackcollapse-perf/stackcollapse-perf bin/ cp ipmitool/src/ipmitool.static bin/ipmitool cp lshw/src/lshw-static bin/lshw cp lspci/lspci bin/ @@ -97,13 +97,6 @@ ifeq ("$(wildcard fio/config.log)","") endif cd fio && make -flamegraph: -ifeq ("$(wildcard flamegraph)","") - git clone https://github.com/brendangregg/FlameGraph.git flamegraph - # small modification to script to include module name in output - cd flamegraph && sed -i '382 a \\t\t\t\t$$func = \$$func."'" "'".\$$mod;\t# add module name' stackcollapse-perf.pl -endif - ipmitool: ifeq ("$(wildcard ipmitool)","") git clone https://github.com/ipmitool/ipmitool.git @@ -195,6 +188,9 @@ ifeq ("$(wildcard sshpass)","") endif cd sshpass && make +stackcollapse-perf: + cd stackcollapse-perf && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build + stress-ng: ifeq ("$(wildcard stress-ng)","") git clone https://github.com/ColinIanKing/stress-ng.git @@ -232,7 +228,6 @@ reset: cd dmidecode && git clean -fdx && git reset --hard cd ethtool && git clean -fdx && git reset --hard cd fio && git clean -fdx && git reset --hard - cd flamegraph && git clean -fdx && git reset --hard cd ipmitool && git clean -fdx && git reset --hard cd lshw && git clean -fdx && git reset --hard cd lspci && git clean -fdx && git reset --hard @@ -240,6 +235,7 @@ reset: cd msr-tools && git clean -fdx && git reset --hard cd spectre-meltdown-checker cd sshpass && make clean + cd stackcollapse-perf && git clean -fdx && git reset --hard cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc @@ -255,5 +251,5 @@ libcrypt.tar.gz: libs: glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz oss-source: reset libs - tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ flamegraph/ ipmitool/ lshw/ lspci/ msr-tools/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz + tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz md5sum oss_source.tgz > oss_source.tgz.md5 diff --git a/tools/stackcollapse-perf/stackcollapse-perf.go b/tools/stackcollapse-perf/stackcollapse-perf.go new file mode 100644 index 00000000..ca63125b --- /dev/null +++ b/tools/stackcollapse-perf/stackcollapse-perf.go @@ -0,0 +1,269 @@ +package main + +// Copyright (C) 2021-2025 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +// This code is a port of the Perl script stackcollapse-perf.pl from Brendan +// Gregg's Flamegraph project -- github.com/brendangregg/FlameGraph. +// All credit to Brendan Gregg for the original implementation and the flamegraph concept. + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" +) + +// In the original Perl code, the following +// are command line arguments. For our use, we don't need to set them as flags, +// but we can keep them for compatibility with the original +var annotateKernel = false +var annotateJit = false + +// var annotateAll = false +var includePname = true +var includePid = false +var includeTid = false +var includeAddrs = false +var tidyJava = true +var tidyGeneric = true + +// var targetPname = "" +var eventFilter = "" + +// var showInline = false +// var showContext = false +// var srcLineInInput = false + +type StackAggregator struct { + collapsed map[string]int +} + +func NewStackAggregator() *StackAggregator { + return &StackAggregator{collapsed: make(map[string]int)} +} + +func (sa *StackAggregator) RememberStack(stack string, count int) { + sa.collapsed[stack] += count +} + +func main() { + aggregator := NewStackAggregator() + + var input *os.File + var err error + + // Check if a file path is provided as a command-line argument + if len(os.Args) > 1 { + input, err = os.Open(os.Args[1]) // Open the file + if err != nil { + fmt.Fprintf(os.Stderr, "Error opening file: %s\n", err) + os.Exit(1) + } + defer input.Close() + } else { + input = os.Stdin // Default to standard input + } + scanner := bufio.NewScanner(input) + + var stack []string + var pname string + var mPeriod int + var mTid string + var mPid string + + eventRegex := regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) + eventTypeRegex := regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) + stackLineRegex := regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) + // inlineRegex := regexp.MustCompile(`(perf-\d+.map|kernel\.|\[[^\]]+\])`) + stripSymbolsRegex := regexp.MustCompile(`\+0x[\da-f]+$`) + stripIdRegex := regexp.MustCompile(`\.\(.*\)\.`) + stripAnonymousRegex := regexp.MustCompile(`\([^a]*anonymous namespace[^)]*\)`) + jitRegex := regexp.MustCompile(`/tmp/perf-\d+\.map`) + + var eventDefaulted bool + var eventWarning bool + + // main loop, read lines from stdin + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "# cmdline") { + // loop through the command line arguments in reverse order + for i := len(os.Args) - 1; i > 0; i-- { + if !strings.HasPrefix(os.Args[i], "-") { + // not used -> target_pname = filepath.Base(os.Args[i]) + break + } + } + } + + // Skip remaining comments + if strings.HasPrefix(line, "#") { + continue + } + + // End of stack + if line == "" { + if pname == "" { + continue + } + if includePname { + // prepend the process name to the stack + stack = append([]string{pname}, stack...) + } + + if stack != nil { + aggregator.RememberStack(strings.Join(stack, ";"), mPeriod) + } + stack = nil + pname = "" + continue + } + + // Event record start + if matches := eventRegex.FindStringSubmatch(line); matches != nil { + comm, pid, tid, period := matches[1], matches[2], matches[3], "" + if tid == "" { + tid = pid + pid = "?" + } + + if eventMatches := eventTypeRegex.FindStringSubmatch(line); eventMatches != nil { + period = eventMatches[1] + event := eventMatches[2] + + if eventFilter == "" { + eventFilter = event + eventDefaulted = true + } else if event != eventFilter { + if eventDefaulted && !eventWarning { + fmt.Fprintf(os.Stderr, "Filtering for events of type: %s\n", event) + eventWarning = true + } + continue + } + } + + if period == "" { + period = "1" + } + periodInt, err := strconv.Atoi(period) + if err != nil { + fmt.Fprintf(os.Stderr, "Error parsing period: %s\n", err) + continue + } + mPid, mTid, mPeriod = pid, tid, periodInt + + if includeTid { + pname = fmt.Sprintf("%s-%s/%s", comm, mPid, mTid) + } else if includePid { + pname = fmt.Sprintf("%s-%s", comm, mPid) + } else { + pname = comm + } + pname = strings.ReplaceAll(pname, " ", "_") + continue + // Stack line + } else if matches := stackLineRegex.FindStringSubmatch(line); matches != nil { + if pname == "" { + continue + } + pc, rawFunc, mod := matches[1], matches[2], matches[3] + + // skip for now as showInline is always false + // if showInline && !inlineRegex.MatchString(mod) { + // inlineRes := inline(pc, rawFunc, mod) + // if inlineRes != "" && inlineRes != "??" && inlineRes != "??:??:0" { + // // prepend the inline result to the stack + // stack = append([]string{inlineRes}, stack...) + // continue + // } + //} + + // strip symbol offsets from rawFunc + // symbol offsets match this regex: \+0x[\da-f]+$ + rawFunc = stripSymbolsRegex.ReplaceAllString(rawFunc, "") + + // skip process names + if strings.HasPrefix(rawFunc, "(") { + continue + } + // var isUnknown bool + var inline []string + for funcname := range strings.SplitSeq(rawFunc, "->") { + if funcname == "[unknown]" { // use module name instead, if known + if mod != "[unknown]" { + funcname = filepath.Base(mod) + } else { + funcname = "unknown" + // isUnknown = true + } + + if includeAddrs { + funcname = fmt.Sprintf("[%s <%s>]", funcname, pc) + } else { + funcname = fmt.Sprintf("[%s]", funcname) + } + } + if tidyGeneric { + funcname = strings.ReplaceAll(funcname, ";", ":") + if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { + index := stripAnonymousRegex.FindStringIndex(funcname) + funcname = funcname[0:index[0]] + } + funcname = strings.ReplaceAll(funcname, "\"", "") + funcname = strings.ReplaceAll(funcname, "'", "") + } + if tidyJava { + if strings.Contains(funcname, "/") { + // strip the leading L + funcname = strings.TrimPrefix(funcname, "L") + } + } + // annotations + if len(inline) > 0 { + if !strings.Contains(funcname, "_[i]") { + funcname = fmt.Sprintf("%s_[i]", funcname) + } else if annotateKernel && (strings.HasPrefix(funcname, "[") || strings.HasSuffix(funcname, "vmlinux")) && !strings.Contains(mod, "unknown") { + funcname = fmt.Sprintf("%s_[k]", funcname) + } else if annotateJit && jitRegex.MatchString(funcname) { + if !strings.Contains(funcname, "_[j]") { + funcname = fmt.Sprintf("%s_[j]", funcname) + } + } + } + + // source lines + // skip for now since srcLineInInput is always false + // if srcLineInInput && !isUnknown { + // } + + inline = append(inline, funcname) + } + + // prepend inline array to the stack array + if len(inline) > 0 { + stack = append(inline, stack...) + } + + } else { + fmt.Fprintf(os.Stderr, "Unknown line format: %s\n", line) + } + } + + // Output results + keys := make([]string, 0, len(aggregator.collapsed)) + for k := range aggregator.collapsed { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + fmt.Printf("%s %d\n", k, aggregator.collapsed[k]) + } +} From e6cb12828aec2710b9bf93d651688e938db8dc42 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:54:07 -0700 Subject: [PATCH 02/28] Add go.mod file for stackcollapse-perf module Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tools/stackcollapse-perf/go.mod diff --git a/tools/stackcollapse-perf/go.mod b/tools/stackcollapse-perf/go.mod new file mode 100644 index 00000000..0ec5c58d --- /dev/null +++ b/tools/stackcollapse-perf/go.mod @@ -0,0 +1,3 @@ +module intel.com/stackcolapse-perf + +go 1.24.1 From 64bd60f6798ec1962558c4769694cf06cabefcfe Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:56:49 -0700 Subject: [PATCH 03/28] spelling Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/stackcollapse-perf/go.mod b/tools/stackcollapse-perf/go.mod index 0ec5c58d..cbed3b20 100644 --- a/tools/stackcollapse-perf/go.mod +++ b/tools/stackcollapse-perf/go.mod @@ -1,3 +1,3 @@ -module intel.com/stackcolapse-perf +module intel.com/stackcollapse-perf go 1.24.1 From 9c0bff0e48667d7ec47ffde907c0e340c1b7634b Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 11:17:25 -0700 Subject: [PATCH 04/28] Remove stackcollapse-perf reset command from Makefile Signed-off-by: Harper, Jason M --- tools/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/Makefile b/tools/Makefile index 9095c7fc..75d9bb29 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -235,7 +235,6 @@ reset: cd msr-tools && git clean -fdx && git reset --hard cd spectre-meltdown-checker cd sshpass && make clean - cd stackcollapse-perf && git clean -fdx && git reset --hard cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc From dbe2716fd6929b467ca508d08e241300e2f2bc5f Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 13:14:35 -0700 Subject: [PATCH 05/28] use go stackcollapse Signed-off-by: Harper, Jason M --- THIRD_PARTY_PROGRAMS | 23 ----------------------- internal/script/script_defs.go | 4 ++-- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/THIRD_PARTY_PROGRAMS b/THIRD_PARTY_PROGRAMS index a0bf8b81..5138ef76 100644 --- a/THIRD_PARTY_PROGRAMS +++ b/THIRD_PARTY_PROGRAMS @@ -567,29 +567,6 @@ NO WARRANTY END OF TERMS AND CONDITIONS ------------------------------------------------------------- -stackcollapse-perf.pl -# Copyright 2012 Joyent, Inc. All rights reserved. -# Copyright 2012 Brendan Gregg. All rights reserved. -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at docs/cddl1.txt or -# http://opensource.org/licenses/CDDL-1.0. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at docs/cddl1.txt. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -------------------------------------------------------------- stress-ng Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 99e8000c..ce9c078b 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1393,8 +1393,8 @@ fi wait ${perf_fp_pid} ${perf_dwarf_pid} # collapse perf data -perf script -i "$perf_dwarf_data" | stackcollapse-perf.pl > "$perf_dwarf_folded" -perf script -i "$perf_fp_data" | stackcollapse-perf.pl > "$perf_fp_folded" +perf script -i "$perf_dwarf_data" | stackcollapse-perf > "$perf_dwarf_folded" +perf script -i "$perf_fp_data" | stackcollapse-perf > "$perf_fp_folded" # Display results if [ -f "$perf_dwarf_folded" ]; then From 22da023eba3e8ff1249a556159b308c724b55817 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:05:01 -0700 Subject: [PATCH 06/28] Use Go implementation of stackcollapse-perf to eliminate Perl dependency. Signed-off-by: Harper, Jason M --- internal/script/script_defs.go | 2 +- tools/Makefile | 21 +- .../stackcollapse-perf/stackcollapse-perf.go | 269 ++++++++++++++++++ 3 files changed, 278 insertions(+), 14 deletions(-) create mode 100644 tools/stackcollapse-perf/stackcollapse-perf.go diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 89c6e111..99e8000c 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1410,7 +1410,7 @@ fi rm -f "$perf_fp_data" "$perf_dwarf_data" "$perf_dwarf_folded" "$perf_fp_folded" `, Superuser: true, - Depends: []string{"perf", "stackcollapse-perf.pl"}, + Depends: []string{"perf", "stackcollapse-perf"}, }, // lock analysis scripts ProfileKernelLockScriptName: { diff --git a/tools/Makefile b/tools/Makefile index afb59d72..ee9273d2 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -5,9 +5,9 @@ # default: tools -.PHONY: default tools async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool lshw lspci msr-tools pcm perf processwatch spectre-meltdown-checker sshpass stress-ng sysstat tsc turbostat +.PHONY: default tools async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm perf processwatch spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat -tools: async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stress-ng sysstat tsc turbostat +tools: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat mkdir -p bin cp -R async-profiler bin/ cp avx-turbo/avx-turbo bin/ @@ -15,7 +15,7 @@ tools: async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool cp dmidecode/dmidecode bin/ cp ethtool/ethtool bin/ cp fio/fio bin/ - cp flamegraph/stackcollapse-perf.pl bin/ + cp stackcollapse-perf/stackcollapse-perf bin/ cp ipmitool/src/ipmitool.static bin/ipmitool cp lshw/src/lshw-static bin/lshw cp lspci/lspci bin/ @@ -103,14 +103,6 @@ ifeq ("$(wildcard fio/config.log)","") endif cd fio && make -FLAMEGRAPH_VERSION := "master" -flamegraph: -ifeq ("$(wildcard flamegraph)","") - git clone https://github.com/brendangregg/FlameGraph.git flamegraph - # small modification to script to include module name in output - cd flamegraph && sed -i '382 a \\t\t\t\t$$func = \$$func."'" "'".\$$mod;\t# add module name' stackcollapse-perf.pl -endif - IPMITOOL_VERSION := "IPMITOOL_1_8_19" ipmitool: ifeq ("$(wildcard ipmitool)","") @@ -213,6 +205,9 @@ ifeq ("$(wildcard sshpass)","") endif cd sshpass && make +stackcollapse-perf: + cd stackcollapse-perf && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build + STRESS_NG_VERSION := "V0.13.08" stress-ng: ifeq ("$(wildcard stress-ng)","") @@ -252,7 +247,6 @@ reset: cd dmidecode && git clean -fdx && git reset --hard cd ethtool && git clean -fdx && git reset --hard cd fio && git clean -fdx && git reset --hard - cd flamegraph && git clean -fdx && git reset --hard cd ipmitool && git clean -fdx && git reset --hard cd lshw && git clean -fdx && git reset --hard cd lspci && git clean -fdx && git reset --hard @@ -260,6 +254,7 @@ reset: cd msr-tools && git clean -fdx && git reset --hard cd spectre-meltdown-checker cd sshpass && make clean + cd stackcollapse-perf && git clean -fdx && git reset --hard cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc @@ -275,5 +270,5 @@ libcrypt.tar.gz: libs: glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz oss-source: reset libs - tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ flamegraph/ ipmitool/ lshw/ lspci/ msr-tools/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz + tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz md5sum oss_source.tgz > oss_source.tgz.md5 diff --git a/tools/stackcollapse-perf/stackcollapse-perf.go b/tools/stackcollapse-perf/stackcollapse-perf.go new file mode 100644 index 00000000..ca63125b --- /dev/null +++ b/tools/stackcollapse-perf/stackcollapse-perf.go @@ -0,0 +1,269 @@ +package main + +// Copyright (C) 2021-2025 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +// This code is a port of the Perl script stackcollapse-perf.pl from Brendan +// Gregg's Flamegraph project -- github.com/brendangregg/FlameGraph. +// All credit to Brendan Gregg for the original implementation and the flamegraph concept. + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" +) + +// In the original Perl code, the following +// are command line arguments. For our use, we don't need to set them as flags, +// but we can keep them for compatibility with the original +var annotateKernel = false +var annotateJit = false + +// var annotateAll = false +var includePname = true +var includePid = false +var includeTid = false +var includeAddrs = false +var tidyJava = true +var tidyGeneric = true + +// var targetPname = "" +var eventFilter = "" + +// var showInline = false +// var showContext = false +// var srcLineInInput = false + +type StackAggregator struct { + collapsed map[string]int +} + +func NewStackAggregator() *StackAggregator { + return &StackAggregator{collapsed: make(map[string]int)} +} + +func (sa *StackAggregator) RememberStack(stack string, count int) { + sa.collapsed[stack] += count +} + +func main() { + aggregator := NewStackAggregator() + + var input *os.File + var err error + + // Check if a file path is provided as a command-line argument + if len(os.Args) > 1 { + input, err = os.Open(os.Args[1]) // Open the file + if err != nil { + fmt.Fprintf(os.Stderr, "Error opening file: %s\n", err) + os.Exit(1) + } + defer input.Close() + } else { + input = os.Stdin // Default to standard input + } + scanner := bufio.NewScanner(input) + + var stack []string + var pname string + var mPeriod int + var mTid string + var mPid string + + eventRegex := regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) + eventTypeRegex := regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) + stackLineRegex := regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) + // inlineRegex := regexp.MustCompile(`(perf-\d+.map|kernel\.|\[[^\]]+\])`) + stripSymbolsRegex := regexp.MustCompile(`\+0x[\da-f]+$`) + stripIdRegex := regexp.MustCompile(`\.\(.*\)\.`) + stripAnonymousRegex := regexp.MustCompile(`\([^a]*anonymous namespace[^)]*\)`) + jitRegex := regexp.MustCompile(`/tmp/perf-\d+\.map`) + + var eventDefaulted bool + var eventWarning bool + + // main loop, read lines from stdin + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "# cmdline") { + // loop through the command line arguments in reverse order + for i := len(os.Args) - 1; i > 0; i-- { + if !strings.HasPrefix(os.Args[i], "-") { + // not used -> target_pname = filepath.Base(os.Args[i]) + break + } + } + } + + // Skip remaining comments + if strings.HasPrefix(line, "#") { + continue + } + + // End of stack + if line == "" { + if pname == "" { + continue + } + if includePname { + // prepend the process name to the stack + stack = append([]string{pname}, stack...) + } + + if stack != nil { + aggregator.RememberStack(strings.Join(stack, ";"), mPeriod) + } + stack = nil + pname = "" + continue + } + + // Event record start + if matches := eventRegex.FindStringSubmatch(line); matches != nil { + comm, pid, tid, period := matches[1], matches[2], matches[3], "" + if tid == "" { + tid = pid + pid = "?" + } + + if eventMatches := eventTypeRegex.FindStringSubmatch(line); eventMatches != nil { + period = eventMatches[1] + event := eventMatches[2] + + if eventFilter == "" { + eventFilter = event + eventDefaulted = true + } else if event != eventFilter { + if eventDefaulted && !eventWarning { + fmt.Fprintf(os.Stderr, "Filtering for events of type: %s\n", event) + eventWarning = true + } + continue + } + } + + if period == "" { + period = "1" + } + periodInt, err := strconv.Atoi(period) + if err != nil { + fmt.Fprintf(os.Stderr, "Error parsing period: %s\n", err) + continue + } + mPid, mTid, mPeriod = pid, tid, periodInt + + if includeTid { + pname = fmt.Sprintf("%s-%s/%s", comm, mPid, mTid) + } else if includePid { + pname = fmt.Sprintf("%s-%s", comm, mPid) + } else { + pname = comm + } + pname = strings.ReplaceAll(pname, " ", "_") + continue + // Stack line + } else if matches := stackLineRegex.FindStringSubmatch(line); matches != nil { + if pname == "" { + continue + } + pc, rawFunc, mod := matches[1], matches[2], matches[3] + + // skip for now as showInline is always false + // if showInline && !inlineRegex.MatchString(mod) { + // inlineRes := inline(pc, rawFunc, mod) + // if inlineRes != "" && inlineRes != "??" && inlineRes != "??:??:0" { + // // prepend the inline result to the stack + // stack = append([]string{inlineRes}, stack...) + // continue + // } + //} + + // strip symbol offsets from rawFunc + // symbol offsets match this regex: \+0x[\da-f]+$ + rawFunc = stripSymbolsRegex.ReplaceAllString(rawFunc, "") + + // skip process names + if strings.HasPrefix(rawFunc, "(") { + continue + } + // var isUnknown bool + var inline []string + for funcname := range strings.SplitSeq(rawFunc, "->") { + if funcname == "[unknown]" { // use module name instead, if known + if mod != "[unknown]" { + funcname = filepath.Base(mod) + } else { + funcname = "unknown" + // isUnknown = true + } + + if includeAddrs { + funcname = fmt.Sprintf("[%s <%s>]", funcname, pc) + } else { + funcname = fmt.Sprintf("[%s]", funcname) + } + } + if tidyGeneric { + funcname = strings.ReplaceAll(funcname, ";", ":") + if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { + index := stripAnonymousRegex.FindStringIndex(funcname) + funcname = funcname[0:index[0]] + } + funcname = strings.ReplaceAll(funcname, "\"", "") + funcname = strings.ReplaceAll(funcname, "'", "") + } + if tidyJava { + if strings.Contains(funcname, "/") { + // strip the leading L + funcname = strings.TrimPrefix(funcname, "L") + } + } + // annotations + if len(inline) > 0 { + if !strings.Contains(funcname, "_[i]") { + funcname = fmt.Sprintf("%s_[i]", funcname) + } else if annotateKernel && (strings.HasPrefix(funcname, "[") || strings.HasSuffix(funcname, "vmlinux")) && !strings.Contains(mod, "unknown") { + funcname = fmt.Sprintf("%s_[k]", funcname) + } else if annotateJit && jitRegex.MatchString(funcname) { + if !strings.Contains(funcname, "_[j]") { + funcname = fmt.Sprintf("%s_[j]", funcname) + } + } + } + + // source lines + // skip for now since srcLineInInput is always false + // if srcLineInInput && !isUnknown { + // } + + inline = append(inline, funcname) + } + + // prepend inline array to the stack array + if len(inline) > 0 { + stack = append(inline, stack...) + } + + } else { + fmt.Fprintf(os.Stderr, "Unknown line format: %s\n", line) + } + } + + // Output results + keys := make([]string, 0, len(aggregator.collapsed)) + for k := range aggregator.collapsed { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + fmt.Printf("%s %d\n", k, aggregator.collapsed[k]) + } +} From b29f02db5707025854490e35c8934613fee5cc1f Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:54:07 -0700 Subject: [PATCH 07/28] Add go.mod file for stackcollapse-perf module Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tools/stackcollapse-perf/go.mod diff --git a/tools/stackcollapse-perf/go.mod b/tools/stackcollapse-perf/go.mod new file mode 100644 index 00000000..0ec5c58d --- /dev/null +++ b/tools/stackcollapse-perf/go.mod @@ -0,0 +1,3 @@ +module intel.com/stackcolapse-perf + +go 1.24.1 From 14317ba0f53121b921f15e288f1f86aa57b0c97d Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:56:49 -0700 Subject: [PATCH 08/28] spelling Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/stackcollapse-perf/go.mod b/tools/stackcollapse-perf/go.mod index 0ec5c58d..cbed3b20 100644 --- a/tools/stackcollapse-perf/go.mod +++ b/tools/stackcollapse-perf/go.mod @@ -1,3 +1,3 @@ -module intel.com/stackcolapse-perf +module intel.com/stackcollapse-perf go 1.24.1 From 94b66624f63aabf731ba47c498cfce50f2f54948 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 11:17:25 -0700 Subject: [PATCH 09/28] Remove stackcollapse-perf reset command from Makefile Signed-off-by: Harper, Jason M --- tools/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/Makefile b/tools/Makefile index ee9273d2..a39bc266 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -254,7 +254,6 @@ reset: cd msr-tools && git clean -fdx && git reset --hard cd spectre-meltdown-checker cd sshpass && make clean - cd stackcollapse-perf && git clean -fdx && git reset --hard cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc From 4145397de2f8f7901ad364440bb639f4fedb6014 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 13:14:35 -0700 Subject: [PATCH 10/28] use go stackcollapse Signed-off-by: Harper, Jason M --- THIRD_PARTY_PROGRAMS | 23 ----------------------- internal/script/script_defs.go | 4 ++-- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/THIRD_PARTY_PROGRAMS b/THIRD_PARTY_PROGRAMS index a0bf8b81..5138ef76 100644 --- a/THIRD_PARTY_PROGRAMS +++ b/THIRD_PARTY_PROGRAMS @@ -567,29 +567,6 @@ NO WARRANTY END OF TERMS AND CONDITIONS ------------------------------------------------------------- -stackcollapse-perf.pl -# Copyright 2012 Joyent, Inc. All rights reserved. -# Copyright 2012 Brendan Gregg. All rights reserved. -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at docs/cddl1.txt or -# http://opensource.org/licenses/CDDL-1.0. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at docs/cddl1.txt. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -------------------------------------------------------------- stress-ng Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 99e8000c..ce9c078b 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1393,8 +1393,8 @@ fi wait ${perf_fp_pid} ${perf_dwarf_pid} # collapse perf data -perf script -i "$perf_dwarf_data" | stackcollapse-perf.pl > "$perf_dwarf_folded" -perf script -i "$perf_fp_data" | stackcollapse-perf.pl > "$perf_fp_folded" +perf script -i "$perf_dwarf_data" | stackcollapse-perf > "$perf_dwarf_folded" +perf script -i "$perf_fp_data" | stackcollapse-perf > "$perf_fp_folded" # Display results if [ -f "$perf_dwarf_folded" ]; then From 7c72161819ef449b53347aa4dc3da3b172d5e5a4 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Wed, 7 May 2025 16:23:49 -0700 Subject: [PATCH 11/28] fix out of range panic Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/stackcollapse-perf.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/stackcollapse-perf/stackcollapse-perf.go b/tools/stackcollapse-perf/stackcollapse-perf.go index ca63125b..d482e6e8 100644 --- a/tools/stackcollapse-perf/stackcollapse-perf.go +++ b/tools/stackcollapse-perf/stackcollapse-perf.go @@ -214,7 +214,9 @@ func main() { funcname = strings.ReplaceAll(funcname, ";", ":") if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { index := stripAnonymousRegex.FindStringIndex(funcname) - funcname = funcname[0:index[0]] + if index != nil { + funcname = funcname[0:index[0]] + } } funcname = strings.ReplaceAll(funcname, "\"", "") funcname = strings.ReplaceAll(funcname, "'", "") From d41eb46a31b32ebcc762005641cf3aef77dd6493 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Wed, 7 May 2025 16:43:26 -0700 Subject: [PATCH 12/28] add maximum depth of flame graph rendering Signed-off-by: Harper, Jason M --- cmd/flame/flame.go | 13 ++++++ internal/report/render_html_flamegraph.go | 56 +++++++++++++---------- internal/report/table_defs.go | 1 + internal/report/table_helpers.go | 14 ++++++ internal/script/script_defs.go | 8 ++++ 5 files changed, 68 insertions(+), 24 deletions(-) diff --git a/cmd/flame/flame.go b/cmd/flame/flame.go index 3695c1da..d1455b67 100644 --- a/cmd/flame/flame.go +++ b/cmd/flame/flame.go @@ -42,6 +42,7 @@ var ( flagFrequency int flagPid int flagNoSystemSummary bool + flagMaxDepth int ) const ( @@ -49,6 +50,7 @@ const ( flagFrequencyName = "frequency" flagPidName = "pid" flagNoSystemSummaryName = "no-summary" + flagMaxDepthName = "max-depth" ) func init() { @@ -58,6 +60,7 @@ func init() { Cmd.Flags().IntVar(&flagFrequency, flagFrequencyName, 11, "") Cmd.Flags().IntVar(&flagPid, flagPidName, 0, "") Cmd.Flags().BoolVar(&flagNoSystemSummary, flagNoSystemSummaryName, false, "") + Cmd.Flags().IntVar(&flagMaxDepth, flagMaxDepthName, 0, "") common.AddTargetFlags(Cmd) @@ -108,6 +111,10 @@ func getFlagGroups() []common.FlagGroup { Name: common.FlagFormatName, Help: fmt.Sprintf("choose output format(s) from: %s", strings.Join(append([]string{report.FormatAll}, report.FormatHtml, report.FormatTxt, report.FormatJson), ", ")), }, + { + Name: flagMaxDepthName, + Help: "maximum render depth of call stack in flamegraph (0 = no limit)", + }, { Name: flagNoSystemSummaryName, Help: "do not include system summary table in report", @@ -164,6 +171,11 @@ func validateFlags(cmd *cobra.Command, args []string) error { fmt.Fprintf(os.Stderr, "Error: %v\n", err) return err } + if flagMaxDepth < 0 { + err := fmt.Errorf("max depth must be greater than or equal to 0") + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + return err + } // common target flags if err := common.ValidateTargetFlags(cmd); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -185,6 +197,7 @@ func runCmd(cmd *cobra.Command, args []string) error { "Frequency": strconv.Itoa(flagFrequency), "Duration": strconv.Itoa(flagDuration), "PID": strconv.Itoa(flagPid), + "MaxDepth": strconv.Itoa(flagMaxDepth), }, TableNames: tableNames, } diff --git a/internal/report/render_html_flamegraph.go b/internal/report/render_html_flamegraph.go index 02d3b8bf..5e679c89 100644 --- a/internal/report/render_html_flamegraph.go +++ b/internal/report/render_html_flamegraph.go @@ -8,8 +8,9 @@ import ( "bytes" "encoding/json" "fmt" - "log" + "log/slog" "math/rand/v2" // nosemgrep + "slices" "strconv" "strings" texttemplate "text/template" // nosemgrep @@ -81,11 +82,6 @@ type flameGraphTemplateStruct struct { // Folded data conversion adapted from https://github.com/spiermar/burn // Copyright © 2017 Martin Spier // Apache License, Version 2.0 -func reverse(strings []string) { - for i, j := 0, len(strings)-1; i < j; i, j = i+1, j-1 { - strings[i], strings[j] = strings[j], strings[i] - } -} type Node struct { Name string @@ -123,24 +119,24 @@ func (n *Node) MarshalJSON() ([]byte, error) { }) } -var maxStackDepth = 50 - -func convertFoldedToJSON(folded string) (out string, err error) { +func convertFoldedToJSON(folded string, maxStackDepth int) (out string, err error) { rootNode := Node{Name: "root", Value: 0, Children: make(map[string]*Node)} scanner := bufio.NewScanner(strings.NewReader(folded)) for scanner.Scan() { line := scanner.Text() - sep := strings.LastIndex(line, " ") - s := line[:sep] - v := line[sep+1:] - stack := strings.Split(s, ";") - reverse(stack) - if len(stack) > maxStackDepth { - log.Printf("Trimming call stack depth from %d to %d", len(stack), maxStackDepth) - stack = stack[:maxStackDepth] + sep := strings.LastIndex(line, " ") // space separates the call stack from the count + callstack := line[:sep] + count := line[sep+1:] + stack := strings.Split(callstack, ";") // semicolon separates the functions in the call stack + slices.Reverse(stack) + if maxStackDepth > 0 { + if len(stack) > maxStackDepth { + slog.Info("Trimming call stack depth", slog.Int("stack length", len(stack)), slog.Int("max depth", maxStackDepth)) + stack = stack[:maxStackDepth] + } } var i int - i, err = strconv.Atoi(v) + i, err = strconv.Atoi(count) if err != nil { return } @@ -152,9 +148,21 @@ func convertFoldedToJSON(folded string) (out string, err error) { } func renderFlameGraph(header string, tableValues TableValues, field string) (out string) { + maxDepthFieldIndex, err := getFieldIndex("Maximum Render Depth", tableValues) + if err != nil { + slog.Error("didn't find expected field (Maximum Render Depth) in table", slog.String("error", err.Error())) + return + } + maxDepth := tableValues.Fields[maxDepthFieldIndex].Values[0] + maxStackDepth, err := strconv.Atoi(strings.TrimSpace(maxDepth)) + if err != nil { + slog.Error("failed to convert maximum stack depth", slog.String("error", err.Error())) + return + } fieldIdx, err := getFieldIndex(field, tableValues) if err != nil { - log.Panicf("didn't find expected field (%s) in table: %v", field, err) + slog.Error("didn't find expected field in table", slog.String("field", field), slog.String("error", err.Error())) + return } folded := tableValues.Fields[fieldIdx].Values[0] if folded == "" { @@ -166,10 +174,10 @@ func renderFlameGraph(header string, tableValues TableValues, field string) (out out += msg return } - jsonStacks, err := convertFoldedToJSON(folded) + jsonStacks, err := convertFoldedToJSON(folded, maxStackDepth) if err != nil { - log.Printf("failed to convert folded data: %v", err) - out += "Error." + slog.Error("failed to convert folded data", slog.String("error", err.Error())) + out = "" return } fg := texttemplate.Must(texttemplate.New("flameGraphTemplate").Parse(flameGraphTemplate)) @@ -180,8 +188,8 @@ func renderFlameGraph(header string, tableValues TableValues, field string) (out Header: header, }) if err != nil { - log.Printf("failed to render flame graph template: %v", err) - out += "Error." + slog.Error("failed to render flame graph template", slog.String("error", err.Error())) + out = "" return } out += buf.String() diff --git a/internal/report/table_defs.go b/internal/report/table_defs.go index 69fb5dec..ff8fcbba 100644 --- a/internal/report/table_defs.go +++ b/internal/report/table_defs.go @@ -2386,6 +2386,7 @@ func codePathFrequencyTableValues(outputs map[string]script.ScriptOutput) []Fiel fields := []Field{ {Name: "System Paths", Values: []string{systemFoldedFromOutput(outputs)}}, {Name: "Java Paths", Values: []string{javaFoldedFromOutput(outputs)}}, + {Name: "Maximum Render Depth", Values: []string{maxRenderDepthFromOutput(outputs)}}, } return fields } diff --git a/internal/report/table_helpers.go b/internal/report/table_helpers.go index 3f75b111..05e2941b 100644 --- a/internal/report/table_helpers.go +++ b/internal/report/table_helpers.go @@ -2141,3 +2141,17 @@ func systemFoldedFromOutput(outputs map[string]script.ScriptOutput) string { } return folded } + +func maxRenderDepthFromOutput(outputs map[string]script.ScriptOutput) string { + sections := getSectionsFromOutput(outputs[script.ProfileSystemScriptName].Stdout) + if len(sections) == 0 { + slog.Warn("no sections in system profiling output") + return "" + } + for header, content := range sections { + if header == "maximum depth" { + return content + } + } + return "" +} diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index ce9c078b..5209b783 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1244,6 +1244,7 @@ fi pid={{.PID}} duration={{.Duration}} frequency={{.Frequency}} +maxdepth={{.MaxDepth}} ap_interval=0 if [ "$frequency" -ne 0 ]; then @@ -1295,6 +1296,9 @@ for idx in "${!java_pids[@]}"; do async-profiler/profiler.sh stop -o collapsed "$pid" done +echo "########## maximum depth ##########" +echo "$maxdepth" + `, Superuser: true, Depends: []string{"async-profiler"}, @@ -1305,6 +1309,7 @@ done pid={{.PID}} duration={{.Duration}} frequency={{.Frequency}} +maxdepth={{.MaxDepth}} # Function to restore original settings and clean up # This function will be called on exit @@ -1406,6 +1411,9 @@ if [ -f "$perf_fp_folded" ]; then cat "$perf_fp_folded" fi +echo "########## maximum depth ##########" +echo "$maxdepth" + # Clean up temporary files rm -f "$perf_fp_data" "$perf_dwarf_data" "$perf_dwarf_folded" "$perf_fp_folded" `, From 775255ce2e060a4df233f95fa5711cb02149635e Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Thu, 8 May 2025 13:16:55 -0700 Subject: [PATCH 13/28] refactor and add unit tests Signed-off-by: Harper, Jason M --- Makefile | 1 + internal/script/script_defs.go | 35 +- .../stackcollapse-perf/stackcollapse-perf.go | 405 ++++++++++-------- .../stackcollapse-perf_test.go | 168 ++++++++ 4 files changed, 399 insertions(+), 210 deletions(-) create mode 100644 tools/stackcollapse-perf/stackcollapse-perf_test.go diff --git a/Makefile b/Makefile index 3b03c1af..9e6105ff 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,7 @@ endif test: @echo "Running unit tests..." go test -v ./... + cd tools/stackcollapse-perf && go test -v ./... .PHONY: update-deps update-deps: diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 5209b783..7fd39aae 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1316,10 +1316,6 @@ maxdepth={{.MaxDepth}} restore_settings() { echo "$PERF_EVENT_PARANOID" > /proc/sys/kernel/perf_event_paranoid echo "$KPTR_RESTRICT" > /proc/sys/kernel/kptr_restrict - rm -f "$perf_fp_data" - rm -f "$perf_dwarf_data" - rm -f "$perf_dwarf_folded" - rm -f "$perf_fp_folded" if [ -n "$perf_fp_pid" ]; then kill -0 $perf_fp_pid 2>/dev/null && kill -INT $perf_fp_pid fi @@ -1328,12 +1324,6 @@ restore_settings() { fi } -# create temporary output files -perf_fp_data=$(mktemp) -perf_dwarf_data=$(mktemp) -perf_dwarf_folded=$(mktemp) -perf_fp_folded=$(mktemp) - # adjust perf_event_paranoid and kptr_restrict PERF_EVENT_PARANOID=$( cat /proc/sys/kernel/perf_event_paranoid ) echo -1 >/proc/sys/kernel/perf_event_paranoid @@ -1354,10 +1344,10 @@ fi # frame pointer mode # if pid was provided, use it if [ "$pid" -ne 0 ]; then - perf record -F "$frequency" -p "$pid" -g -o "$perf_fp_data" -m 129 & + perf record -F "$frequency" -p "$pid" -g -o perf_fp_data -m 129 & else # if no pid was provided, use system-wide profiling - perf record -F "$frequency" -a -g -o "$perf_fp_data" -m 129 & + perf record -F "$frequency" -a -g -o perf_fp_data -m 129 & fi perf_fp_pid=$! if ! kill -0 $perf_fp_pid 2>/dev/null; then @@ -1368,10 +1358,10 @@ fi # dwarf mode # if pid was provided, use it if [ "$pid" -ne 0 ]; then - perf record -F "$frequency" -p "$pid" -g -o "$perf_dwarf_data" -m 257 --call-graph dwarf,8192 & + perf record -F "$frequency" -p "$pid" -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & else # if no pid was provided, use system-wide profiling - perf record -F "$frequency" -a -g -o "$perf_dwarf_data" -m 257 --call-graph dwarf,8192 & + perf record -F "$frequency" -a -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & fi perf_dwarf_pid=$! if ! kill -0 $perf_dwarf_pid 2>/dev/null; then @@ -1398,24 +1388,23 @@ fi wait ${perf_fp_pid} ${perf_dwarf_pid} # collapse perf data -perf script -i "$perf_dwarf_data" | stackcollapse-perf > "$perf_dwarf_folded" -perf script -i "$perf_fp_data" | stackcollapse-perf > "$perf_fp_folded" +perf script -i perf_dwarf_data > perf_dwarf_stacks +stackcollapse-perf perf_dwarf_stacks > perf_dwarf_folded +perf script -i perf_fp_data > perf_fp_stacks +stackcollapse-perf perf_fp_stacks > perf_fp_folded # Display results -if [ -f "$perf_dwarf_folded" ]; then +if [ -f perf_dwarf_folded ]; then echo "########## perf_dwarf ##########" - cat "$perf_dwarf_folded" + cat perf_dwarf_folded fi -if [ -f "$perf_fp_folded" ]; then +if [ -f perf_fp_folded ]; then echo "########## perf_fp ##########" - cat "$perf_fp_folded" + cat perf_fp_folded fi echo "########## maximum depth ##########" echo "$maxdepth" - -# Clean up temporary files -rm -f "$perf_fp_data" "$perf_dwarf_data" "$perf_dwarf_folded" "$perf_fp_folded" `, Superuser: true, Depends: []string{"perf", "stackcollapse-perf"}, diff --git a/tools/stackcollapse-perf/stackcollapse-perf.go b/tools/stackcollapse-perf/stackcollapse-perf.go index d482e6e8..e0ef9b0b 100644 --- a/tools/stackcollapse-perf/stackcollapse-perf.go +++ b/tools/stackcollapse-perf/stackcollapse-perf.go @@ -10,6 +10,7 @@ package main import ( "bufio" "fmt" + "io" "os" "path/filepath" "regexp" @@ -18,42 +19,41 @@ import ( "strings" ) -// In the original Perl code, the following -// are command line arguments. For our use, we don't need to set them as flags, -// but we can keep them for compatibility with the original -var annotateKernel = false -var annotateJit = false - -// var annotateAll = false -var includePname = true -var includePid = false -var includeTid = false -var includeAddrs = false -var tidyJava = true -var tidyGeneric = true - -// var targetPname = "" -var eventFilter = "" - -// var showInline = false -// var showContext = false -// var srcLineInInput = false +// Config holds configuration options for processing stacks. +// It includes options for annotating kernel and JIT symbols, including process names, PIDs, TIDs, and addresses, +// as well as options for tidying Java and generic function names, filtering events, and showing inline or context information. +type Config struct { + AnnotateKernel bool + AnnotateJit bool + IncludePname bool + IncludePid bool + IncludeTid bool + IncludeAddrs bool + TidyJava bool + TidyGeneric bool + EventFilter string + ShowInline bool + ShowContext bool + SrcLineInInput bool +} +// StackAggregator aggregates stack traces and their counts. +// It provides a method to remember stacks and their associated counts. type StackAggregator struct { collapsed map[string]int } +// NewStackAggregator creates and returns a new StackAggregator instance. func NewStackAggregator() *StackAggregator { return &StackAggregator{collapsed: make(map[string]int)} } +// RememberStack adds a stack trace and its count to the aggregator. func (sa *StackAggregator) RememberStack(stack string, count int) { sa.collapsed[stack] += count } func main() { - aggregator := NewStackAggregator() - var input *os.File var err error @@ -68,204 +68,235 @@ func main() { } else { input = os.Stdin // Default to standard input } + + var config = Config{ + AnnotateKernel: false, + AnnotateJit: false, + IncludePname: true, + IncludePid: false, + IncludeTid: false, + IncludeAddrs: false, + TidyJava: true, + TidyGeneric: true, + EventFilter: "", + ShowInline: false, + ShowContext: false, + SrcLineInInput: false, + } + + err = ProcessStacks(input, os.Stdout, config) + if err != nil { + fmt.Fprintf(os.Stderr, "Error processing stacks: %s\n", err) + os.Exit(1) + } +} + +// Regular expressions for parsing the perf output +var ( + eventLineRegex = regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) + eventTypeRegex = regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) + stackLineRegex = regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) + // inlineRegex = regexp.MustCompile(`(perf-\d+.map|kernel\.|\[[^\]]+\])`) + stripSymbolsRegex = regexp.MustCompile(`\+0x[\da-f]+$`) + stripIdRegex = regexp.MustCompile(`\.\(.*\)\.`) + stripAnonymousRegex = regexp.MustCompile(`\([^a]*anonymous namespace[^)]*\)`) + jitRegex = regexp.MustCompile(`/tmp/perf-\d+\.map`) +) + +// ProcessStacks processes stack traces from the input reader and writes the collapsed stacks to the output writer. +// It uses the provided configuration to control the processing behavior. +func ProcessStacks(input io.Reader, output io.Writer, config Config) error { + aggregator := NewStackAggregator() scanner := bufio.NewScanner(input) var stack []string - var pname string - var mPeriod int - var mTid string - var mPid string - - eventRegex := regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) - eventTypeRegex := regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) - stackLineRegex := regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) - // inlineRegex := regexp.MustCompile(`(perf-\d+.map|kernel\.|\[[^\]]+\])`) - stripSymbolsRegex := regexp.MustCompile(`\+0x[\da-f]+$`) - stripIdRegex := regexp.MustCompile(`\.\(.*\)\.`) - stripAnonymousRegex := regexp.MustCompile(`\([^a]*anonymous namespace[^)]*\)`) - jitRegex := regexp.MustCompile(`/tmp/perf-\d+\.map`) - - var eventDefaulted bool - var eventWarning bool + var processName string + var period int // main loop, read lines from stdin for scanner.Scan() { line := scanner.Text() - if strings.HasPrefix(line, "# cmdline") { - // loop through the command line arguments in reverse order - for i := len(os.Args) - 1; i > 0; i-- { - if !strings.HasPrefix(os.Args[i], "-") { - // not used -> target_pname = filepath.Base(os.Args[i]) - break - } - } + if strings.HasPrefix(line, "#") { + continue } - // Skip remaining comments - if strings.HasPrefix(line, "#") { + if line == "" && processName == "" { continue } - // End of stack - if line == "" { - if pname == "" { - continue + if line == "" { // check for End of stack + if config.IncludePname { + stack = append([]string{processName}, stack...) } - if includePname { - // prepend the process name to the stack - stack = append([]string{pname}, stack...) - } - if stack != nil { - aggregator.RememberStack(strings.Join(stack, ";"), mPeriod) + aggregator.RememberStack(strings.Join(stack, ";"), period) } stack = nil - pname = "" + processName = "" continue } + if err := handleEventRecord(line, &processName, &period, config); err != nil { + fmt.Fprintf(output, "Error: %s\n", err) + continue + } else if err := handleStackLine(line, &stack, processName, config); err != nil { + fmt.Fprintf(output, "Error: %s\n", err) + continue + } + } - // Event record start - if matches := eventRegex.FindStringSubmatch(line); matches != nil { - comm, pid, tid, period := matches[1], matches[2], matches[3], "" - if tid == "" { - tid = pid - pid = "?" - } + // Check for errors during scanning + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Error reading input: %s\n", err) + return err + } - if eventMatches := eventTypeRegex.FindStringSubmatch(line); eventMatches != nil { - period = eventMatches[1] - event := eventMatches[2] - - if eventFilter == "" { - eventFilter = event - eventDefaulted = true - } else if event != eventFilter { - if eventDefaulted && !eventWarning { - fmt.Fprintf(os.Stderr, "Filtering for events of type: %s\n", event) - eventWarning = true - } - continue - } - } + // Output results + keys := make([]string, 0, len(aggregator.collapsed)) + for k := range aggregator.collapsed { + keys = append(keys, k) + } + sort.Strings(keys) - if period == "" { - period = "1" - } - periodInt, err := strconv.Atoi(period) + for _, k := range keys { + fmt.Fprintf(output, "%s %d\n", k, aggregator.collapsed[k]) + } + + return nil +} + +// handleEventRecord parses an event record line and updates the process name and period based on the configuration. +func handleEventRecord(line string, processName *string, period *int, config Config) error { + matches := eventLineRegex.FindStringSubmatch(line) + if matches == nil { + return nil + } + + comm, pid, tid := matches[1], matches[2], matches[3] + if tid == "" { + tid = pid + pid = "?" + } + + if eventMatches := eventTypeRegex.FindStringSubmatch(line); eventMatches != nil { + eventPeriod := eventMatches[1] + if eventPeriod == "" { + *period = 1 + } else { + eventPeriodInt, err := strconv.Atoi(eventPeriod) if err != nil { - fmt.Fprintf(os.Stderr, "Error parsing period: %s\n", err) - continue + return fmt.Errorf("failed to parse event period: %s, error: %v", eventPeriod, err) } - mPid, mTid, mPeriod = pid, tid, periodInt + *period = eventPeriodInt + } + event := eventMatches[2] + + if config.EventFilter == "" { + config.EventFilter = event + } else if event != config.EventFilter { + return fmt.Errorf("event type mismatch: %s != %s", event, config.EventFilter) + } + } + + if config.IncludeTid { + *processName = fmt.Sprintf("%s-%s/%s", comm, pid, tid) + } else if config.IncludePid { + *processName = fmt.Sprintf("%s-%s", comm, pid) + } else { + *processName = comm + } + *processName = strings.ReplaceAll(*processName, " ", "_") + return nil +} + +// handleStackLine parses a stack line and appends the function name to the stack based on the configuration. +func handleStackLine(line string, stack *[]string, pname string, config Config) error { + matches := stackLineRegex.FindStringSubmatch(line) + if matches == nil || pname == "" { + return nil + } - if includeTid { - pname = fmt.Sprintf("%s-%s/%s", comm, mPid, mTid) - } else if includePid { - pname = fmt.Sprintf("%s-%s", comm, mPid) + pc, rawFunc, mod := matches[1], matches[2], matches[3] + + // skip for now as showInline is always false + // if showInline && !inlineRegex.MatchString(mod) { + // inlineRes := inline(pc, rawFunc, mod) + // if inlineRes != "" && inlineRes != "??" && inlineRes != "??:??:0" { + // // prepend the inline result to the stack + // stack = append([]string{inlineRes}, stack...) + // continue + // } + //} + + // strip symbol offsets from rawFunc + // symbol offsets match this regex: \+0x[\da-f]+$ + rawFunc = stripSymbolsRegex.ReplaceAllString(rawFunc, "") + + // skip process names + if strings.HasPrefix(rawFunc, "(") { + return nil + } + + *stack = append(processFunctionName(rawFunc, mod, pc, config), *stack...) + return nil +} + +// processFunctionName processes a raw function name, module, and program counter (PC) based on the configuration. +// It returns a slice of processed function names. +func processFunctionName(rawFunc, mod, pc string, config Config) []string { + // var isUnknown bool + var inline []string + for funcname := range strings.SplitSeq(rawFunc, "->") { + if funcname == "[unknown]" { // use module name instead, if known + if mod != "[unknown]" { + funcname = filepath.Base(mod) } else { - pname = comm - } - pname = strings.ReplaceAll(pname, " ", "_") - continue - // Stack line - } else if matches := stackLineRegex.FindStringSubmatch(line); matches != nil { - if pname == "" { - continue + funcname = "unknown" + // isUnknown = true } - pc, rawFunc, mod := matches[1], matches[2], matches[3] - - // skip for now as showInline is always false - // if showInline && !inlineRegex.MatchString(mod) { - // inlineRes := inline(pc, rawFunc, mod) - // if inlineRes != "" && inlineRes != "??" && inlineRes != "??:??:0" { - // // prepend the inline result to the stack - // stack = append([]string{inlineRes}, stack...) - // continue - // } - //} - - // strip symbol offsets from rawFunc - // symbol offsets match this regex: \+0x[\da-f]+$ - rawFunc = stripSymbolsRegex.ReplaceAllString(rawFunc, "") - - // skip process names - if strings.HasPrefix(rawFunc, "(") { - continue + + if config.IncludeAddrs { + funcname = fmt.Sprintf("[%s <%s>]", funcname, pc) + } else { + funcname = fmt.Sprintf("[%s]", funcname) } - // var isUnknown bool - var inline []string - for funcname := range strings.SplitSeq(rawFunc, "->") { - if funcname == "[unknown]" { // use module name instead, if known - if mod != "[unknown]" { - funcname = filepath.Base(mod) - } else { - funcname = "unknown" - // isUnknown = true - } - - if includeAddrs { - funcname = fmt.Sprintf("[%s <%s>]", funcname, pc) - } else { - funcname = fmt.Sprintf("[%s]", funcname) - } - } - if tidyGeneric { - funcname = strings.ReplaceAll(funcname, ";", ":") - if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { - index := stripAnonymousRegex.FindStringIndex(funcname) - if index != nil { - funcname = funcname[0:index[0]] - } - } - funcname = strings.ReplaceAll(funcname, "\"", "") - funcname = strings.ReplaceAll(funcname, "'", "") - } - if tidyJava { - if strings.Contains(funcname, "/") { - // strip the leading L - funcname = strings.TrimPrefix(funcname, "L") - } - } - // annotations - if len(inline) > 0 { - if !strings.Contains(funcname, "_[i]") { - funcname = fmt.Sprintf("%s_[i]", funcname) - } else if annotateKernel && (strings.HasPrefix(funcname, "[") || strings.HasSuffix(funcname, "vmlinux")) && !strings.Contains(mod, "unknown") { - funcname = fmt.Sprintf("%s_[k]", funcname) - } else if annotateJit && jitRegex.MatchString(funcname) { - if !strings.Contains(funcname, "_[j]") { - funcname = fmt.Sprintf("%s_[j]", funcname) - } - } + } + if config.TidyGeneric { + funcname = strings.ReplaceAll(funcname, ";", ":") + if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { + index := stripAnonymousRegex.FindStringIndex(funcname) + if index != nil { + funcname = funcname[0:index[0]] } - - // source lines - // skip for now since srcLineInInput is always false - // if srcLineInInput && !isUnknown { - // } - - inline = append(inline, funcname) } - - // prepend inline array to the stack array - if len(inline) > 0 { - stack = append(inline, stack...) + funcname = strings.ReplaceAll(funcname, "\"", "") + funcname = strings.ReplaceAll(funcname, "'", "") + } + if config.TidyJava { + if strings.Contains(funcname, "/") { + // strip the leading L + funcname = strings.TrimPrefix(funcname, "L") + } + } + // annotations + if len(inline) > 0 { + if !strings.Contains(funcname, "_[i]") { + funcname = fmt.Sprintf("%s_[i]", funcname) + } else if config.AnnotateKernel && (strings.HasPrefix(funcname, "[") || strings.HasSuffix(funcname, "vmlinux")) && !strings.Contains(mod, "unknown") { + funcname = fmt.Sprintf("%s_[k]", funcname) + } else if config.AnnotateJit && jitRegex.MatchString(funcname) { + if !strings.Contains(funcname, "_[j]") { + funcname = fmt.Sprintf("%s_[j]", funcname) + } } - - } else { - fmt.Fprintf(os.Stderr, "Unknown line format: %s\n", line) } - } - // Output results - keys := make([]string, 0, len(aggregator.collapsed)) - for k := range aggregator.collapsed { - keys = append(keys, k) - } - sort.Strings(keys) + // source lines + // skip for now since srcLineInInput is always false + // if srcLineInInput && !isUnknown { + // } - for _, k := range keys { - fmt.Printf("%s %d\n", k, aggregator.collapsed[k]) + inline = append(inline, funcname) } + return inline } diff --git a/tools/stackcollapse-perf/stackcollapse-perf_test.go b/tools/stackcollapse-perf/stackcollapse-perf_test.go new file mode 100644 index 00000000..e77bc723 --- /dev/null +++ b/tools/stackcollapse-perf/stackcollapse-perf_test.go @@ -0,0 +1,168 @@ +package main + +// Copyright (C) 2021-2025 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "bytes" + "strings" + "testing" +) + +// keep for debugging +// func TestProcessStacksFromFile(t *testing.T) { +// filePath := filepath.Join("testdata", "perf_fp_stacks") +// file, err := os.Open(filePath) +// if err != nil { +// t.Fatalf("failed to open test file: %v", err) +// } +// defer file.Close() + +// output := &bytes.Buffer{} +// config := Config{ +// IncludePname: true, +// IncludePid: false, +// IncludeTid: false, +// IncludeAddrs: false, +// TidyJava: true, +// TidyGeneric: true, +// } + +// err = ProcessStacks(file, output, config) +// if err != nil { +// t.Fatalf("unexpected error: %v", err) +// } + +// if output.Len() == 0 { +// t.Errorf("expected output, got empty result") +// } +// } + +func TestProcessStacks(t *testing.T) { + input := strings.NewReader(` +stress-ng-cpu 1230556 [121] 6223127.073349: 293637623 cycles:P: + 61e248df6091 [unknown] (/usr/bin/stress-ng) + +stress-ng-cpu 1230793 [098] 6223127.074783: 307465331 cycles:P: + ffffffffa7c00f0b asm_sysvec_apic_timer_interrupt+0x1b ([kernel.kallsyms]) + 760c9702dc5d [unknown] (/usr/lib/x86_64-linux-gnu/libm.so.6) + 760c96fda3a2 sincosf64x+0x122 (/usr/lib/x86_64-linux-gnu/libm.so.6) + + `) + output := &bytes.Buffer{} + + config := Config{ + IncludePname: true, + IncludePid: false, + IncludeTid: false, + IncludeAddrs: false, + TidyJava: true, + TidyGeneric: true, + } + + err := ProcessStacks(input, output, config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expected := "stress-ng-cpu;[stress-ng] 293637623\nstress-ng-cpu;sincosf64x;[libm.so.6];asm_sysvec_apic_timer_interrupt 307465331\n" + if output.String() != expected { + t.Errorf("expected %q, got %q", expected, output.String()) + } +} + +func TestHandleEventRecord(t *testing.T) { + line := "stress-ng-cpu 1230556 [121] 6223127.073349: 293637623 cycles:P: " + var processName string + var period int + config := Config{ + IncludePname: true, + IncludePid: false, + IncludeTid: false, + IncludeAddrs: false, + TidyJava: true, + TidyGeneric: true, + } + + err := handleEventRecord(line, &processName, &period, config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expectedProcessName := "stress-ng-cpu" + if processName != "stress-ng-cpu" { + t.Errorf("expected processName to be '%s', got %q", expectedProcessName, processName) + } + expectedPeriod := 293637623 + if period != expectedPeriod { + t.Errorf("expected period to be %d, got %d", expectedPeriod, period) + } +} + +func TestHandleStackLine(t *testing.T) { + line := "0x1234 someFunction (module)" + var stack []string + processName := "main" + config := Config{ + IncludePname: true, + IncludePid: false, + IncludeTid: false, + IncludeAddrs: false, + TidyJava: true, + TidyGeneric: true, + } + + err := handleStackLine(line, &stack, processName, config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(stack) != 1 || stack[0] != "someFunction" { + t.Errorf("expected stack to contain 'someFunction', got %v", stack) + } +} + +func TestProcessFunctionName(t *testing.T) { + tests := []struct { + rawFunc string + mod string + pc string + config Config + expected []string + }{ + { + rawFunc: "[unknown]", + mod: "[kernel]", + pc: "0x1234", + config: Config{IncludeAddrs: true}, + expected: []string{"[[kernel] <0x1234>]"}, + }, + { + rawFunc: "Lcom/example/MyClass", + mod: "[unknown]", + pc: "0x5678", + config: Config{TidyJava: true}, + expected: []string{"com/example/MyClass"}, + }, + { + rawFunc: "someFunction;bar\"hello'world.(anonymous namespace).foo", + mod: "module.so", + pc: "0x9abc", + config: Config{TidyGeneric: true}, + expected: []string{"someFunction:barhelloworld."}, + }, + } + + for _, test := range tests { + result := processFunctionName(test.rawFunc, test.mod, test.pc, test.config) + if len(result) != len(test.expected) { + t.Errorf("expected %d results, got %d", len(test.expected), len(result)) + continue + } + for i := range result { + if result[i] != test.expected[i] { + t.Errorf("expected %q, got %q", test.expected[i], result[i]) + } + } + } +} From ff19d06c7318fdff9b56b4b730072041153d750c Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:05:01 -0700 Subject: [PATCH 14/28] Use Go implementation of stackcollapse-perf to eliminate Perl dependency. Signed-off-by: Harper, Jason M --- internal/script/script_defs.go | 2 +- tools/Makefile | 21 +- .../stackcollapse-perf/stackcollapse-perf.go | 269 ++++++++++++++++++ 3 files changed, 278 insertions(+), 14 deletions(-) create mode 100644 tools/stackcollapse-perf/stackcollapse-perf.go diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 89c6e111..99e8000c 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1410,7 +1410,7 @@ fi rm -f "$perf_fp_data" "$perf_dwarf_data" "$perf_dwarf_folded" "$perf_fp_folded" `, Superuser: true, - Depends: []string{"perf", "stackcollapse-perf.pl"}, + Depends: []string{"perf", "stackcollapse-perf"}, }, // lock analysis scripts ProfileKernelLockScriptName: { diff --git a/tools/Makefile b/tools/Makefile index afb59d72..ee9273d2 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -5,9 +5,9 @@ # default: tools -.PHONY: default tools async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool lshw lspci msr-tools pcm perf processwatch spectre-meltdown-checker sshpass stress-ng sysstat tsc turbostat +.PHONY: default tools async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm perf processwatch spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat -tools: async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stress-ng sysstat tsc turbostat +tools: async-profiler avx-turbo cpuid dmidecode ethtool fio ipmitool lshw lspci msr-tools pcm spectre-meltdown-checker sshpass stackcollapse-perf stress-ng sysstat tsc turbostat mkdir -p bin cp -R async-profiler bin/ cp avx-turbo/avx-turbo bin/ @@ -15,7 +15,7 @@ tools: async-profiler avx-turbo cpuid dmidecode ethtool fio flamegraph ipmitool cp dmidecode/dmidecode bin/ cp ethtool/ethtool bin/ cp fio/fio bin/ - cp flamegraph/stackcollapse-perf.pl bin/ + cp stackcollapse-perf/stackcollapse-perf bin/ cp ipmitool/src/ipmitool.static bin/ipmitool cp lshw/src/lshw-static bin/lshw cp lspci/lspci bin/ @@ -103,14 +103,6 @@ ifeq ("$(wildcard fio/config.log)","") endif cd fio && make -FLAMEGRAPH_VERSION := "master" -flamegraph: -ifeq ("$(wildcard flamegraph)","") - git clone https://github.com/brendangregg/FlameGraph.git flamegraph - # small modification to script to include module name in output - cd flamegraph && sed -i '382 a \\t\t\t\t$$func = \$$func."'" "'".\$$mod;\t# add module name' stackcollapse-perf.pl -endif - IPMITOOL_VERSION := "IPMITOOL_1_8_19" ipmitool: ifeq ("$(wildcard ipmitool)","") @@ -213,6 +205,9 @@ ifeq ("$(wildcard sshpass)","") endif cd sshpass && make +stackcollapse-perf: + cd stackcollapse-perf && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build + STRESS_NG_VERSION := "V0.13.08" stress-ng: ifeq ("$(wildcard stress-ng)","") @@ -252,7 +247,6 @@ reset: cd dmidecode && git clean -fdx && git reset --hard cd ethtool && git clean -fdx && git reset --hard cd fio && git clean -fdx && git reset --hard - cd flamegraph && git clean -fdx && git reset --hard cd ipmitool && git clean -fdx && git reset --hard cd lshw && git clean -fdx && git reset --hard cd lspci && git clean -fdx && git reset --hard @@ -260,6 +254,7 @@ reset: cd msr-tools && git clean -fdx && git reset --hard cd spectre-meltdown-checker cd sshpass && make clean + cd stackcollapse-perf && git clean -fdx && git reset --hard cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc @@ -275,5 +270,5 @@ libcrypt.tar.gz: libs: glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz oss-source: reset libs - tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ flamegraph/ ipmitool/ lshw/ lspci/ msr-tools/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz + tar --exclude-vcs -czf oss_source.tgz async-profiler/ cpuid/ dmidecode/ ethtool/ fio/ ipmitool/ lshw/ lspci/ msr-tools/ pcm/ spectre-meltdown-checker/ sshpass/ stress-ng/ sysstat/ linux_turbostat/tools/power/x86/turbostat glibc-2.19.tar.bz2 zlib.tar.gz libcrypt.tar.gz md5sum oss_source.tgz > oss_source.tgz.md5 diff --git a/tools/stackcollapse-perf/stackcollapse-perf.go b/tools/stackcollapse-perf/stackcollapse-perf.go new file mode 100644 index 00000000..ca63125b --- /dev/null +++ b/tools/stackcollapse-perf/stackcollapse-perf.go @@ -0,0 +1,269 @@ +package main + +// Copyright (C) 2021-2025 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +// This code is a port of the Perl script stackcollapse-perf.pl from Brendan +// Gregg's Flamegraph project -- github.com/brendangregg/FlameGraph. +// All credit to Brendan Gregg for the original implementation and the flamegraph concept. + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" +) + +// In the original Perl code, the following +// are command line arguments. For our use, we don't need to set them as flags, +// but we can keep them for compatibility with the original +var annotateKernel = false +var annotateJit = false + +// var annotateAll = false +var includePname = true +var includePid = false +var includeTid = false +var includeAddrs = false +var tidyJava = true +var tidyGeneric = true + +// var targetPname = "" +var eventFilter = "" + +// var showInline = false +// var showContext = false +// var srcLineInInput = false + +type StackAggregator struct { + collapsed map[string]int +} + +func NewStackAggregator() *StackAggregator { + return &StackAggregator{collapsed: make(map[string]int)} +} + +func (sa *StackAggregator) RememberStack(stack string, count int) { + sa.collapsed[stack] += count +} + +func main() { + aggregator := NewStackAggregator() + + var input *os.File + var err error + + // Check if a file path is provided as a command-line argument + if len(os.Args) > 1 { + input, err = os.Open(os.Args[1]) // Open the file + if err != nil { + fmt.Fprintf(os.Stderr, "Error opening file: %s\n", err) + os.Exit(1) + } + defer input.Close() + } else { + input = os.Stdin // Default to standard input + } + scanner := bufio.NewScanner(input) + + var stack []string + var pname string + var mPeriod int + var mTid string + var mPid string + + eventRegex := regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) + eventTypeRegex := regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) + stackLineRegex := regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) + // inlineRegex := regexp.MustCompile(`(perf-\d+.map|kernel\.|\[[^\]]+\])`) + stripSymbolsRegex := regexp.MustCompile(`\+0x[\da-f]+$`) + stripIdRegex := regexp.MustCompile(`\.\(.*\)\.`) + stripAnonymousRegex := regexp.MustCompile(`\([^a]*anonymous namespace[^)]*\)`) + jitRegex := regexp.MustCompile(`/tmp/perf-\d+\.map`) + + var eventDefaulted bool + var eventWarning bool + + // main loop, read lines from stdin + for scanner.Scan() { + line := scanner.Text() + + if strings.HasPrefix(line, "# cmdline") { + // loop through the command line arguments in reverse order + for i := len(os.Args) - 1; i > 0; i-- { + if !strings.HasPrefix(os.Args[i], "-") { + // not used -> target_pname = filepath.Base(os.Args[i]) + break + } + } + } + + // Skip remaining comments + if strings.HasPrefix(line, "#") { + continue + } + + // End of stack + if line == "" { + if pname == "" { + continue + } + if includePname { + // prepend the process name to the stack + stack = append([]string{pname}, stack...) + } + + if stack != nil { + aggregator.RememberStack(strings.Join(stack, ";"), mPeriod) + } + stack = nil + pname = "" + continue + } + + // Event record start + if matches := eventRegex.FindStringSubmatch(line); matches != nil { + comm, pid, tid, period := matches[1], matches[2], matches[3], "" + if tid == "" { + tid = pid + pid = "?" + } + + if eventMatches := eventTypeRegex.FindStringSubmatch(line); eventMatches != nil { + period = eventMatches[1] + event := eventMatches[2] + + if eventFilter == "" { + eventFilter = event + eventDefaulted = true + } else if event != eventFilter { + if eventDefaulted && !eventWarning { + fmt.Fprintf(os.Stderr, "Filtering for events of type: %s\n", event) + eventWarning = true + } + continue + } + } + + if period == "" { + period = "1" + } + periodInt, err := strconv.Atoi(period) + if err != nil { + fmt.Fprintf(os.Stderr, "Error parsing period: %s\n", err) + continue + } + mPid, mTid, mPeriod = pid, tid, periodInt + + if includeTid { + pname = fmt.Sprintf("%s-%s/%s", comm, mPid, mTid) + } else if includePid { + pname = fmt.Sprintf("%s-%s", comm, mPid) + } else { + pname = comm + } + pname = strings.ReplaceAll(pname, " ", "_") + continue + // Stack line + } else if matches := stackLineRegex.FindStringSubmatch(line); matches != nil { + if pname == "" { + continue + } + pc, rawFunc, mod := matches[1], matches[2], matches[3] + + // skip for now as showInline is always false + // if showInline && !inlineRegex.MatchString(mod) { + // inlineRes := inline(pc, rawFunc, mod) + // if inlineRes != "" && inlineRes != "??" && inlineRes != "??:??:0" { + // // prepend the inline result to the stack + // stack = append([]string{inlineRes}, stack...) + // continue + // } + //} + + // strip symbol offsets from rawFunc + // symbol offsets match this regex: \+0x[\da-f]+$ + rawFunc = stripSymbolsRegex.ReplaceAllString(rawFunc, "") + + // skip process names + if strings.HasPrefix(rawFunc, "(") { + continue + } + // var isUnknown bool + var inline []string + for funcname := range strings.SplitSeq(rawFunc, "->") { + if funcname == "[unknown]" { // use module name instead, if known + if mod != "[unknown]" { + funcname = filepath.Base(mod) + } else { + funcname = "unknown" + // isUnknown = true + } + + if includeAddrs { + funcname = fmt.Sprintf("[%s <%s>]", funcname, pc) + } else { + funcname = fmt.Sprintf("[%s]", funcname) + } + } + if tidyGeneric { + funcname = strings.ReplaceAll(funcname, ";", ":") + if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { + index := stripAnonymousRegex.FindStringIndex(funcname) + funcname = funcname[0:index[0]] + } + funcname = strings.ReplaceAll(funcname, "\"", "") + funcname = strings.ReplaceAll(funcname, "'", "") + } + if tidyJava { + if strings.Contains(funcname, "/") { + // strip the leading L + funcname = strings.TrimPrefix(funcname, "L") + } + } + // annotations + if len(inline) > 0 { + if !strings.Contains(funcname, "_[i]") { + funcname = fmt.Sprintf("%s_[i]", funcname) + } else if annotateKernel && (strings.HasPrefix(funcname, "[") || strings.HasSuffix(funcname, "vmlinux")) && !strings.Contains(mod, "unknown") { + funcname = fmt.Sprintf("%s_[k]", funcname) + } else if annotateJit && jitRegex.MatchString(funcname) { + if !strings.Contains(funcname, "_[j]") { + funcname = fmt.Sprintf("%s_[j]", funcname) + } + } + } + + // source lines + // skip for now since srcLineInInput is always false + // if srcLineInInput && !isUnknown { + // } + + inline = append(inline, funcname) + } + + // prepend inline array to the stack array + if len(inline) > 0 { + stack = append(inline, stack...) + } + + } else { + fmt.Fprintf(os.Stderr, "Unknown line format: %s\n", line) + } + } + + // Output results + keys := make([]string, 0, len(aggregator.collapsed)) + for k := range aggregator.collapsed { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, k := range keys { + fmt.Printf("%s %d\n", k, aggregator.collapsed[k]) + } +} From 811d8a6399d83ae56c8c154f32d07cd7cb8f323b Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:54:07 -0700 Subject: [PATCH 15/28] Add go.mod file for stackcollapse-perf module Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/go.mod | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 tools/stackcollapse-perf/go.mod diff --git a/tools/stackcollapse-perf/go.mod b/tools/stackcollapse-perf/go.mod new file mode 100644 index 00000000..0ec5c58d --- /dev/null +++ b/tools/stackcollapse-perf/go.mod @@ -0,0 +1,3 @@ +module intel.com/stackcolapse-perf + +go 1.24.1 From 556d58a8d74e89ffddf11b941e5c7eb8bdd8ce62 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:56:49 -0700 Subject: [PATCH 16/28] spelling Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/stackcollapse-perf/go.mod b/tools/stackcollapse-perf/go.mod index 0ec5c58d..cbed3b20 100644 --- a/tools/stackcollapse-perf/go.mod +++ b/tools/stackcollapse-perf/go.mod @@ -1,3 +1,3 @@ -module intel.com/stackcolapse-perf +module intel.com/stackcollapse-perf go 1.24.1 From d788587e5ab4d41861866eadd06dccc362013966 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 11:17:25 -0700 Subject: [PATCH 17/28] Remove stackcollapse-perf reset command from Makefile Signed-off-by: Harper, Jason M --- tools/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/Makefile b/tools/Makefile index ee9273d2..a39bc266 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -254,7 +254,6 @@ reset: cd msr-tools && git clean -fdx && git reset --hard cd spectre-meltdown-checker cd sshpass && make clean - cd stackcollapse-perf && git clean -fdx && git reset --hard cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc From c6c2797579d20c6327208e4ba24d4933b7d93a20 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 13:14:35 -0700 Subject: [PATCH 18/28] use go stackcollapse Signed-off-by: Harper, Jason M --- THIRD_PARTY_PROGRAMS | 23 ----------------------- internal/script/script_defs.go | 4 ++-- 2 files changed, 2 insertions(+), 25 deletions(-) diff --git a/THIRD_PARTY_PROGRAMS b/THIRD_PARTY_PROGRAMS index a0bf8b81..5138ef76 100644 --- a/THIRD_PARTY_PROGRAMS +++ b/THIRD_PARTY_PROGRAMS @@ -567,29 +567,6 @@ NO WARRANTY END OF TERMS AND CONDITIONS ------------------------------------------------------------- -stackcollapse-perf.pl -# Copyright 2012 Joyent, Inc. All rights reserved. -# Copyright 2012 Brendan Gregg. All rights reserved. -# -# CDDL HEADER START -# -# The contents of this file are subject to the terms of the -# Common Development and Distribution License (the "License"). -# You may not use this file except in compliance with the License. -# -# You can obtain a copy of the license at docs/cddl1.txt or -# http://opensource.org/licenses/CDDL-1.0. -# See the License for the specific language governing permissions -# and limitations under the License. -# -# When distributing Covered Code, include this CDDL HEADER in each -# file and include the License file at docs/cddl1.txt. -# If applicable, add the following below this CDDL HEADER, with the -# fields enclosed by brackets "[]" replaced with your own identifying -# information: Portions Copyright [yyyy] [name of copyright owner] -# -# CDDL HEADER END -------------------------------------------------------------- stress-ng Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 99e8000c..ce9c078b 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1393,8 +1393,8 @@ fi wait ${perf_fp_pid} ${perf_dwarf_pid} # collapse perf data -perf script -i "$perf_dwarf_data" | stackcollapse-perf.pl > "$perf_dwarf_folded" -perf script -i "$perf_fp_data" | stackcollapse-perf.pl > "$perf_fp_folded" +perf script -i "$perf_dwarf_data" | stackcollapse-perf > "$perf_dwarf_folded" +perf script -i "$perf_fp_data" | stackcollapse-perf > "$perf_fp_folded" # Display results if [ -f "$perf_dwarf_folded" ]; then From d21319c407dddf4bfa9fa59ded1960e5354a0e87 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:05:01 -0700 Subject: [PATCH 19/28] Use Go implementation of stackcollapse-perf to eliminate Perl dependency. Signed-off-by: Harper, Jason M --- tools/Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/Makefile b/tools/Makefile index a39bc266..ee9273d2 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -254,6 +254,7 @@ reset: cd msr-tools && git clean -fdx && git reset --hard cd spectre-meltdown-checker cd sshpass && make clean + cd stackcollapse-perf && git clean -fdx && git reset --hard cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc From cfdb78c6860aa320b5393950b3a6e6244cd08da0 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 10:54:07 -0700 Subject: [PATCH 20/28] Add go.mod file for stackcollapse-perf module Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/stackcollapse-perf/go.mod b/tools/stackcollapse-perf/go.mod index cbed3b20..b3cd282a 100644 --- a/tools/stackcollapse-perf/go.mod +++ b/tools/stackcollapse-perf/go.mod @@ -1,3 +1,3 @@ module intel.com/stackcollapse-perf -go 1.24.1 +go 1.24 From cf72f78040267e248089ee5d6538162123e34008 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Tue, 6 May 2025 11:17:25 -0700 Subject: [PATCH 21/28] Remove stackcollapse-perf reset command from Makefile Signed-off-by: Harper, Jason M --- tools/Makefile | 1 - 1 file changed, 1 deletion(-) diff --git a/tools/Makefile b/tools/Makefile index ee9273d2..a39bc266 100644 --- a/tools/Makefile +++ b/tools/Makefile @@ -254,7 +254,6 @@ reset: cd msr-tools && git clean -fdx && git reset --hard cd spectre-meltdown-checker cd sshpass && make clean - cd stackcollapse-perf && git clean -fdx && git reset --hard cd stress-ng && git clean -fdx && git reset --hard cd sysstat && git clean -fdx && git reset --hard cd tsc && rm -f tsc From bbf2669a7c8237e4721fb9f3c4af6687a43fb0c7 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Wed, 7 May 2025 16:23:49 -0700 Subject: [PATCH 22/28] fix out of range panic Signed-off-by: Harper, Jason M --- tools/stackcollapse-perf/stackcollapse-perf.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/stackcollapse-perf/stackcollapse-perf.go b/tools/stackcollapse-perf/stackcollapse-perf.go index ca63125b..d482e6e8 100644 --- a/tools/stackcollapse-perf/stackcollapse-perf.go +++ b/tools/stackcollapse-perf/stackcollapse-perf.go @@ -214,7 +214,9 @@ func main() { funcname = strings.ReplaceAll(funcname, ";", ":") if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { index := stripAnonymousRegex.FindStringIndex(funcname) - funcname = funcname[0:index[0]] + if index != nil { + funcname = funcname[0:index[0]] + } } funcname = strings.ReplaceAll(funcname, "\"", "") funcname = strings.ReplaceAll(funcname, "'", "") From e4d46740eb81d655fdddef3575ac442f91188af0 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Wed, 7 May 2025 16:43:26 -0700 Subject: [PATCH 23/28] add maximum depth of flame graph rendering Signed-off-by: Harper, Jason M --- cmd/flame/flame.go | 13 ++++++ internal/report/render_html_flamegraph.go | 56 +++++++++++++---------- internal/report/table_defs.go | 1 + internal/report/table_helpers.go | 14 ++++++ internal/script/script_defs.go | 8 ++++ 5 files changed, 68 insertions(+), 24 deletions(-) diff --git a/cmd/flame/flame.go b/cmd/flame/flame.go index 3695c1da..d1455b67 100644 --- a/cmd/flame/flame.go +++ b/cmd/flame/flame.go @@ -42,6 +42,7 @@ var ( flagFrequency int flagPid int flagNoSystemSummary bool + flagMaxDepth int ) const ( @@ -49,6 +50,7 @@ const ( flagFrequencyName = "frequency" flagPidName = "pid" flagNoSystemSummaryName = "no-summary" + flagMaxDepthName = "max-depth" ) func init() { @@ -58,6 +60,7 @@ func init() { Cmd.Flags().IntVar(&flagFrequency, flagFrequencyName, 11, "") Cmd.Flags().IntVar(&flagPid, flagPidName, 0, "") Cmd.Flags().BoolVar(&flagNoSystemSummary, flagNoSystemSummaryName, false, "") + Cmd.Flags().IntVar(&flagMaxDepth, flagMaxDepthName, 0, "") common.AddTargetFlags(Cmd) @@ -108,6 +111,10 @@ func getFlagGroups() []common.FlagGroup { Name: common.FlagFormatName, Help: fmt.Sprintf("choose output format(s) from: %s", strings.Join(append([]string{report.FormatAll}, report.FormatHtml, report.FormatTxt, report.FormatJson), ", ")), }, + { + Name: flagMaxDepthName, + Help: "maximum render depth of call stack in flamegraph (0 = no limit)", + }, { Name: flagNoSystemSummaryName, Help: "do not include system summary table in report", @@ -164,6 +171,11 @@ func validateFlags(cmd *cobra.Command, args []string) error { fmt.Fprintf(os.Stderr, "Error: %v\n", err) return err } + if flagMaxDepth < 0 { + err := fmt.Errorf("max depth must be greater than or equal to 0") + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + return err + } // common target flags if err := common.ValidateTargetFlags(cmd); err != nil { fmt.Fprintf(os.Stderr, "Error: %v\n", err) @@ -185,6 +197,7 @@ func runCmd(cmd *cobra.Command, args []string) error { "Frequency": strconv.Itoa(flagFrequency), "Duration": strconv.Itoa(flagDuration), "PID": strconv.Itoa(flagPid), + "MaxDepth": strconv.Itoa(flagMaxDepth), }, TableNames: tableNames, } diff --git a/internal/report/render_html_flamegraph.go b/internal/report/render_html_flamegraph.go index 02d3b8bf..5e679c89 100644 --- a/internal/report/render_html_flamegraph.go +++ b/internal/report/render_html_flamegraph.go @@ -8,8 +8,9 @@ import ( "bytes" "encoding/json" "fmt" - "log" + "log/slog" "math/rand/v2" // nosemgrep + "slices" "strconv" "strings" texttemplate "text/template" // nosemgrep @@ -81,11 +82,6 @@ type flameGraphTemplateStruct struct { // Folded data conversion adapted from https://github.com/spiermar/burn // Copyright © 2017 Martin Spier // Apache License, Version 2.0 -func reverse(strings []string) { - for i, j := 0, len(strings)-1; i < j; i, j = i+1, j-1 { - strings[i], strings[j] = strings[j], strings[i] - } -} type Node struct { Name string @@ -123,24 +119,24 @@ func (n *Node) MarshalJSON() ([]byte, error) { }) } -var maxStackDepth = 50 - -func convertFoldedToJSON(folded string) (out string, err error) { +func convertFoldedToJSON(folded string, maxStackDepth int) (out string, err error) { rootNode := Node{Name: "root", Value: 0, Children: make(map[string]*Node)} scanner := bufio.NewScanner(strings.NewReader(folded)) for scanner.Scan() { line := scanner.Text() - sep := strings.LastIndex(line, " ") - s := line[:sep] - v := line[sep+1:] - stack := strings.Split(s, ";") - reverse(stack) - if len(stack) > maxStackDepth { - log.Printf("Trimming call stack depth from %d to %d", len(stack), maxStackDepth) - stack = stack[:maxStackDepth] + sep := strings.LastIndex(line, " ") // space separates the call stack from the count + callstack := line[:sep] + count := line[sep+1:] + stack := strings.Split(callstack, ";") // semicolon separates the functions in the call stack + slices.Reverse(stack) + if maxStackDepth > 0 { + if len(stack) > maxStackDepth { + slog.Info("Trimming call stack depth", slog.Int("stack length", len(stack)), slog.Int("max depth", maxStackDepth)) + stack = stack[:maxStackDepth] + } } var i int - i, err = strconv.Atoi(v) + i, err = strconv.Atoi(count) if err != nil { return } @@ -152,9 +148,21 @@ func convertFoldedToJSON(folded string) (out string, err error) { } func renderFlameGraph(header string, tableValues TableValues, field string) (out string) { + maxDepthFieldIndex, err := getFieldIndex("Maximum Render Depth", tableValues) + if err != nil { + slog.Error("didn't find expected field (Maximum Render Depth) in table", slog.String("error", err.Error())) + return + } + maxDepth := tableValues.Fields[maxDepthFieldIndex].Values[0] + maxStackDepth, err := strconv.Atoi(strings.TrimSpace(maxDepth)) + if err != nil { + slog.Error("failed to convert maximum stack depth", slog.String("error", err.Error())) + return + } fieldIdx, err := getFieldIndex(field, tableValues) if err != nil { - log.Panicf("didn't find expected field (%s) in table: %v", field, err) + slog.Error("didn't find expected field in table", slog.String("field", field), slog.String("error", err.Error())) + return } folded := tableValues.Fields[fieldIdx].Values[0] if folded == "" { @@ -166,10 +174,10 @@ func renderFlameGraph(header string, tableValues TableValues, field string) (out out += msg return } - jsonStacks, err := convertFoldedToJSON(folded) + jsonStacks, err := convertFoldedToJSON(folded, maxStackDepth) if err != nil { - log.Printf("failed to convert folded data: %v", err) - out += "Error." + slog.Error("failed to convert folded data", slog.String("error", err.Error())) + out = "" return } fg := texttemplate.Must(texttemplate.New("flameGraphTemplate").Parse(flameGraphTemplate)) @@ -180,8 +188,8 @@ func renderFlameGraph(header string, tableValues TableValues, field string) (out Header: header, }) if err != nil { - log.Printf("failed to render flame graph template: %v", err) - out += "Error." + slog.Error("failed to render flame graph template", slog.String("error", err.Error())) + out = "" return } out += buf.String() diff --git a/internal/report/table_defs.go b/internal/report/table_defs.go index 69fb5dec..ff8fcbba 100644 --- a/internal/report/table_defs.go +++ b/internal/report/table_defs.go @@ -2386,6 +2386,7 @@ func codePathFrequencyTableValues(outputs map[string]script.ScriptOutput) []Fiel fields := []Field{ {Name: "System Paths", Values: []string{systemFoldedFromOutput(outputs)}}, {Name: "Java Paths", Values: []string{javaFoldedFromOutput(outputs)}}, + {Name: "Maximum Render Depth", Values: []string{maxRenderDepthFromOutput(outputs)}}, } return fields } diff --git a/internal/report/table_helpers.go b/internal/report/table_helpers.go index 3f75b111..05e2941b 100644 --- a/internal/report/table_helpers.go +++ b/internal/report/table_helpers.go @@ -2141,3 +2141,17 @@ func systemFoldedFromOutput(outputs map[string]script.ScriptOutput) string { } return folded } + +func maxRenderDepthFromOutput(outputs map[string]script.ScriptOutput) string { + sections := getSectionsFromOutput(outputs[script.ProfileSystemScriptName].Stdout) + if len(sections) == 0 { + slog.Warn("no sections in system profiling output") + return "" + } + for header, content := range sections { + if header == "maximum depth" { + return content + } + } + return "" +} diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index ce9c078b..5209b783 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1244,6 +1244,7 @@ fi pid={{.PID}} duration={{.Duration}} frequency={{.Frequency}} +maxdepth={{.MaxDepth}} ap_interval=0 if [ "$frequency" -ne 0 ]; then @@ -1295,6 +1296,9 @@ for idx in "${!java_pids[@]}"; do async-profiler/profiler.sh stop -o collapsed "$pid" done +echo "########## maximum depth ##########" +echo "$maxdepth" + `, Superuser: true, Depends: []string{"async-profiler"}, @@ -1305,6 +1309,7 @@ done pid={{.PID}} duration={{.Duration}} frequency={{.Frequency}} +maxdepth={{.MaxDepth}} # Function to restore original settings and clean up # This function will be called on exit @@ -1406,6 +1411,9 @@ if [ -f "$perf_fp_folded" ]; then cat "$perf_fp_folded" fi +echo "########## maximum depth ##########" +echo "$maxdepth" + # Clean up temporary files rm -f "$perf_fp_data" "$perf_dwarf_data" "$perf_dwarf_folded" "$perf_fp_folded" `, From e3454b2716fa4b76aad9d20977bc7af5c32e4fbe Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Thu, 8 May 2025 13:16:55 -0700 Subject: [PATCH 24/28] refactor and add unit tests Signed-off-by: Harper, Jason M --- Makefile | 1 + internal/script/script_defs.go | 35 +- .../stackcollapse-perf/stackcollapse-perf.go | 405 ++++++++++-------- .../stackcollapse-perf_test.go | 168 ++++++++ 4 files changed, 399 insertions(+), 210 deletions(-) create mode 100644 tools/stackcollapse-perf/stackcollapse-perf_test.go diff --git a/Makefile b/Makefile index 3b03c1af..9e6105ff 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,7 @@ endif test: @echo "Running unit tests..." go test -v ./... + cd tools/stackcollapse-perf && go test -v ./... .PHONY: update-deps update-deps: diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 5209b783..7fd39aae 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -1316,10 +1316,6 @@ maxdepth={{.MaxDepth}} restore_settings() { echo "$PERF_EVENT_PARANOID" > /proc/sys/kernel/perf_event_paranoid echo "$KPTR_RESTRICT" > /proc/sys/kernel/kptr_restrict - rm -f "$perf_fp_data" - rm -f "$perf_dwarf_data" - rm -f "$perf_dwarf_folded" - rm -f "$perf_fp_folded" if [ -n "$perf_fp_pid" ]; then kill -0 $perf_fp_pid 2>/dev/null && kill -INT $perf_fp_pid fi @@ -1328,12 +1324,6 @@ restore_settings() { fi } -# create temporary output files -perf_fp_data=$(mktemp) -perf_dwarf_data=$(mktemp) -perf_dwarf_folded=$(mktemp) -perf_fp_folded=$(mktemp) - # adjust perf_event_paranoid and kptr_restrict PERF_EVENT_PARANOID=$( cat /proc/sys/kernel/perf_event_paranoid ) echo -1 >/proc/sys/kernel/perf_event_paranoid @@ -1354,10 +1344,10 @@ fi # frame pointer mode # if pid was provided, use it if [ "$pid" -ne 0 ]; then - perf record -F "$frequency" -p "$pid" -g -o "$perf_fp_data" -m 129 & + perf record -F "$frequency" -p "$pid" -g -o perf_fp_data -m 129 & else # if no pid was provided, use system-wide profiling - perf record -F "$frequency" -a -g -o "$perf_fp_data" -m 129 & + perf record -F "$frequency" -a -g -o perf_fp_data -m 129 & fi perf_fp_pid=$! if ! kill -0 $perf_fp_pid 2>/dev/null; then @@ -1368,10 +1358,10 @@ fi # dwarf mode # if pid was provided, use it if [ "$pid" -ne 0 ]; then - perf record -F "$frequency" -p "$pid" -g -o "$perf_dwarf_data" -m 257 --call-graph dwarf,8192 & + perf record -F "$frequency" -p "$pid" -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & else # if no pid was provided, use system-wide profiling - perf record -F "$frequency" -a -g -o "$perf_dwarf_data" -m 257 --call-graph dwarf,8192 & + perf record -F "$frequency" -a -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & fi perf_dwarf_pid=$! if ! kill -0 $perf_dwarf_pid 2>/dev/null; then @@ -1398,24 +1388,23 @@ fi wait ${perf_fp_pid} ${perf_dwarf_pid} # collapse perf data -perf script -i "$perf_dwarf_data" | stackcollapse-perf > "$perf_dwarf_folded" -perf script -i "$perf_fp_data" | stackcollapse-perf > "$perf_fp_folded" +perf script -i perf_dwarf_data > perf_dwarf_stacks +stackcollapse-perf perf_dwarf_stacks > perf_dwarf_folded +perf script -i perf_fp_data > perf_fp_stacks +stackcollapse-perf perf_fp_stacks > perf_fp_folded # Display results -if [ -f "$perf_dwarf_folded" ]; then +if [ -f perf_dwarf_folded ]; then echo "########## perf_dwarf ##########" - cat "$perf_dwarf_folded" + cat perf_dwarf_folded fi -if [ -f "$perf_fp_folded" ]; then +if [ -f perf_fp_folded ]; then echo "########## perf_fp ##########" - cat "$perf_fp_folded" + cat perf_fp_folded fi echo "########## maximum depth ##########" echo "$maxdepth" - -# Clean up temporary files -rm -f "$perf_fp_data" "$perf_dwarf_data" "$perf_dwarf_folded" "$perf_fp_folded" `, Superuser: true, Depends: []string{"perf", "stackcollapse-perf"}, diff --git a/tools/stackcollapse-perf/stackcollapse-perf.go b/tools/stackcollapse-perf/stackcollapse-perf.go index d482e6e8..e0ef9b0b 100644 --- a/tools/stackcollapse-perf/stackcollapse-perf.go +++ b/tools/stackcollapse-perf/stackcollapse-perf.go @@ -10,6 +10,7 @@ package main import ( "bufio" "fmt" + "io" "os" "path/filepath" "regexp" @@ -18,42 +19,41 @@ import ( "strings" ) -// In the original Perl code, the following -// are command line arguments. For our use, we don't need to set them as flags, -// but we can keep them for compatibility with the original -var annotateKernel = false -var annotateJit = false - -// var annotateAll = false -var includePname = true -var includePid = false -var includeTid = false -var includeAddrs = false -var tidyJava = true -var tidyGeneric = true - -// var targetPname = "" -var eventFilter = "" - -// var showInline = false -// var showContext = false -// var srcLineInInput = false +// Config holds configuration options for processing stacks. +// It includes options for annotating kernel and JIT symbols, including process names, PIDs, TIDs, and addresses, +// as well as options for tidying Java and generic function names, filtering events, and showing inline or context information. +type Config struct { + AnnotateKernel bool + AnnotateJit bool + IncludePname bool + IncludePid bool + IncludeTid bool + IncludeAddrs bool + TidyJava bool + TidyGeneric bool + EventFilter string + ShowInline bool + ShowContext bool + SrcLineInInput bool +} +// StackAggregator aggregates stack traces and their counts. +// It provides a method to remember stacks and their associated counts. type StackAggregator struct { collapsed map[string]int } +// NewStackAggregator creates and returns a new StackAggregator instance. func NewStackAggregator() *StackAggregator { return &StackAggregator{collapsed: make(map[string]int)} } +// RememberStack adds a stack trace and its count to the aggregator. func (sa *StackAggregator) RememberStack(stack string, count int) { sa.collapsed[stack] += count } func main() { - aggregator := NewStackAggregator() - var input *os.File var err error @@ -68,204 +68,235 @@ func main() { } else { input = os.Stdin // Default to standard input } + + var config = Config{ + AnnotateKernel: false, + AnnotateJit: false, + IncludePname: true, + IncludePid: false, + IncludeTid: false, + IncludeAddrs: false, + TidyJava: true, + TidyGeneric: true, + EventFilter: "", + ShowInline: false, + ShowContext: false, + SrcLineInInput: false, + } + + err = ProcessStacks(input, os.Stdout, config) + if err != nil { + fmt.Fprintf(os.Stderr, "Error processing stacks: %s\n", err) + os.Exit(1) + } +} + +// Regular expressions for parsing the perf output +var ( + eventLineRegex = regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) + eventTypeRegex = regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) + stackLineRegex = regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) + // inlineRegex = regexp.MustCompile(`(perf-\d+.map|kernel\.|\[[^\]]+\])`) + stripSymbolsRegex = regexp.MustCompile(`\+0x[\da-f]+$`) + stripIdRegex = regexp.MustCompile(`\.\(.*\)\.`) + stripAnonymousRegex = regexp.MustCompile(`\([^a]*anonymous namespace[^)]*\)`) + jitRegex = regexp.MustCompile(`/tmp/perf-\d+\.map`) +) + +// ProcessStacks processes stack traces from the input reader and writes the collapsed stacks to the output writer. +// It uses the provided configuration to control the processing behavior. +func ProcessStacks(input io.Reader, output io.Writer, config Config) error { + aggregator := NewStackAggregator() scanner := bufio.NewScanner(input) var stack []string - var pname string - var mPeriod int - var mTid string - var mPid string - - eventRegex := regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) - eventTypeRegex := regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) - stackLineRegex := regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) - // inlineRegex := regexp.MustCompile(`(perf-\d+.map|kernel\.|\[[^\]]+\])`) - stripSymbolsRegex := regexp.MustCompile(`\+0x[\da-f]+$`) - stripIdRegex := regexp.MustCompile(`\.\(.*\)\.`) - stripAnonymousRegex := regexp.MustCompile(`\([^a]*anonymous namespace[^)]*\)`) - jitRegex := regexp.MustCompile(`/tmp/perf-\d+\.map`) - - var eventDefaulted bool - var eventWarning bool + var processName string + var period int // main loop, read lines from stdin for scanner.Scan() { line := scanner.Text() - if strings.HasPrefix(line, "# cmdline") { - // loop through the command line arguments in reverse order - for i := len(os.Args) - 1; i > 0; i-- { - if !strings.HasPrefix(os.Args[i], "-") { - // not used -> target_pname = filepath.Base(os.Args[i]) - break - } - } + if strings.HasPrefix(line, "#") { + continue } - // Skip remaining comments - if strings.HasPrefix(line, "#") { + if line == "" && processName == "" { continue } - // End of stack - if line == "" { - if pname == "" { - continue + if line == "" { // check for End of stack + if config.IncludePname { + stack = append([]string{processName}, stack...) } - if includePname { - // prepend the process name to the stack - stack = append([]string{pname}, stack...) - } - if stack != nil { - aggregator.RememberStack(strings.Join(stack, ";"), mPeriod) + aggregator.RememberStack(strings.Join(stack, ";"), period) } stack = nil - pname = "" + processName = "" continue } + if err := handleEventRecord(line, &processName, &period, config); err != nil { + fmt.Fprintf(output, "Error: %s\n", err) + continue + } else if err := handleStackLine(line, &stack, processName, config); err != nil { + fmt.Fprintf(output, "Error: %s\n", err) + continue + } + } - // Event record start - if matches := eventRegex.FindStringSubmatch(line); matches != nil { - comm, pid, tid, period := matches[1], matches[2], matches[3], "" - if tid == "" { - tid = pid - pid = "?" - } + // Check for errors during scanning + if err := scanner.Err(); err != nil { + fmt.Fprintf(os.Stderr, "Error reading input: %s\n", err) + return err + } - if eventMatches := eventTypeRegex.FindStringSubmatch(line); eventMatches != nil { - period = eventMatches[1] - event := eventMatches[2] - - if eventFilter == "" { - eventFilter = event - eventDefaulted = true - } else if event != eventFilter { - if eventDefaulted && !eventWarning { - fmt.Fprintf(os.Stderr, "Filtering for events of type: %s\n", event) - eventWarning = true - } - continue - } - } + // Output results + keys := make([]string, 0, len(aggregator.collapsed)) + for k := range aggregator.collapsed { + keys = append(keys, k) + } + sort.Strings(keys) - if period == "" { - period = "1" - } - periodInt, err := strconv.Atoi(period) + for _, k := range keys { + fmt.Fprintf(output, "%s %d\n", k, aggregator.collapsed[k]) + } + + return nil +} + +// handleEventRecord parses an event record line and updates the process name and period based on the configuration. +func handleEventRecord(line string, processName *string, period *int, config Config) error { + matches := eventLineRegex.FindStringSubmatch(line) + if matches == nil { + return nil + } + + comm, pid, tid := matches[1], matches[2], matches[3] + if tid == "" { + tid = pid + pid = "?" + } + + if eventMatches := eventTypeRegex.FindStringSubmatch(line); eventMatches != nil { + eventPeriod := eventMatches[1] + if eventPeriod == "" { + *period = 1 + } else { + eventPeriodInt, err := strconv.Atoi(eventPeriod) if err != nil { - fmt.Fprintf(os.Stderr, "Error parsing period: %s\n", err) - continue + return fmt.Errorf("failed to parse event period: %s, error: %v", eventPeriod, err) } - mPid, mTid, mPeriod = pid, tid, periodInt + *period = eventPeriodInt + } + event := eventMatches[2] + + if config.EventFilter == "" { + config.EventFilter = event + } else if event != config.EventFilter { + return fmt.Errorf("event type mismatch: %s != %s", event, config.EventFilter) + } + } + + if config.IncludeTid { + *processName = fmt.Sprintf("%s-%s/%s", comm, pid, tid) + } else if config.IncludePid { + *processName = fmt.Sprintf("%s-%s", comm, pid) + } else { + *processName = comm + } + *processName = strings.ReplaceAll(*processName, " ", "_") + return nil +} + +// handleStackLine parses a stack line and appends the function name to the stack based on the configuration. +func handleStackLine(line string, stack *[]string, pname string, config Config) error { + matches := stackLineRegex.FindStringSubmatch(line) + if matches == nil || pname == "" { + return nil + } - if includeTid { - pname = fmt.Sprintf("%s-%s/%s", comm, mPid, mTid) - } else if includePid { - pname = fmt.Sprintf("%s-%s", comm, mPid) + pc, rawFunc, mod := matches[1], matches[2], matches[3] + + // skip for now as showInline is always false + // if showInline && !inlineRegex.MatchString(mod) { + // inlineRes := inline(pc, rawFunc, mod) + // if inlineRes != "" && inlineRes != "??" && inlineRes != "??:??:0" { + // // prepend the inline result to the stack + // stack = append([]string{inlineRes}, stack...) + // continue + // } + //} + + // strip symbol offsets from rawFunc + // symbol offsets match this regex: \+0x[\da-f]+$ + rawFunc = stripSymbolsRegex.ReplaceAllString(rawFunc, "") + + // skip process names + if strings.HasPrefix(rawFunc, "(") { + return nil + } + + *stack = append(processFunctionName(rawFunc, mod, pc, config), *stack...) + return nil +} + +// processFunctionName processes a raw function name, module, and program counter (PC) based on the configuration. +// It returns a slice of processed function names. +func processFunctionName(rawFunc, mod, pc string, config Config) []string { + // var isUnknown bool + var inline []string + for funcname := range strings.SplitSeq(rawFunc, "->") { + if funcname == "[unknown]" { // use module name instead, if known + if mod != "[unknown]" { + funcname = filepath.Base(mod) } else { - pname = comm - } - pname = strings.ReplaceAll(pname, " ", "_") - continue - // Stack line - } else if matches := stackLineRegex.FindStringSubmatch(line); matches != nil { - if pname == "" { - continue + funcname = "unknown" + // isUnknown = true } - pc, rawFunc, mod := matches[1], matches[2], matches[3] - - // skip for now as showInline is always false - // if showInline && !inlineRegex.MatchString(mod) { - // inlineRes := inline(pc, rawFunc, mod) - // if inlineRes != "" && inlineRes != "??" && inlineRes != "??:??:0" { - // // prepend the inline result to the stack - // stack = append([]string{inlineRes}, stack...) - // continue - // } - //} - - // strip symbol offsets from rawFunc - // symbol offsets match this regex: \+0x[\da-f]+$ - rawFunc = stripSymbolsRegex.ReplaceAllString(rawFunc, "") - - // skip process names - if strings.HasPrefix(rawFunc, "(") { - continue + + if config.IncludeAddrs { + funcname = fmt.Sprintf("[%s <%s>]", funcname, pc) + } else { + funcname = fmt.Sprintf("[%s]", funcname) } - // var isUnknown bool - var inline []string - for funcname := range strings.SplitSeq(rawFunc, "->") { - if funcname == "[unknown]" { // use module name instead, if known - if mod != "[unknown]" { - funcname = filepath.Base(mod) - } else { - funcname = "unknown" - // isUnknown = true - } - - if includeAddrs { - funcname = fmt.Sprintf("[%s <%s>]", funcname, pc) - } else { - funcname = fmt.Sprintf("[%s]", funcname) - } - } - if tidyGeneric { - funcname = strings.ReplaceAll(funcname, ";", ":") - if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { - index := stripAnonymousRegex.FindStringIndex(funcname) - if index != nil { - funcname = funcname[0:index[0]] - } - } - funcname = strings.ReplaceAll(funcname, "\"", "") - funcname = strings.ReplaceAll(funcname, "'", "") - } - if tidyJava { - if strings.Contains(funcname, "/") { - // strip the leading L - funcname = strings.TrimPrefix(funcname, "L") - } - } - // annotations - if len(inline) > 0 { - if !strings.Contains(funcname, "_[i]") { - funcname = fmt.Sprintf("%s_[i]", funcname) - } else if annotateKernel && (strings.HasPrefix(funcname, "[") || strings.HasSuffix(funcname, "vmlinux")) && !strings.Contains(mod, "unknown") { - funcname = fmt.Sprintf("%s_[k]", funcname) - } else if annotateJit && jitRegex.MatchString(funcname) { - if !strings.Contains(funcname, "_[j]") { - funcname = fmt.Sprintf("%s_[j]", funcname) - } - } + } + if config.TidyGeneric { + funcname = strings.ReplaceAll(funcname, ";", ":") + if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { + index := stripAnonymousRegex.FindStringIndex(funcname) + if index != nil { + funcname = funcname[0:index[0]] } - - // source lines - // skip for now since srcLineInInput is always false - // if srcLineInInput && !isUnknown { - // } - - inline = append(inline, funcname) } - - // prepend inline array to the stack array - if len(inline) > 0 { - stack = append(inline, stack...) + funcname = strings.ReplaceAll(funcname, "\"", "") + funcname = strings.ReplaceAll(funcname, "'", "") + } + if config.TidyJava { + if strings.Contains(funcname, "/") { + // strip the leading L + funcname = strings.TrimPrefix(funcname, "L") + } + } + // annotations + if len(inline) > 0 { + if !strings.Contains(funcname, "_[i]") { + funcname = fmt.Sprintf("%s_[i]", funcname) + } else if config.AnnotateKernel && (strings.HasPrefix(funcname, "[") || strings.HasSuffix(funcname, "vmlinux")) && !strings.Contains(mod, "unknown") { + funcname = fmt.Sprintf("%s_[k]", funcname) + } else if config.AnnotateJit && jitRegex.MatchString(funcname) { + if !strings.Contains(funcname, "_[j]") { + funcname = fmt.Sprintf("%s_[j]", funcname) + } } - - } else { - fmt.Fprintf(os.Stderr, "Unknown line format: %s\n", line) } - } - // Output results - keys := make([]string, 0, len(aggregator.collapsed)) - for k := range aggregator.collapsed { - keys = append(keys, k) - } - sort.Strings(keys) + // source lines + // skip for now since srcLineInInput is always false + // if srcLineInInput && !isUnknown { + // } - for _, k := range keys { - fmt.Printf("%s %d\n", k, aggregator.collapsed[k]) + inline = append(inline, funcname) } + return inline } diff --git a/tools/stackcollapse-perf/stackcollapse-perf_test.go b/tools/stackcollapse-perf/stackcollapse-perf_test.go new file mode 100644 index 00000000..e77bc723 --- /dev/null +++ b/tools/stackcollapse-perf/stackcollapse-perf_test.go @@ -0,0 +1,168 @@ +package main + +// Copyright (C) 2021-2025 Intel Corporation +// SPDX-License-Identifier: BSD-3-Clause + +import ( + "bytes" + "strings" + "testing" +) + +// keep for debugging +// func TestProcessStacksFromFile(t *testing.T) { +// filePath := filepath.Join("testdata", "perf_fp_stacks") +// file, err := os.Open(filePath) +// if err != nil { +// t.Fatalf("failed to open test file: %v", err) +// } +// defer file.Close() + +// output := &bytes.Buffer{} +// config := Config{ +// IncludePname: true, +// IncludePid: false, +// IncludeTid: false, +// IncludeAddrs: false, +// TidyJava: true, +// TidyGeneric: true, +// } + +// err = ProcessStacks(file, output, config) +// if err != nil { +// t.Fatalf("unexpected error: %v", err) +// } + +// if output.Len() == 0 { +// t.Errorf("expected output, got empty result") +// } +// } + +func TestProcessStacks(t *testing.T) { + input := strings.NewReader(` +stress-ng-cpu 1230556 [121] 6223127.073349: 293637623 cycles:P: + 61e248df6091 [unknown] (/usr/bin/stress-ng) + +stress-ng-cpu 1230793 [098] 6223127.074783: 307465331 cycles:P: + ffffffffa7c00f0b asm_sysvec_apic_timer_interrupt+0x1b ([kernel.kallsyms]) + 760c9702dc5d [unknown] (/usr/lib/x86_64-linux-gnu/libm.so.6) + 760c96fda3a2 sincosf64x+0x122 (/usr/lib/x86_64-linux-gnu/libm.so.6) + + `) + output := &bytes.Buffer{} + + config := Config{ + IncludePname: true, + IncludePid: false, + IncludeTid: false, + IncludeAddrs: false, + TidyJava: true, + TidyGeneric: true, + } + + err := ProcessStacks(input, output, config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expected := "stress-ng-cpu;[stress-ng] 293637623\nstress-ng-cpu;sincosf64x;[libm.so.6];asm_sysvec_apic_timer_interrupt 307465331\n" + if output.String() != expected { + t.Errorf("expected %q, got %q", expected, output.String()) + } +} + +func TestHandleEventRecord(t *testing.T) { + line := "stress-ng-cpu 1230556 [121] 6223127.073349: 293637623 cycles:P: " + var processName string + var period int + config := Config{ + IncludePname: true, + IncludePid: false, + IncludeTid: false, + IncludeAddrs: false, + TidyJava: true, + TidyGeneric: true, + } + + err := handleEventRecord(line, &processName, &period, config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + expectedProcessName := "stress-ng-cpu" + if processName != "stress-ng-cpu" { + t.Errorf("expected processName to be '%s', got %q", expectedProcessName, processName) + } + expectedPeriod := 293637623 + if period != expectedPeriod { + t.Errorf("expected period to be %d, got %d", expectedPeriod, period) + } +} + +func TestHandleStackLine(t *testing.T) { + line := "0x1234 someFunction (module)" + var stack []string + processName := "main" + config := Config{ + IncludePname: true, + IncludePid: false, + IncludeTid: false, + IncludeAddrs: false, + TidyJava: true, + TidyGeneric: true, + } + + err := handleStackLine(line, &stack, processName, config) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if len(stack) != 1 || stack[0] != "someFunction" { + t.Errorf("expected stack to contain 'someFunction', got %v", stack) + } +} + +func TestProcessFunctionName(t *testing.T) { + tests := []struct { + rawFunc string + mod string + pc string + config Config + expected []string + }{ + { + rawFunc: "[unknown]", + mod: "[kernel]", + pc: "0x1234", + config: Config{IncludeAddrs: true}, + expected: []string{"[[kernel] <0x1234>]"}, + }, + { + rawFunc: "Lcom/example/MyClass", + mod: "[unknown]", + pc: "0x5678", + config: Config{TidyJava: true}, + expected: []string{"com/example/MyClass"}, + }, + { + rawFunc: "someFunction;bar\"hello'world.(anonymous namespace).foo", + mod: "module.so", + pc: "0x9abc", + config: Config{TidyGeneric: true}, + expected: []string{"someFunction:barhelloworld."}, + }, + } + + for _, test := range tests { + result := processFunctionName(test.rawFunc, test.mod, test.pc, test.config) + if len(result) != len(test.expected) { + t.Errorf("expected %d results, got %d", len(test.expected), len(result)) + continue + } + for i := range result { + if result[i] != test.expected[i] { + t.Errorf("expected %q, got %q", test.expected[i], result[i]) + } + } + } +} From 139f6525c3bf568789ba0ce9314690f74910fc75 Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Fri, 9 May 2025 10:43:52 -0700 Subject: [PATCH 25/28] support multiple pids Signed-off-by: Harper, Jason M --- cmd/flame/flame.go | 25 ++-- internal/report/render_html.go | 6 +- internal/report/table_defs.go | 21 ++-- internal/report/table_helpers.go | 35 +++--- internal/script/script.go | 4 +- internal/script/script_defs.go | 204 ++++++++++++++----------------- internal/util/util.go | 9 ++ internal/util/util_test.go | 19 +++ 8 files changed, 164 insertions(+), 159 deletions(-) diff --git a/cmd/flame/flame.go b/cmd/flame/flame.go index d1455b67..75f45635 100644 --- a/cmd/flame/flame.go +++ b/cmd/flame/flame.go @@ -9,6 +9,7 @@ import ( "os" "perfspect/internal/common" "perfspect/internal/report" + "perfspect/internal/util" "slices" "strconv" "strings" @@ -40,7 +41,7 @@ var Cmd = &cobra.Command{ var ( flagDuration int flagFrequency int - flagPid int + flagPids []int flagNoSystemSummary bool flagMaxDepth int ) @@ -48,7 +49,7 @@ var ( const ( flagDurationName = "duration" flagFrequencyName = "frequency" - flagPidName = "pid" + flagPidsName = "pids" flagNoSystemSummaryName = "no-summary" flagMaxDepthName = "max-depth" ) @@ -58,7 +59,7 @@ func init() { Cmd.Flags().StringSliceVar(&common.FlagFormat, common.FlagFormatName, []string{report.FormatAll}, "") Cmd.Flags().IntVar(&flagDuration, flagDurationName, 30, "") Cmd.Flags().IntVar(&flagFrequency, flagFrequencyName, 11, "") - Cmd.Flags().IntVar(&flagPid, flagPidName, 0, "") + Cmd.Flags().IntSliceVar(&flagPids, flagPidsName, nil, "") Cmd.Flags().BoolVar(&flagNoSystemSummary, flagNoSystemSummaryName, false, "") Cmd.Flags().IntVar(&flagMaxDepth, flagMaxDepthName, 0, "") @@ -104,8 +105,8 @@ func getFlagGroups() []common.FlagGroup { Help: "number of samples taken per second", }, { - Name: flagPidName, - Help: "pid to collect data from. If not specified, all pids will be collected", + Name: flagPidsName, + Help: "comma separated list of PIDs. If not specified, all PIDs will be collected", }, { Name: common.FlagFormatName, @@ -166,10 +167,12 @@ func validateFlags(cmd *cobra.Command, args []string) error { fmt.Fprintf(os.Stderr, "Error: %v\n", err) return err } - if flagPid < 0 { - err := fmt.Errorf("pid must be greater than or equal to 0") - fmt.Fprintf(os.Stderr, "Error: %v\n", err) - return err + for _, pid := range flagPids { + if pid < 0 { + err := fmt.Errorf("PID must be greater than or equal to 0") + fmt.Fprintf(os.Stderr, "Error: %v\n", err) + return err + } } if flagMaxDepth < 0 { err := fmt.Errorf("max depth must be greater than or equal to 0") @@ -189,14 +192,14 @@ func runCmd(cmd *cobra.Command, args []string) error { if !flagNoSystemSummary { tableNames = append(tableNames, report.BriefSysSummaryTableName) } - tableNames = append(tableNames, report.CodePathFrequencyTableName) + tableNames = append(tableNames, report.CallStackFrequencyTableName) reportingCommand := common.ReportingCommand{ Cmd: cmd, ReportNamePost: "flame", ScriptParams: map[string]string{ "Frequency": strconv.Itoa(flagFrequency), "Duration": strconv.Itoa(flagDuration), - "PID": strconv.Itoa(flagPid), + "PIDs": strings.Join(util.IntSliceToStringSlice(flagPids), ","), "MaxDepth": strconv.Itoa(flagMaxDepth), }, TableNames: tableNames, diff --git a/internal/report/render_html.go b/internal/report/render_html.go index 715ac60d..c7778601 100644 --- a/internal/report/render_html.go +++ b/internal/report/render_html.go @@ -1284,7 +1284,7 @@ func gaudiTelemetryTableHTMLRenderer(tableValues TableValues, targetName string) return out } -func codePathFrequencyTableHTMLRenderer(tableValues TableValues, targetName string) string { +func callStackFrequencyTableHTMLRenderer(tableValues TableValues, targetName string) string { out := ` ` - out += renderFlameGraph("System", tableValues, "System Paths") - out += renderFlameGraph("Java", tableValues, "Java Paths") + out += renderFlameGraph("Native", tableValues, "Native Stacks") + out += renderFlameGraph("Java", tableValues, "Java Stacks") return out } diff --git a/internal/report/table_defs.go b/internal/report/table_defs.go index ff8fcbba..6d6e1a40 100644 --- a/internal/report/table_defs.go +++ b/internal/report/table_defs.go @@ -129,7 +129,7 @@ const ( // config table names ConfigurationTableName = "Configuration" // flamegraph table names - CodePathFrequencyTableName = "Code Path Frequency" + CallStackFrequencyTableName = "Call Stack Frequency" // lock table names KernelLockAnalysisTableName = "Kernel Lock Analysis" // common table names @@ -742,15 +742,14 @@ var tableDefinitions = map[string]TableDefinition{ // // flamegraph tables // - CodePathFrequencyTableName: { - Name: CodePathFrequencyTableName, - MenuLabel: CodePathFrequencyTableName, + CallStackFrequencyTableName: { + Name: CallStackFrequencyTableName, + MenuLabel: CallStackFrequencyTableName, ScriptNames: []string{ - script.ProfileJavaScriptName, - script.ProfileSystemScriptName, + script.CollapsedCallStacksScriptName, }, - FieldsFunc: codePathFrequencyTableValues, - HTMLTableRendererFunc: codePathFrequencyTableHTMLRenderer}, + FieldsFunc: callStackFrequencyTableValues, + HTMLTableRendererFunc: callStackFrequencyTableHTMLRenderer}, // // kernel lock analysis tables // @@ -2382,10 +2381,10 @@ func gaudiTelemetryTableValues(outputs map[string]script.ScriptOutput) []Field { return fields } -func codePathFrequencyTableValues(outputs map[string]script.ScriptOutput) []Field { +func callStackFrequencyTableValues(outputs map[string]script.ScriptOutput) []Field { fields := []Field{ - {Name: "System Paths", Values: []string{systemFoldedFromOutput(outputs)}}, - {Name: "Java Paths", Values: []string{javaFoldedFromOutput(outputs)}}, + {Name: "Native Stacks", Values: []string{nativeFoldedFromOutput(outputs)}}, + {Name: "Java Stacks", Values: []string{javaFoldedFromOutput(outputs)}}, {Name: "Maximum Render Depth", Values: []string{maxRenderDepthFromOutput(outputs)}}, } return fields diff --git a/internal/report/table_helpers.go b/internal/report/table_helpers.go index 05e2941b..85554ac8 100644 --- a/internal/report/table_helpers.go +++ b/internal/report/table_helpers.go @@ -2080,29 +2080,28 @@ func sectionValueFromOutput(output string, sectionName string) string { } func javaFoldedFromOutput(outputs map[string]script.ScriptOutput) string { - sections := getSectionsFromOutput(outputs[script.ProfileJavaScriptName].Stdout) + sections := getSectionsFromOutput(outputs[script.CollapsedCallStacksScriptName].Stdout) if len(sections) == 0 { - slog.Warn("no sections in java profiling output") + slog.Warn("no sections in collapsed call stack output") return "" } javaFolded := make(map[string]string) re := regexp.MustCompile(`^async-profiler (\d+) (.*)$`) for header, stacks := range sections { - if stacks == "" { - slog.Info("no stacks for java process", slog.String("header", header)) - continue - } - if strings.HasPrefix(stacks, "Failed to inject profiler") { - slog.Warn("profiling data error", slog.String("header", header)) - continue - } match := re.FindStringSubmatch(header) if match == nil { - slog.Warn("profiling data error, regex didn't match header", slog.String("header", header)) continue } pid := match[1] processName := match[2] + if stacks == "" { + slog.Warn("no stacks for java process", slog.String("header", header)) + continue + } + if strings.HasPrefix(stacks, "Failed to inject profiler") { + slog.Error("profiling data error", slog.String("header", header)) + continue + } _, ok := javaFolded[processName] if processName == "" { processName = "java (" + pid + ")" @@ -2113,15 +2112,15 @@ func javaFoldedFromOutput(outputs map[string]script.ScriptOutput) string { } folded, err := mergeJavaFolded(javaFolded) if err != nil { - slog.Warn("err merging java stacks", slog.String("error", err.Error())) + slog.Error("failed to merge java stacks", slog.String("error", err.Error())) } return folded } -func systemFoldedFromOutput(outputs map[string]script.ScriptOutput) string { - sections := getSectionsFromOutput(outputs[script.ProfileSystemScriptName].Stdout) +func nativeFoldedFromOutput(outputs map[string]script.ScriptOutput) string { + sections := getSectionsFromOutput(outputs[script.CollapsedCallStacksScriptName].Stdout) if len(sections) == 0 { - slog.Warn("no sections in system profiling output") + slog.Warn("no sections in collapsed call stack output") return "" } var dwarfFolded, fpFolded string @@ -2137,15 +2136,15 @@ func systemFoldedFromOutput(outputs map[string]script.ScriptOutput) string { } folded, err := mergeSystemFolded(fpFolded, dwarfFolded) if err != nil { - slog.Warn("error merging folded stacks", slog.String("error", err.Error())) + slog.Error("failed to merge native stacks", slog.String("error", err.Error())) } return folded } func maxRenderDepthFromOutput(outputs map[string]script.ScriptOutput) string { - sections := getSectionsFromOutput(outputs[script.ProfileSystemScriptName].Stdout) + sections := getSectionsFromOutput(outputs[script.CollapsedCallStacksScriptName].Stdout) if len(sections) == 0 { - slog.Warn("no sections in system profiling output") + slog.Warn("no sections in collapsed call stack output") return "" } for header, content := range sections { diff --git a/internal/script/script.go b/internal/script/script.go index 49d63260..3957256c 100644 --- a/internal/script/script.go +++ b/internal/script/script.go @@ -146,9 +146,9 @@ func RunScripts(myTarget target.Target, scripts []ScriptDefinition, ignoreScript return nil, err } else if script.Superuser && !myTarget.IsSuperUser() { // run script with sudo, "-S" to read password from stdin. Note: password won't be asked for if password-less sudo is configured. - cmd = exec.Command("sudo", "-S", "bash", scriptPath) + cmd = exec.Command(fmt.Sprintf("cd %s && sudo -S bash %s", myTarget.GetTempDirectory(), scriptPath)) } else { - cmd = exec.Command("bash", scriptPath) + cmd = exec.Command(fmt.Sprintf("cd %s && bash %s", myTarget.GetTempDirectory(), scriptPath)) } stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, false) if err != nil { diff --git a/internal/script/script_defs.go b/internal/script/script_defs.go index 7fd39aae..568dff0c 100644 --- a/internal/script/script_defs.go +++ b/internal/script/script_defs.go @@ -112,10 +112,8 @@ const ( TurbostatTelemetryScriptName = "turbostat telemetry" InstructionTelemetryScriptName = "instruction telemetry" GaudiTelemetryScriptName = "gaudi telemetry" - // flamegraph scripts - ProfileJavaScriptName = "profile java" - ProfileSystemScriptName = "profile system" + CollapsedCallStacksScriptName = "collapsed call stacks" // lock scripts ProfileKernelLockScriptName = "profile kernel lock" ) @@ -1237,177 +1235,155 @@ fi Superuser: true, NeedsKill: true, }, - // profile (flamegraph) scripts - ProfileJavaScriptName: { - Name: ProfileJavaScriptName, - ScriptTemplate: `# JAVA app (async profiler) call stack collection -pid={{.PID}} + // flamegraph scripts + CollapsedCallStacksScriptName: { + Name: CollapsedCallStacksScriptName, + ScriptTemplate: `# Combined (perf record and async profiler) call stack collection +pids={{.PIDs}} duration={{.Duration}} frequency={{.Frequency}} maxdepth={{.MaxDepth}} ap_interval=0 if [ "$frequency" -ne 0 ]; then - ap_interval=$((1000000000 / frequency)) -fi - -# if pid is provided, use it -if [ "$pid" -ne 0 ]; then - # check if the provided pid is running - if [ ! -d "/proc/$pid" ]; then - echo "pid $pid not running" - exit 1 - fi - # check if pid is a java process, i.e., command line contains java - if ! tr '\000' ' ' < /proc/"$pid"/cmdline | grep -q java; then - echo "pid $pid is not a java process" - exit 1 - fi - pids="$pid" -else - # get all java pids - pids=$( pgrep java ) + ap_interval=$((1000000000 / frequency)) fi -# check if any java pids are found -if [ -z "$pids" ]; then - echo "no java processes found" - exit 1 -fi - -# start java profiling for each java pid -declare -a java_pids=() -declare -a java_cmds=() -for pid in $pids ; do - java_pids+=("$pid") - java_cmds+=("$( tr '\000' ' ' < /proc/"$pid"/cmdline )") - # profile pid in background - async-profiler/profiler.sh start -i "$ap_interval" -o collapsed "$pid" -done - -# wait for the specified duration -sleep "$duration" - -# stop java profiling for each java pid -for idx in "${!java_pids[@]}"; do - pid="${java_pids[$idx]}" - cmd="${java_cmds[$idx]}" - echo "########## async-profiler $pid $cmd ##########" - async-profiler/profiler.sh stop -o collapsed "$pid" -done - -echo "########## maximum depth ##########" -echo "$maxdepth" - -`, - Superuser: true, - Depends: []string{"async-profiler"}, - }, - ProfileSystemScriptName: { - Name: ProfileSystemScriptName, - ScriptTemplate: `# native (perf record) call stack collection -pid={{.PID}} -duration={{.Duration}} -frequency={{.Frequency}} -maxdepth={{.MaxDepth}} - # Function to restore original settings and clean up -# This function will be called on exit restore_settings() { - echo "$PERF_EVENT_PARANOID" > /proc/sys/kernel/perf_event_paranoid - echo "$KPTR_RESTRICT" > /proc/sys/kernel/kptr_restrict + echo "$PERF_EVENT_PARANOID" > /proc/sys/kernel/perf_event_paranoid + echo "$KPTR_RESTRICT" > /proc/sys/kernel/kptr_restrict if [ -n "$perf_fp_pid" ]; then kill -0 $perf_fp_pid 2>/dev/null && kill -INT $perf_fp_pid fi if [ -n "$perf_dwarf_pid" ]; then kill -0 $perf_dwarf_pid 2>/dev/null && kill -INT $perf_dwarf_pid fi + for pid in "${java_pids[@]}"; do + async-profiler/profiler.sh stop -o collapsed "$pid" + done } -# adjust perf_event_paranoid and kptr_restrict -PERF_EVENT_PARANOID=$( cat /proc/sys/kernel/perf_event_paranoid ) +# Adjust perf_event_paranoid and kptr_restrict +PERF_EVENT_PARANOID=$(cat /proc/sys/kernel/perf_event_paranoid) echo -1 >/proc/sys/kernel/perf_event_paranoid -KPTR_RESTRICT=$( cat /proc/sys/kernel/kptr_restrict ) +KPTR_RESTRICT=$(cat /proc/sys/kernel/kptr_restrict) echo 0 >/proc/sys/kernel/kptr_restrict # Ensure settings are restored on exit trap restore_settings EXIT -# if pid is not zero, check if the process is running -if [ "$pid" -ne 0 ]; then - if ! ps -p "$pid" > /dev/null; then - echo "Error: Process $pid is not running." - exit 1 - fi +# Check if at least one process is running +if [ -n "$pids" ]; then + IFS=',' read -r -a pid_array <<< "$pids" + for p in "${pid_array[@]}"; do + if ps -p "$p" > /dev/null; then + if tr '\000' ' ' < /proc/"$p"/cmdline | grep -q java; then + java_pids+=("$p") + fi + else + echo "Error: Process $p is not running." >&2 + exit 1 + fi + done +else + mapfile -t java_pids < <(pgrep java) fi -# frame pointer mode -# if pid was provided, use it -if [ "$pid" -ne 0 ]; then - perf record -F "$frequency" -p "$pid" -g -o perf_fp_data -m 129 & +# Frame pointer mode +if [ -n "$pids" ]; then + perf record -F "$frequency" -p "$pids" -g -o perf_fp_data -m 129 & else - # if no pid was provided, use system-wide profiling perf record -F "$frequency" -a -g -o perf_fp_data -m 129 & fi perf_fp_pid=$! if ! kill -0 $perf_fp_pid 2>/dev/null; then - echo "Failed to start perf record in frame pointer mode" - exit 1 + echo "Failed to start perf record in frame pointer mode" >&2 + exit 1 fi -# dwarf mode -# if pid was provided, use it -if [ "$pid" -ne 0 ]; then - perf record -F "$frequency" -p "$pid" -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & +# Dwarf mode +if [ -n "$pids" ]; then + perf record -F "$frequency" -p "$pids" -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & else - # if no pid was provided, use system-wide profiling perf record -F "$frequency" -a -g -o perf_dwarf_data -m 257 --call-graph dwarf,8192 & fi perf_dwarf_pid=$! if ! kill -0 $perf_dwarf_pid 2>/dev/null; then - echo "Failed to start perf record in dwarf mode" - exit 1 + echo "Failed to start perf record in dwarf mode" >&2 + exit 1 fi -# wait for the specified duration +# Start Java profiling for each Java PID +for pid in "${java_pids[@]}"; do + java_cmds+=("$(tr '\000' ' ' < /proc/"$pid"/cmdline)") + async-profiler/profiler.sh start -i "$ap_interval" -o collapsed "$pid" +done + +# Wait for the specified duration sleep "$duration" -# stop perf recording +# Stop perf recording if ! kill -0 $perf_fp_pid 2>/dev/null; then - echo "Frame pointer mode already stopped" + echo "Frame pointer mode already stopped" >&2 else kill -INT $perf_fp_pid fi if ! kill -0 $perf_dwarf_pid 2>/dev/null; then - echo "Dwarf mode already stopped" + echo "Dwarf mode already stopped" >&2 else kill -INT $perf_dwarf_pid fi -# wait for perf to finish +# Stop Java profiling, write output to ap_folded_ files +for pid in "${java_pids[@]}"; do + async-profiler/profiler.sh stop -o collapsed -f ap_folded_"$pid" "$pid" +done + +# Wait for perf to finish wait ${perf_fp_pid} ${perf_dwarf_pid} -# collapse perf data -perf script -i perf_dwarf_data > perf_dwarf_stacks -stackcollapse-perf perf_dwarf_stacks > perf_dwarf_folded -perf script -i perf_fp_data > perf_fp_stacks -stackcollapse-perf perf_fp_stacks > perf_fp_folded +# Collapse perf data +if [ -f perf_dwarf_data ]; then + perf script -i perf_dwarf_data > perf_dwarf_stacks + stackcollapse-perf perf_dwarf_stacks > perf_dwarf_folded +else + echo "Error: perf_dwarf_data file not found" >&2 +fi +if [ -f perf_fp_data ]; then + perf script -i perf_fp_data > perf_fp_stacks + stackcollapse-perf perf_fp_stacks > perf_fp_folded +else + echo "Error: perf_fp_data file not found" >&2 +fi + +# Dump results to stdout +echo "########## maximum depth ##########" +echo "$maxdepth" -# Display results if [ -f perf_dwarf_folded ]; then - echo "########## perf_dwarf ##########" - cat perf_dwarf_folded + echo "########## perf_dwarf ##########" + cat perf_dwarf_folded fi if [ -f perf_fp_folded ]; then - echo "########## perf_fp ##########" - cat perf_fp_folded + echo "########## perf_fp ##########" + cat perf_fp_folded fi -echo "########## maximum depth ##########" -echo "$maxdepth" +for idx in "${!java_pids[@]}"; do + pid="${java_pids[$idx]}" + cmd="${java_cmds[$idx]}" + echo "########## async-profiler $pid $cmd ##########" + if [ -f ap_folded_"$pid" ]; then + cat ap_folded_"$pid" + else + echo "Error: async-profiler output file not found for PID $pid" >&2 + fi +done `, - Superuser: true, - Depends: []string{"perf", "stackcollapse-perf"}, + Superuser: true, + Sequential: true, + Depends: []string{"async-profiler", "perf", "stackcollapse-perf"}, }, // lock analysis scripts ProfileKernelLockScriptName: { diff --git a/internal/util/util.go b/internal/util/util.go index c2176719..d76f7df4 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -521,3 +521,12 @@ func SelectiveIntRangeToIntList(input string) ([]int, error) { } return result, nil } + +// IntSliceToStringSlice converts a slice of integers to a slice of strings. +func IntSliceToStringSlice(ints []int) []string { + strs := make([]string, len(ints)) + for i, v := range ints { + strs[i] = strconv.Itoa(v) + } + return strs +} diff --git a/internal/util/util_test.go b/internal/util/util_test.go index 17adc27e..7ce36a83 100644 --- a/internal/util/util_test.go +++ b/internal/util/util_test.go @@ -149,3 +149,22 @@ func TestSelectiveIntRangeToIntList(t *testing.T) { } } } +func TestIntSliceToStringSlice(t *testing.T) { + tests := []struct { + input []int + expected []string + }{ + {[]int{1, 2, 3}, []string{"1", "2", "3"}}, // Simple case + {[]int{-1, 0, 1}, []string{"-1", "0", "1"}}, // Negative, zero, and positive integers + {[]int{}, []string{}}, // Empty slice + {[]int{123, 456, 789}, []string{"123", "456", "789"}}, // Larger numbers + {[]int{-123, -456, -789}, []string{"-123", "-456", "-789"}}, // Negative larger numbers + } + + for _, test := range tests { + result := IntSliceToStringSlice(test.input) + if !slices.Equal(result, test.expected) { + t.Errorf("expected %v, got %v for input %v", test.expected, result, test.input) + } + } +} From 96adc0ef42e199ccf6def121b1ca5a48d0904d9b Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Fri, 9 May 2025 13:05:59 -0700 Subject: [PATCH 26/28] set cwd to temp dir in script Signed-off-by: Harper, Jason M --- internal/script/script.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/internal/script/script.go b/internal/script/script.go index 3957256c..c992611b 100644 --- a/internal/script/script.go +++ b/internal/script/script.go @@ -146,9 +146,9 @@ func RunScripts(myTarget target.Target, scripts []ScriptDefinition, ignoreScript return nil, err } else if script.Superuser && !myTarget.IsSuperUser() { // run script with sudo, "-S" to read password from stdin. Note: password won't be asked for if password-less sudo is configured. - cmd = exec.Command(fmt.Sprintf("cd %s && sudo -S bash %s", myTarget.GetTempDirectory(), scriptPath)) + cmd = exec.Command("sudo", "-S", "bash", scriptPath) } else { - cmd = exec.Command(fmt.Sprintf("cd %s && bash %s", myTarget.GetTempDirectory(), scriptPath)) + cmd = exec.Command("bash", scriptPath) } stdout, stderr, exitcode, err := myTarget.RunCommand(cmd, 0, false) if err != nil { @@ -468,11 +468,13 @@ func prepareTargetToRunScripts(myTarget target.Target, scripts []ScriptDefinitio for _, dependency := range script.Depends { dependenciesToCopy[path.Join(targetArchitecture, dependency)] = 1 } - // add user's path to script - scriptWithPath := fmt.Sprintf("export PATH=\"%s\"\n%s", userPath, script.ScriptTemplate) + // add cd command to the script to change to the target's local temp directory + targetScript := fmt.Sprintf("cd %s\n%s", targetTempDirectory, script.ScriptTemplate) + // add PATH (including the target temporary directory) to the script + targetScript = fmt.Sprintf("export PATH=\"%s\"\n%s", userPath, targetScript) // write script to the target's local temp directory scriptPath := path.Join(localTempDir, myTarget.GetName(), scriptNameToFilename(script.Name)) - err = os.WriteFile(scriptPath, []byte(scriptWithPath), 0644) + err = os.WriteFile(scriptPath, []byte(targetScript), 0644) if err != nil { err = fmt.Errorf("error writing script to local file: %v", err) return From 52c4a8fa94f697865f8147c1956d95c66183838d Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Sun, 11 May 2025 07:29:53 -0700 Subject: [PATCH 27/28] more refactoring Signed-off-by: Harper, Jason M --- Makefile | 21 +- .../stackcollapse-perf/stackcollapse-perf.go | 198 ++++++++++-------- .../stackcollapse-perf_test.go | 30 ++- 3 files changed, 159 insertions(+), 90 deletions(-) diff --git a/Makefile b/Makefile index 9e6105ff..51ce2e10 100644 --- a/Makefile +++ b/Makefile @@ -104,6 +104,25 @@ check_lint: go install github.com/golangci/golangci-lint/cmd/golangci-lint@latest golangci-lint run +.PHONY: check_vuln +check_vuln: + @echo "Running govulncheck to check for vulnerabilities..." + go install golang.org/x/vuln/cmd/govulncheck@latest + govulncheck ./... + +.PHONY: check_sec +check_sec: + @echo "Running gosec to check for security issues..." + go install github.com/securego/gosec/v2/cmd/gosec@latest + gosec ./... + +.PHONY: check_semgrep +check_semgrep: + @echo "Running semgrep to check for security issues..." + @echo "Please install semgrep from https://semgrep.dev/docs/getting-started/installation/ if not already installed." + @echo "Running semgrep..." + semgrep scan + .PHONY: check_modernize check_modernize: @echo "Running go-modernize to check for modernization opportunities..." @@ -115,7 +134,7 @@ modernize: go run golang.org/x/tools/gopls/internal/analysis/modernize/cmd/modernize@latest -fix -test ./... .PHONY: check -check: check_format check_vet check_static check_license check_lint check_modernize +check: check_format check_vet check_static check_license check_lint check_vuln check_modernize .PHONY: sweep sweep: diff --git a/tools/stackcollapse-perf/stackcollapse-perf.go b/tools/stackcollapse-perf/stackcollapse-perf.go index e0ef9b0b..e6bf29b8 100644 --- a/tools/stackcollapse-perf/stackcollapse-perf.go +++ b/tools/stackcollapse-perf/stackcollapse-perf.go @@ -9,6 +9,7 @@ package main import ( "bufio" + "flag" "fmt" "io" "os" @@ -54,34 +55,63 @@ func (sa *StackAggregator) RememberStack(stack string, count int) { } func main() { + var config Config + + flag.BoolVar(&config.AnnotateKernel, "kernel", false, "annotate kernel functions with a _[k]") + flag.BoolVar(&config.AnnotateJit, "jit", false, "annotate jit functions with a _[j]") + var annotateAll bool + flag.BoolVar(&annotateAll, "all", false, "all annotations (--kernel --jit)") + flag.BoolVar(&config.IncludePname, "pname", true, "include process names in stacks") + flag.BoolVar(&config.IncludePid, "pid", false, "include PID with process names") + flag.BoolVar(&config.IncludeTid, "tid", false, "include TID and PID with process names") + flag.BoolVar(&config.IncludeAddrs, "addrs", false, "include raw addresses where symbols can't be found") + flag.BoolVar(&config.TidyJava, "java", true, "condense Java signatures") + flag.BoolVar(&config.TidyGeneric, "generic", true, "clean up function names a little") + flag.StringVar(&config.EventFilter, "event-filter", "", "event name filter") + flag.BoolVar(&config.ShowInline, "inline", false, "un-inline using addr2line") + flag.BoolVar(&config.ShowContext, "context", false, "adds source context to --inline") + flag.BoolVar(&config.SrcLineInInput, "srcline", false, "parses output of 'perf script -F+srcline' and adds source context") + + flag.Usage = func() { + fmt.Fprintf(os.Stderr, "USAGE: %s [options] [infile] > outfile\n", os.Args[0]) + fmt.Fprintf(os.Stderr, "\nOptions:\n") + flag.PrintDefaults() + fmt.Fprintf(os.Stderr, "\n[1] perf script must emit both PID and TIDs for these to work; eg, Linux < 4.1:\n") + fmt.Fprintf(os.Stderr, " perf script -f comm,pid,tid,cpu,time,event,ip,sym,dso,trace\n") + fmt.Fprintf(os.Stderr, " for Linux >= 4.1:\n") + fmt.Fprintf(os.Stderr, " perf script -F comm,pid,tid,cpu,time,event,ip,sym,dso,trace\n") + fmt.Fprintf(os.Stderr, " If you save this output add --header on Linux >= 3.14 to include perf info.\n") + } + + flag.Parse() + + if annotateAll { + config.AnnotateKernel = true + config.AnnotateJit = true + } + + if config.ShowInline { + fmt.Fprintf(os.Stderr, "--inline is not implemented\n") + os.Exit(1) + } + if config.SrcLineInInput { + fmt.Fprintf(os.Stderr, "--srcline is not implemented\n") + os.Exit(1) + } + var input *os.File var err error - // Check if a file path is provided as a command-line argument - if len(os.Args) > 1 { - input, err = os.Open(os.Args[1]) // Open the file + args := flag.Args() + if len(args) > 0 { + input, err = os.Open(args[0]) if err != nil { fmt.Fprintf(os.Stderr, "Error opening file: %s\n", err) os.Exit(1) } defer input.Close() } else { - input = os.Stdin // Default to standard input - } - - var config = Config{ - AnnotateKernel: false, - AnnotateJit: false, - IncludePname: true, - IncludePid: false, - IncludeTid: false, - IncludeAddrs: false, - TidyJava: true, - TidyGeneric: true, - EventFilter: "", - ShowInline: false, - ShowContext: false, - SrcLineInInput: false, + input = os.Stdin } err = ProcessStacks(input, os.Stdout, config) @@ -93,39 +123,36 @@ func main() { // Regular expressions for parsing the perf output var ( - eventLineRegex = regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) - eventTypeRegex = regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) - stackLineRegex = regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) - // inlineRegex = regexp.MustCompile(`(perf-\d+.map|kernel\.|\[[^\]]+\])`) - stripSymbolsRegex = regexp.MustCompile(`\+0x[\da-f]+$`) - stripIdRegex = regexp.MustCompile(`\.\(.*\)\.`) - stripAnonymousRegex = regexp.MustCompile(`\([^a]*anonymous namespace[^)]*\)`) - jitRegex = regexp.MustCompile(`/tmp/perf-\d+\.map`) + eventLineRegex = regexp.MustCompile(`^(\S.+?)\s+(\d+)\/*(\d+)*\s+`) + eventTypeRegex = regexp.MustCompile(`:\s*(\d+)*\s+(\S+):\s*$`) + stackLineRegex = regexp.MustCompile(`^\s*(\w+)\s*(.+) \((.*)\)`) + stripSymbolOffsetRegex = regexp.MustCompile(`\+0x[\da-f]+$`) + goMethodRegex = regexp.MustCompile(`\.\(.*\)\.`) + jitRegex = regexp.MustCompile(`/tmp/perf-\d+\.map`) ) // ProcessStacks processes stack traces from the input reader and writes the collapsed stacks to the output writer. // It uses the provided configuration to control the processing behavior. func ProcessStacks(input io.Reader, output io.Writer, config Config) error { - aggregator := NewStackAggregator() - scanner := bufio.NewScanner(input) - var stack []string var processName string var period int + aggregator := NewStackAggregator() + scanner := bufio.NewScanner(input) // main loop, read lines from stdin for scanner.Scan() { line := scanner.Text() - + // skip comments if strings.HasPrefix(line, "#") { continue } - + // skip empty lines that are not after a stack if line == "" && processName == "" { continue } - - if line == "" { // check for End of stack + // check for end of stack + if line == "" { if config.IncludePname { stack = append([]string{processName}, stack...) } @@ -136,40 +163,46 @@ func ProcessStacks(input io.Reader, output io.Writer, config Config) error { processName = "" continue } - if err := handleEventRecord(line, &processName, &period, config); err != nil { - fmt.Fprintf(output, "Error: %s\n", err) + // check for event record + if eventLineRegex.MatchString(line) { + var err error + processName, period, err = handleEventRecord(line, config) + if err != nil { + fmt.Fprintf(output, "Error: %s\n", err) + } continue - } else if err := handleStackLine(line, &stack, processName, config); err != nil { - fmt.Fprintf(output, "Error: %s\n", err) + } + // check for stack line + if stackLineRegex.MatchString(line) { + err := handleStackLine(line, &stack, processName, config) + if err != nil { + fmt.Fprintf(output, "Error: %s\n", err) + } continue } } - // Check for errors during scanning if err := scanner.Err(); err != nil { fmt.Fprintf(os.Stderr, "Error reading input: %s\n", err) return err } - // Output results keys := make([]string, 0, len(aggregator.collapsed)) for k := range aggregator.collapsed { keys = append(keys, k) } sort.Strings(keys) - for _, k := range keys { fmt.Fprintf(output, "%s %d\n", k, aggregator.collapsed[k]) } - return nil } // handleEventRecord parses an event record line and updates the process name and period based on the configuration. -func handleEventRecord(line string, processName *string, period *int, config Config) error { +func handleEventRecord(line string, config Config) (processName string, period int, err error) { matches := eventLineRegex.FindStringSubmatch(line) if matches == nil { - return nil + return } comm, pid, tid := matches[1], matches[2], matches[3] @@ -181,32 +214,35 @@ func handleEventRecord(line string, processName *string, period *int, config Con if eventMatches := eventTypeRegex.FindStringSubmatch(line); eventMatches != nil { eventPeriod := eventMatches[1] if eventPeriod == "" { - *period = 1 + period = 1 } else { - eventPeriodInt, err := strconv.Atoi(eventPeriod) + var eventPeriodInt int + eventPeriodInt, err = strconv.Atoi(eventPeriod) if err != nil { - return fmt.Errorf("failed to parse event period: %s, error: %v", eventPeriod, err) + err = fmt.Errorf("failed to parse event period: %s, error: %v", eventPeriod, err) + return } - *period = eventPeriodInt + period = eventPeriodInt } event := eventMatches[2] if config.EventFilter == "" { config.EventFilter = event } else if event != config.EventFilter { - return fmt.Errorf("event type mismatch: %s != %s", event, config.EventFilter) + err = fmt.Errorf("event type mismatch: %s != %s", event, config.EventFilter) + return } } if config.IncludeTid { - *processName = fmt.Sprintf("%s-%s/%s", comm, pid, tid) + processName = fmt.Sprintf("%s-%s/%s", comm, pid, tid) } else if config.IncludePid { - *processName = fmt.Sprintf("%s-%s", comm, pid) + processName = fmt.Sprintf("%s-%s", comm, pid) } else { - *processName = comm + processName = comm } - *processName = strings.ReplaceAll(*processName, " ", "_") - return nil + processName = strings.ReplaceAll(processName, " ", "_") + return } // handleStackLine parses a stack line and appends the function name to the stack based on the configuration. @@ -218,19 +254,7 @@ func handleStackLine(line string, stack *[]string, pname string, config Config) pc, rawFunc, mod := matches[1], matches[2], matches[3] - // skip for now as showInline is always false - // if showInline && !inlineRegex.MatchString(mod) { - // inlineRes := inline(pc, rawFunc, mod) - // if inlineRes != "" && inlineRes != "??" && inlineRes != "??:??:0" { - // // prepend the inline result to the stack - // stack = append([]string{inlineRes}, stack...) - // continue - // } - //} - - // strip symbol offsets from rawFunc - // symbol offsets match this regex: \+0x[\da-f]+$ - rawFunc = stripSymbolsRegex.ReplaceAllString(rawFunc, "") + rawFunc = stripSymbolOffsetRegex.ReplaceAllString(rawFunc, "") // skip process names if strings.HasPrefix(rawFunc, "(") { @@ -244,7 +268,6 @@ func handleStackLine(line string, stack *[]string, pname string, config Config) // processFunctionName processes a raw function name, module, and program counter (PC) based on the configuration. // It returns a slice of processed function names. func processFunctionName(rawFunc, mod, pc string, config Config) []string { - // var isUnknown bool var inline []string for funcname := range strings.SplitSeq(rawFunc, "->") { if funcname == "[unknown]" { // use module name instead, if known @@ -252,7 +275,6 @@ func processFunctionName(rawFunc, mod, pc string, config Config) []string { funcname = filepath.Base(mod) } else { funcname = "unknown" - // isUnknown = true } if config.IncludeAddrs { @@ -263,18 +285,14 @@ func processFunctionName(rawFunc, mod, pc string, config Config) []string { } if config.TidyGeneric { funcname = strings.ReplaceAll(funcname, ";", ":") - if matches := stripIdRegex.FindStringSubmatch(funcname); matches != nil { - index := stripAnonymousRegex.FindStringIndex(funcname) - if index != nil { - funcname = funcname[0:index[0]] - } + if !goMethodRegex.MatchString(funcname) { + funcname = stripParenArgsUnlessAnonymous(funcname) } funcname = strings.ReplaceAll(funcname, "\"", "") funcname = strings.ReplaceAll(funcname, "'", "") } if config.TidyJava { if strings.Contains(funcname, "/") { - // strip the leading L funcname = strings.TrimPrefix(funcname, "L") } } @@ -282,21 +300,29 @@ func processFunctionName(rawFunc, mod, pc string, config Config) []string { if len(inline) > 0 { if !strings.Contains(funcname, "_[i]") { funcname = fmt.Sprintf("%s_[i]", funcname) - } else if config.AnnotateKernel && (strings.HasPrefix(funcname, "[") || strings.HasSuffix(funcname, "vmlinux")) && !strings.Contains(mod, "unknown") { - funcname = fmt.Sprintf("%s_[k]", funcname) - } else if config.AnnotateJit && jitRegex.MatchString(funcname) { - if !strings.Contains(funcname, "_[j]") { - funcname = fmt.Sprintf("%s_[j]", funcname) - } + } + } else if config.AnnotateKernel && (strings.HasPrefix(mod, "[") || strings.HasSuffix(mod, "vmlinux")) && !strings.Contains(mod, "unknown") { + funcname = fmt.Sprintf("%s_[k]", funcname) + } else if config.AnnotateJit && jitRegex.MatchString(mod) { + if !strings.Contains(funcname, "_[j]") { + funcname = fmt.Sprintf("%s_[j]", funcname) } } - // source lines - // skip for now since srcLineInInput is always false - // if srcLineInInput && !isUnknown { - // } - inline = append(inline, funcname) } return inline } + +// stripParenArgsUnlessAnonymous removes everything after the first '(' unless it is immediately followed by 'anonymous namespace'. +func stripParenArgsUnlessAnonymous(s string) string { + idx := strings.Index(s, "(") + if idx == -1 { + return s + } + // Check if what follows is 'anonymous namespace' + if strings.HasPrefix(s[idx:], "(anonymous namespace") { + return s + } + return s[:idx] +} diff --git a/tools/stackcollapse-perf/stackcollapse-perf_test.go b/tools/stackcollapse-perf/stackcollapse-perf_test.go index e77bc723..acc38ec5 100644 --- a/tools/stackcollapse-perf/stackcollapse-perf_test.go +++ b/tools/stackcollapse-perf/stackcollapse-perf_test.go @@ -84,7 +84,7 @@ func TestHandleEventRecord(t *testing.T) { TidyGeneric: true, } - err := handleEventRecord(line, &processName, &period, config) + processName, period, err := handleEventRecord(line, config) if err != nil { t.Fatalf("unexpected error: %v", err) } @@ -145,11 +145,11 @@ func TestProcessFunctionName(t *testing.T) { expected: []string{"com/example/MyClass"}, }, { - rawFunc: "someFunction;bar\"hello'world.(anonymous namespace).foo", + rawFunc: "someFunction;bar\"hello'world(remove me).foo", mod: "module.so", pc: "0x9abc", config: Config{TidyGeneric: true}, - expected: []string{"someFunction:barhelloworld."}, + expected: []string{"someFunction:barhelloworld"}, }, } @@ -166,3 +166,27 @@ func TestProcessFunctionName(t *testing.T) { } } } +func TestStripParenArgsUnlessAnonymous(t *testing.T) { + tests := []struct { + input string + expected string + }{ + {"foo(int, float)", "foo"}, + {"bar()", "bar"}, + {"baz", "baz"}, + {"qux(anonymous namespace)", "qux(anonymous namespace)"}, + {"func(anonymous namespace::Type)", "func(anonymous namespace::Type)"}, + {"func(anonymous namespace", "func(anonymous namespace"}, + {"func(int) (anonymous namespace)", "func"}, + {"func()", "func"}, + {"func", "func"}, + {"func(abc", "func"}, + } + + for _, test := range tests { + result := stripParenArgsUnlessAnonymous(test.input) + if result != test.expected { + t.Errorf("stripParenArgsUnlessAnonymous(%q) = %q; want %q", test.input, result, test.expected) + } + } +} From 9203d3f96403406dcf4bab7dc0ce91be7a34a8ed Mon Sep 17 00:00:00 2001 From: "Harper, Jason M" Date: Mon, 12 May 2025 15:03:31 -0700 Subject: [PATCH 28/28] replace log with slog Signed-off-by: Harper, Jason M --- internal/report/table_helpers_dimm.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/internal/report/table_helpers_dimm.go b/internal/report/table_helpers_dimm.go index 3e0191c1..e0ad06c5 100644 --- a/internal/report/table_helpers_dimm.go +++ b/internal/report/table_helpers_dimm.go @@ -5,7 +5,6 @@ package report import ( "fmt" - "log" "log/slog" "perfspect/internal/script" "regexp" @@ -67,7 +66,7 @@ func installedMemoryFromOutput(outputs map[string]script.ScriptOutput) string { if match != nil { size, err := strconv.Atoi(match[1]) if err != nil { - log.Printf("Don't recognize DIMM size format: %s", fields[1]) + slog.Error("Don't recognize DIMM size format.", slog.String("field", fields[1])) return "" } sum := count * size