From 4c13ab6a43b88f2313df1f7254c0a4e19666cfbf Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Fri, 21 Aug 2015 00:49:35 -0700 Subject: [PATCH 01/19] Add raw parser for parsing go tool pprof -raw --- main.go | 7 ++- raw.go | 173 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 raw.go diff --git a/main.go b/main.go index dc5a107..600defc 100644 --- a/main.go +++ b/main.go @@ -191,11 +191,12 @@ func (com *defaultCommander) goTorchCommand(c *cli.Context) { if err != nil { log.Fatal(err) } - flamegraphInput, err := com.grapher.GraphAsText(out) + + flamegraphInputBytes, err := newRawParser().Parse(out) if err != nil { log.Fatal(err) } - flamegraphInput = strings.TrimSpace(flamegraphInput) + flamegraphInput := strings.TrimSpace(string(flamegraphInputBytes)) if raw { fmt.Println(flamegraphInput) log.Info("raw call graph output been printed to stdout") @@ -209,7 +210,7 @@ func (com *defaultCommander) goTorchCommand(c *cli.Context) { // runPprofCommand runs the `go tool pprof` command to profile an application. // It returns the output of the underlying command. func (p *defaultPprofer) runPprofCommand(args ...string) ([]byte, error) { - allArgs := []string{"tool", "pprof", "-dot", "-lines"} + allArgs := []string{"tool", "pprof", "-raw"} allArgs = append(allArgs, args...) var buf bytes.Buffer diff --git a/raw.go b/raw.go new file mode 100644 index 0000000..2d3ebd4 --- /dev/null +++ b/raw.go @@ -0,0 +1,173 @@ +package main + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os/exec" + "strconv" + "strings" + "time" +) + +type readMode int + +const ( + ignore readMode = iota + samplesHeader + samples + locations + mappings +) + +type funcID int + +type rawParser struct { + funcName map[funcID]string + records []*stackRecord +} + +func newRawParser() *rawParser { + return &rawParser{ + funcName: make(map[funcID]string), + } +} + +func (p *rawParser) Parse(input []byte) ([]byte, error) { + var mode readMode + reader := bufio.NewReader(bytes.NewReader(input)) + + for { + line, err := reader.ReadString('\n') + line = strings.TrimSpace(line) + if err != nil { + if err == io.EOF { + break + } + return nil, err + } + + switch mode { + case ignore: + if strings.HasPrefix(line, "Samples") { + mode = samplesHeader + continue + } + case samplesHeader: + mode = samples + case samples: + if strings.HasPrefix(line, "Locations") { + mode = locations + continue + } + p.addSample(line) + case locations: + if strings.HasPrefix(line, "Mappings") { + mode = mappings + continue + } + p.addLocation(line) + case mappings: + // Nothing to process. + } + } + + pr, pw := io.Pipe() + go p.Print(pw) + return p.CollapseStacks(pr) +} + +func (p *rawParser) Print(w io.WriteCloser) { + for _, r := range p.records { + r.Serialize(p.funcName, w) + fmt.Fprintln(w) + } + w.Close() +} + +func findStackCollapse() string { + for _, v := range []string{"stackcollapse.pl", "./stackcollapse.pl", "./FlameGraph/stackcollapse.pl"} { + if path, err := exec.LookPath(v); err == nil { + return path + } + } + return "" +} + +func (p *rawParser) CollapseStacks(stacks io.Reader) ([]byte, error) { + stackCollapse := findStackCollapse() + if stackCollapse == "" { + return nil, errors.New("stackcollapse.pl not found") + } + + cmd := exec.Command(stackCollapse) + cmd.Stdin = stacks + return cmd.Output() +} + +func (p *rawParser) addSample(line string) { + // Parse a sample which looks like: + // 1 10000000: 1 2 3 4 + parts := splitIgnoreEmpty(line, " ") + + samples, err := strconv.Atoi(parts[0]) + if err != nil { + panic(err) + } + + duration, err := strconv.Atoi(strings.TrimSuffix(parts[1], ":")) + if err != nil { + panic(err) + } + + var stack []funcID + for _, fIDStr := range parts[2:] { + stack = append(stack, toFuncID(fIDStr)) + } + + p.records = append(p.records, &stackRecord{samples, time.Duration(duration), stack}) +} + +func (p *rawParser) addLocation(line string) { + // 292: 0x49dee1 github.com/uber/tchannel/golang.(*Frame).ReadIn :0 s=0 + parts := splitIgnoreEmpty(line, " ") + funcID := toFuncID(strings.TrimSuffix(parts[0], ":")) + p.funcName[funcID] = parts[2] +} + +type stackRecord struct { + samples int + duration time.Duration + stack []funcID +} + +func (r *stackRecord) Serialize(funcName map[funcID]string, w io.Writer) { + // Go backwards through the stack + for _, funcID := range r.stack { + fmt.Fprintln(w, funcName[funcID]) + } + fmt.Fprintln(w, r.samples) +} + +func toFuncID(s string) funcID { + i, err := strconv.Atoi(s) + if err != nil { + panic(err) + } + return funcID(i) +} + +// splitIgnoreEmpty does a strings.Split and then removes all empty strings. +func splitIgnoreEmpty(s string, splitter string) []string { + vals := strings.Split(s, splitter) + var res []string + for _, v := range vals { + if len(v) != 0 { + res = append(res, v) + } + } + + return res +} From 08b04a7768b34a3230b3dd46b98c1cc6f51ce44d Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 14:51:10 -0700 Subject: [PATCH 02/19] Add pprof package for getting output from pprof Add simple runner which runs pprof given a set of Options. --- pprof/pprof.go | 90 ++++++++++++++++++++++++ pprof/pprof_test.go | 128 +++++++++++++++++++++++++++++++++++ pprof/testdata/pprof.1.pb.gz | Bin 0 -> 3068 bytes 3 files changed, 218 insertions(+) create mode 100644 pprof/pprof.go create mode 100644 pprof/pprof_test.go create mode 100644 pprof/testdata/pprof.1.pb.gz diff --git a/pprof/pprof.go b/pprof/pprof.go new file mode 100644 index 0000000..86c6b7d --- /dev/null +++ b/pprof/pprof.go @@ -0,0 +1,90 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package pprof + +import ( + "bytes" + "fmt" + "net/url" + "os/exec" +) + +// Options are parameters for pprof. +type Options struct { + BaseURL string `short:"u" long:"url" default:"http://localhost:8080" description:"Base URL of your Go program"` + URLSuffix string `short:"s" long:"suffix" default:"/debug/pprof/profile" description:"URL path of pprof profile"` + BinaryFile string `short:"b" long:"binaryinput" description:"file path of previously saved binary profile. (binary profile is anything accepted by https://golang.org/cmd/pprof)"` + BinaryName string `long:"binaryname" description:"file path of the binary that the binaryinput is for, used for pprof inputs"` + TimeSeconds int `short:"t" long:"time" default:"30" description:"Duration to profile for"` +} + +// GetRaw returns the raw output from pprof for the given options. +func GetRaw(opts Options) ([]byte, error) { + args, err := getArgs(opts) + if err != nil { + return nil, err + } + + return runPProf(args...) +} + +// getArgs gets the arguments to run pprof with for a given set of Options. +func getArgs(opts Options) ([]string, error) { + var pprofArgs []string + if opts.BinaryFile != "" { + if opts.BinaryName != "" { + pprofArgs = append(pprofArgs, opts.BinaryName) + } + pprofArgs = append(pprofArgs, opts.BinaryFile) + } else { + u, err := url.Parse(opts.BaseURL) + if err != nil { + return nil, fmt.Errorf("failed to parse URL: %v", err) + } + + u.Path = opts.URLSuffix + pprofArgs = append(pprofArgs, "-seconds", fmt.Sprint(opts.TimeSeconds), u.String()) + } + + return pprofArgs, nil +} + +func runPProf(args ...string) ([]byte, error) { + allArgs := []string{"tool", "pprof", "-raw"} + allArgs = append(allArgs, args...) + + var buf bytes.Buffer + cmd := exec.Command("go", allArgs...) + cmd.Stderr = &buf + out, err := cmd.Output() + if err != nil { + return nil, fmt.Errorf("pprof error: %v\nSTDERR:\n%s", err, buf.Bytes()) + } + + // @HACK because 'go tool pprof' doesn't exit on errors with nonzero status codes. + // Ironically, this means that Go's own os/exec package does not detect its errors. + // See issue here https://github.com/golang/go/issues/11510 + if len(out) == 0 { + return nil, fmt.Errorf("pprof error:\n%s", buf.Bytes()) + } + + return out, nil +} diff --git a/pprof/pprof_test.go b/pprof/pprof_test.go new file mode 100644 index 0000000..bebe1bf --- /dev/null +++ b/pprof/pprof_test.go @@ -0,0 +1,128 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package pprof + +import ( + "bytes" + "reflect" + "testing" +) + +func TestGetArgs(t *testing.T) { + tests := []struct { + opts Options + expected []string + }{ + { + opts: Options{ + BaseURL: "http://localhost:1234", + URLSuffix: "/path/to/profile", + TimeSeconds: 5, + }, + expected: []string{"-seconds", "5", "http://localhost:1234/path/to/profile"}, + }, + { + opts: Options{ + BaseURL: "http://localhost:1234/", + URLSuffix: "/path/to/profile", + TimeSeconds: 5, + }, + expected: []string{"-seconds", "5", "http://localhost:1234/path/to/profile"}, + }, + { + opts: Options{ + BaseURL: "http://localhost:1234/test", + URLSuffix: "/path/to/profile", + TimeSeconds: 5, + }, + expected: []string{"-seconds", "5", "http://localhost:1234/path/to/profile"}, + }, + { + opts: Options{ + BinaryFile: "/path/to/binaryfile", + BaseURL: "http://localhost:1234", + URLSuffix: "/profile", + TimeSeconds: 5}, + expected: []string{"/path/to/binaryfile"}, + }, + { + opts: Options{ + BinaryFile: "/path/to/binaryfile", + BinaryName: "/path/to/binaryname", + BaseURL: "http://localhost:1234", + URLSuffix: "/profile", + TimeSeconds: 5}, + expected: []string{"/path/to/binaryname", "/path/to/binaryfile"}, + }, + } + + for _, tt := range tests { + got, err := getArgs(tt.opts) + if err != nil { + t.Errorf("failed to get pprof args: %v", err) + continue + } + + if !reflect.DeepEqual(tt.expected, got) { + t.Errorf("got incorrect args for %v:\n got %v\n want %v", tt.opts, got, tt.expected) + } + } +} + +func TestRunPProfUnknownFlag(t *testing.T) { + if _, err := runPProf("-unknownFlag"); err == nil { + t.Fatalf("expected error for unknown flag") + } +} + +func TestRunPProfMissingFile(t *testing.T) { + if _, err := runPProf("unknown-file"); err == nil { + t.Fatalf("expected error for unknown file") + } +} + +func TestRunPProfInvalidURL(t *testing.T) { + if _, err := runPProf("http://127.0.0.1:999/profile"); err == nil { + t.Fatalf("expected error for unknown file") + } +} + +func TestGetPProfRaw(t *testing.T) { + opts := Options{ + BinaryFile: "testdata/pprof.1.pb.gz", + } + raw, err := GetRaw(opts) + if err != nil { + t.Fatalf("getPProfRaw failed: %v", err) + } + + expectedSubstrings := []string{ + "Duration: 3s", + "Samples", + "Locations", + "main.fib", + } + for _, substr := range expectedSubstrings { + if !bytes.Contains(raw, []byte(substr)) { + t.Errorf("pprof raw output missing expected string: %s\ngot:\n%s", substr, raw) + } + } +} diff --git a/pprof/testdata/pprof.1.pb.gz b/pprof/testdata/pprof.1.pb.gz new file mode 100644 index 0000000000000000000000000000000000000000..2e6f1178ad6a9db5447ec15b9d0afc49911d86dd GIT binary patch literal 3068 zcmVpkIdx1a8WKakubL9 z&=yV82_YLdZQ1=Y-K4v^sOlyzvKSXGl7*;?xEX{%HUSeiiW?CX1VzM!8#gKlqM|sU z5HW&ytGZv+y}$E2uR@W2r{2f8=lss^{O*0P-lV!d=mc&x93(e z*_!FG^S=vr(JrN>^!|H)SXkY*<>|@E^rG-`JQiCfcK5Qi%H_IC_5rsWwh|;}Ahks7 zBQ83&N)io|MYM>MJ05TyiO}UNQb?o}9S;8@@UsD+VpWo_EWK7aflCdNF~o|8A-L3o z@g))nvT(39Nh+>MTud!F!`M=zmEhp!aRbN=0-cP9L2Q`2CIDS3Wnosu;aQN3(eENdQK@S=$!a=UXeWz8>j9J8vZ z8xs=5_()MkdIDlKkt8z72wS+p*%-%{DvD?VQZ-Q}_FWsY04d3Lrm#euJ9d>~B3&6B z?rkD3oc9bh;;t%~ZUAHx)e;E-WN2EZ19GW`;6wEqw2@f^Y!ySQSDSM=DwarX+pbv@ zJkBYEox1DVb<0{5+eEr0uC#@$oWP{x$kns9C9psRpMe1Dm`gN!jh@&6;B3Xp*_IPi zehlJMvHE(GRZWen8wfCVd`JTxDYz<;sI-KTcaf{B_Of;M5&Nip%sy_f*eC2&`=ou! zUQ6rg{Ra=8^#Tu=n_inZ7fL>N0xtw#pI1bxRG;QZ3T|q$&-ZyppF%cV<@ygSJ+t5(X~b+ zSR_?MUr3j7UVwO-YIAA5GK(HuEk{-3GKwWItd7t7$<>gu<7KmB@T8$7M#LBpBXH1! z8e!m)oMIIvYopu5MK>s~%pKRmm?T}1GhVRfB|Pa&6VmF!TZ687bzDNyht9xF#iBe& zIb)4MZ(t?u<0T=kd`UF2i+NYa5l3!59<95vcMKo9D^f6`zzC&Mn{J@Syv3xT&-Q_^ zSH$?d^%7F4{0#Nk1;5My_3(_3_~Z#Ci$`gg(;YwC8gRn;99ufv(tA6`9*hyyfX)~@ z1&cIfy_PySd|GiP}ECBh<; zjz7|K8OAtSapI6I2N)^1TEprFe;Lr1jmpZiMlgtL#*m~>v}GJ&*-cbQ?tXA67zFD#Gf{lr_43&TZVzG!3avF}U$jx(vZ4(4~bYbT@z$FO~fjZXu--p|)&R zYyv263o=Hsu~`qOURKAKN=PZzw24SXDq%sl+$DD-vhCP{4csh4B>Gex_cORDTTb($ zZzP%)NdS>zEeLKbse(*JR!#kVnyVROaKR5+^GmD>_1!6JB^EqE+Rxp|L*lp*i=?T5 zIo*p@8fuHA9CJ%t$!1{PB})OWZd#OpgvNt03s)~rUkqSa$nR1W1!gY=E@M=%6kOCO zdfQ}Cz};afXxiKMx*#Pl3_7MX*|~W6MJ6E|Ii{1++~d~;WGBjmF-8TwEt`_j6U+LO z4Fj>Q6J!uaBizp{@hlkW%P7IRl@?>`;(0X!jpWdVg6QgWL|$oGEK-UpIh_$%VoOqO z*~ChjW|775)4GPwS6M2OEb$N!TazM#QLritXPw}^F4STTJK-dLIgBSVr_nbibciuA z0I`gQOt`!Y4Jo?fq)f`Gsl;p_eL07xgBTGkDlkG^Ayz4|eAnK0(CBvVmOL7j@JUzL z3gD(LP>>zO8dh<`c#6(LO``W_C|yPr*=Uh;rt8c^;j83_aB^qL5m^z&UBZ&vbJ!u- zt3I^WFX4rrbAKiiF~TWz!^*1hRZ)_dwMA*6P>_393_pqB=1UxLGam9W7NNK~Lx;eX zRGen46o)V-NTw<>$EXrukkW#jo2Yfjs)DsY{$*uXtJIAi_DU?U{P8DpwW&M|#MX>E zp2+5A^y5p+87WQ$inkfO`OVy+c!qoeM=i@J&k{`_M#YhA(HIpOv2esnj;>e2=%5mzXeeF3GrjlMY9!IZ_{q(RChJ%|Sq5-+LmAUpY!}vp z(8jkc3M!+3oE!v2w}Sbd^U07BC`q2jg=vM~w5UZ7cBY~yJ3mAWZHVW##YqR*E|aHI zLTCD}XDI(jNLf+xPJ~g)jq!+D-64;q48()SBL|;?Ukz|ftPr_4Jf(<&LU*ds7@SOr zl|3%x!m4aYfD;B1qq@T`V12GKHFm@lhwCH`y}(+=Y=>!9#EnNhU`jJ8~$sg+b*B->39M%OJ$mDCt6o&~E8iHl^I z%Iqe@gC&=fpTg`iDLrzVu}DNzC;f!NB3p!{RwqD)U51ydsofSynCLR2R8OrMmk`M& zl}G&F2 zn1ix9l$66NF)@a^RV}tAlR4}-8dt+=(#km&*M3M_pV6|d(q&UrM4n6IHStW*@>1aq0h^a*?R@`d} z4YuI7SOcI-*^LoZrH(g^A&gfET7}E!DN~}w)s6ne2IH5piNH`%^Z5ND=}fL%3c$xT zom7W}5I8?do#Ev&O|GrNnMOyMxI`ATU3J1Zg_woTu2pQwJ^b`FV@uvN?yZdr zGjSj0dl%eyKjTevt!fwDcYav&cH-?N_qFd`_V(CE*?lFy(7K=J7gpSlatb-p$R1`%QjA)4h}LU3PEhX6n9~n=9_k++214$S+%Sf6dKx_v_qjxd-{Vm)tkb z&8ofZe(~FJ?2F!Z?&0^V-d^(dBktXAjqS3xA9e5lKJm8o_G9jipVYlw@%H2H&p#X6 zRc~K$5C1r}Yul$l`T}Z97e!G}fT*Ysf(yFWaZ8@#En%~;A?&^M9NgJ-=x7D=intoeLm))}8*3;Cb ze%nY_+=|~e(^a?Xx6A38Tl3pAU3cs4|7({=z2m`AHfk^Kot*Vg+sWSGtX=E(`X{4o z@1(y!YTM)9VgF;V9DcD~nONgL?fTH4e0ZFle{1I7?ww>`Iy`M(nH}9b*d1lZy}`lB zFxwp+-s|^{+L!V0Fgwb6qpZD}9~fWqTJP2DY%tr`KRL}V_D#ndo%V*O$L&V`eSCb- zZslKx*(f_54G#PL!~VT?I=yP|AlpAX%GztQxF`F%9*S`1V3=jcgVWvP*|q)bH9zRy ztL>F(5 Date: Thu, 10 Sep 2015 14:55:05 -0700 Subject: [PATCH 03/19] Add pprof raw output parser --- pprof/raw.go | 218 ++++++++++++++++++++++++++ pprof/raw_test.go | 92 +++++++++++ pprof/testdata/pprof.raw.txt | 291 +++++++++++++++++++++++++++++++++++ 3 files changed, 601 insertions(+) create mode 100644 pprof/raw.go create mode 100644 pprof/raw_test.go create mode 100644 pprof/testdata/pprof.raw.txt diff --git a/pprof/raw.go b/pprof/raw.go new file mode 100644 index 0000000..5dbb60b --- /dev/null +++ b/pprof/raw.go @@ -0,0 +1,218 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package pprof + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" +) + +type readMode int + +const ( + ignore readMode = iota + samplesHeader + samples + locations + mappings +) + +// funcID is the ID of a given Location in the pprof raw output. +type funcID int + +type rawParser struct { + // err is the first error encountered by the parser. + err error + + mode readMode + funcName map[funcID]string + records []*stackRecord +} + +// ParseRaw parses the raw pprof output and returns call stacks. +func ParseRaw(input []byte) ([]byte, error) { + parser := newRawParser() + if err := parser.parse(input); err != nil { + return nil, err + } + + pr, pw := io.Pipe() + go parser.print(pw) + return CollapseStacks(pr) +} + +func newRawParser() *rawParser { + return &rawParser{ + funcName: make(map[funcID]string), + } +} + +func (p *rawParser) parse(input []byte) error { + reader := bufio.NewReader(bytes.NewReader(input)) + + for { + line, err := reader.ReadString('\n') + line = strings.TrimSpace(line) + if err != nil { + if err == io.EOF { + break + } + return err + } + + p.processLine(line) + } + + return p.err +} + +func (p *rawParser) processLine(line string) { + switch p.mode { + case ignore: + if strings.HasPrefix(line, "Samples") { + p.mode = samplesHeader + return + } + case samplesHeader: + p.mode = samples + case samples: + if strings.HasPrefix(line, "Locations") { + p.mode = locations + return + } + p.addSample(line) + case locations: + if strings.HasPrefix(line, "Mappings") { + p.mode = mappings + return + } + p.addLocation(line) + case mappings: + // Nothing to process. + } +} + +// print prints out the stack traces collected from the raw pprof output. +func (p *rawParser) print(w io.WriteCloser) { + for _, r := range p.records { + r.Serialize(p.funcName, w) + fmt.Fprintln(w) + } + w.Close() +} + +func findStackCollapse() string { + for _, v := range []string{"stackcollapse.pl", "./stackcollapse.pl", "./FlameGraph/stackcollapse.pl"} { + if path, err := exec.LookPath(v); err == nil { + return path + } + } + return "" +} + +// CollapseStacks runs the flamegraph's collapse stacks script. +func CollapseStacks(stacks io.Reader) ([]byte, error) { + stackCollapse := findStackCollapse() + if stackCollapse == "" { + return nil, errors.New("stackcollapse.pl not found") + } + + cmd := exec.Command(stackCollapse) + cmd.Stdin = stacks + cmd.Stderr = os.Stderr + return cmd.Output() +} + +// addSample parses a sample that looks like: +// 1 10000000: 1 2 3 4 +// and creates a stackRecord for it. +func (p *rawParser) addSample(line string) { + // Parse a sample which looks like: + parts := splitBySpace(line) + + samples, err := strconv.Atoi(parts[0]) + if err != nil { + p.err = err + return + } + + duration, err := strconv.Atoi(strings.TrimSuffix(parts[1], ":")) + if err != nil { + p.err = err + return + } + + var stack []funcID + for _, fIDStr := range parts[2:] { + stack = append(stack, p.toFuncID(fIDStr)) + } + + p.records = append(p.records, &stackRecord{samples, time.Duration(duration), stack}) +} + +// addLocation parses a location that looks like: +// 292: 0x49dee1 github.com/uber/tchannel/golang.(*Frame).ReadIn :0 s=0 +// and creates a mapping from funcID to function name. +func (p *rawParser) addLocation(line string) { + parts := splitBySpace(line) + funcID := p.toFuncID(strings.TrimSuffix(parts[0], ":")) + p.funcName[funcID] = parts[2] +} + +type stackRecord struct { + samples int + duration time.Duration + stack []funcID +} + +// Serialize serializes a call stack for a given stackRecord given the funcID mapping. +func (r *stackRecord) Serialize(funcName map[funcID]string, w io.Writer) { + for _, funcID := range r.stack { + fmt.Fprintln(w, funcName[funcID]) + } + fmt.Fprintln(w, r.samples) +} + +// toFuncID converts a string like "8" to a funcID. +func (p *rawParser) toFuncID(s string) funcID { + i, err := strconv.Atoi(s) + if err != nil { + p.err = fmt.Errorf("failed to parse funcID: %v", err) + return 0 + } + return funcID(i) +} + +var spaceSplitter = regexp.MustCompile(`\s+`) + +// splitBySpace splits values separated by 1 or more spaces. +func splitBySpace(s string) []string { + return spaceSplitter.Split(strings.TrimSpace(s), -1) +} diff --git a/pprof/raw_test.go b/pprof/raw_test.go new file mode 100644 index 0000000..a447a7a --- /dev/null +++ b/pprof/raw_test.go @@ -0,0 +1,92 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package pprof + +import ( + "io/ioutil" + "reflect" + "testing" + "time" +) + +func TestParse(t *testing.T) { + rawBytes, err := ioutil.ReadFile("testdata/pprof.raw.txt") + if err != nil { + t.Fatalf("Failed to read testdata/pprof.raw.txt: %v", err) + } + + parser := newRawParser() + if err := parser.parse(rawBytes); err != nil { + t.Fatalf("Parse failed: %v", err) + } + + // line 7 - 249 are stack records in the test file. + const expectedNumRecords = 242 + if len(parser.records) != expectedNumRecords { + t.Errorf("Failed to parse all records, got %v records, expected %v", + len(parser.records), expectedNumRecords) + } + expectedRecords := map[int]*stackRecord{ + 0: &stackRecord{1, time.Duration(10000000), []funcID{1, 2, 2, 2, 3, 3, 2, 2, 3, 3, 2, 2, 2, 3, 3, 3, 2, 3, 2, 3, 2, 2, 3, 2, 2, 3, 4, 5, 6}}, + 18: &stackRecord{1, time.Duration(10000000), []funcID{14, 2, 2, 3, 2, 2, 3, 2, 2, 3, 3, 3, 2, 2, 2, 3, 3, 2, 3, 3, 3, 3, 3, 2, 4, 5, 6}}, + 45: &stackRecord{12, time.Duration(120000000), []funcID{23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34}}, + } + for recordNum, expected := range expectedRecords { + if got := parser.records[recordNum]; !reflect.DeepEqual(got, expected) { + t.Errorf("Unexpected record for %v:\n got %#v\n want %#v", recordNum, got, expected) + } + } + + // line 250 - 290 are locations (or funcID mappings) + const expectedFuncIDs = 41 + if len(parser.funcName) != expectedFuncIDs { + t.Errorf("Failed to parse func ID mappings, got %v records, expected %v", + len(parser.funcName), expectedFuncIDs) + } + knownMappings := map[funcID]string{ + 1: "main.fib", + 20: "main.fib", + 34: "runtime.morestack", + } + for funcID, expected := range knownMappings { + if got := parser.funcName[funcID]; got != expected { + t.Errorf("Unexpected mapping for %v: got %v, want %v", funcID, got, expected) + } + } +} + +func TestSplitBySpace(t *testing.T) { + tests := []struct { + s string + expected []string + }{ + {"", []string{""}}, + {"test", []string{"test"}}, + {"1 2", []string{"1", "2"}}, + {"1 2 3 4 ", []string{"1", "2", "3", "4"}}, + } + + for _, tt := range tests { + if got := splitBySpace(tt.s); !reflect.DeepEqual(got, tt.expected) { + t.Errorf("splitBySpace(%v) failed:\n got %#v\n want %#v", tt.s, got, tt.expected) + } + } +} diff --git a/pprof/testdata/pprof.raw.txt b/pprof/testdata/pprof.raw.txt new file mode 100644 index 0000000..5170190 --- /dev/null +++ b/pprof/testdata/pprof.raw.txt @@ -0,0 +1,291 @@ +PeriodType: cpu nanoseconds +Period: 10000000 +Time: 2015-09-10 13:53:30.696637683 -0700 PDT +Duration: 3s +Samples: +samples/count cpu/nanoseconds + 1 10000000: 1 2 2 2 3 3 2 2 3 3 2 2 2 3 3 3 2 3 2 3 2 2 3 2 2 3 4 5 6 + 1 10000000: 7 2 3 3 3 3 3 3 3 2 2 3 3 3 2 3 3 3 3 3 3 3 3 2 3 3 3 3 2 3 3 2 4 5 6 + 1 10000000: 8 2 2 3 3 3 2 3 3 3 3 3 2 3 3 3 2 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 2 4 5 6 + 1 10000000: 9 3 2 2 2 3 2 3 3 2 3 2 3 2 3 3 2 3 3 2 3 2 3 3 3 2 4 5 6 + 1 10000000: 10 2 3 3 3 3 3 3 3 2 3 2 2 3 3 3 3 2 3 2 3 2 2 3 3 3 2 4 5 6 + 1 10000000: 1 3 3 3 3 2 3 3 3 3 2 3 3 2 3 3 2 3 2 2 2 2 3 2 3 4 5 6 + 1 10000000: 1 2 2 2 2 3 2 2 3 2 2 3 2 3 2 2 3 3 3 2 3 2 3 3 3 3 4 5 6 + 1 10000000: 10 3 2 3 3 2 3 2 3 3 2 3 3 2 2 2 3 3 3 3 3 3 3 3 3 3 3 2 3 3 3 2 4 5 6 + 1 10000000: 11 3 2 3 3 3 3 2 2 3 3 3 2 2 3 3 3 2 3 2 3 3 2 3 3 3 2 4 5 6 + 1 10000000: 12 3 3 2 2 2 2 3 3 2 3 2 2 2 2 2 2 3 3 3 3 2 3 3 2 4 5 6 + 1 10000000: 10 3 3 3 2 3 2 2 3 2 3 2 3 3 3 2 3 2 2 2 3 3 3 3 3 3 3 3 2 3 2 4 5 6 + 1 10000000: 11 3 2 3 2 2 2 2 3 2 3 2 3 3 3 2 3 2 3 3 3 3 3 3 2 2 4 5 6 + 1 10000000: 13 3 3 3 3 3 2 3 3 3 3 2 3 3 3 2 3 2 2 3 3 3 3 3 3 3 2 3 3 2 2 2 4 5 6 + 1 10000000: 14 3 3 3 3 2 2 3 2 2 3 3 3 3 2 2 3 3 3 3 3 3 3 3 3 3 2 2 3 3 2 3 4 5 6 + 1 10000000: 11 3 3 3 3 3 3 2 3 2 3 3 2 2 3 3 3 3 3 3 3 3 3 3 3 2 3 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 15 3 2 3 3 3 3 3 3 3 3 3 2 2 3 3 3 2 3 3 2 2 2 2 3 3 3 3 3 3 2 3 4 5 6 + 1 10000000: 13 3 3 3 3 3 3 3 3 3 3 3 2 3 2 3 3 3 3 2 2 3 3 3 3 2 3 3 3 2 3 3 3 3 3 4 5 6 + 1 10000000: 14 2 2 3 2 3 3 3 3 3 3 3 3 3 3 3 3 2 3 3 2 2 3 2 3 3 3 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 14 2 2 3 2 2 3 2 2 3 3 3 2 2 2 3 3 2 3 3 3 3 3 2 4 5 6 + 1 10000000: 16 3 2 3 2 2 3 2 3 2 3 3 2 3 2 2 3 3 3 2 2 3 3 2 2 2 2 4 5 6 + 1 10000000: 1 2 3 2 3 2 3 3 3 2 3 3 2 2 3 3 2 2 3 2 2 3 3 3 2 2 4 5 6 + 1 10000000: 14 3 3 3 2 2 3 2 3 3 3 3 3 3 2 3 3 3 3 3 3 3 2 2 3 3 2 3 3 3 3 3 4 5 6 + 1 10000000: 13 3 3 2 2 2 3 3 3 3 3 2 3 2 3 2 2 2 3 3 3 3 2 2 3 3 2 2 4 5 6 + 1 10000000: 17 3 2 3 3 3 3 2 3 3 3 3 3 3 3 2 3 3 3 2 3 3 2 3 3 3 3 3 3 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 7 2 3 2 3 3 2 2 3 2 3 3 3 3 3 3 3 3 3 3 3 3 2 3 2 3 2 3 2 3 3 3 3 4 5 6 + 1 10000000: 14 2 3 3 3 3 3 2 2 2 2 3 2 3 2 2 3 3 2 3 3 3 2 3 3 3 4 5 6 + 1 10000000: 11 3 3 3 2 3 3 3 3 2 3 3 3 2 2 2 3 2 3 2 2 2 3 2 3 2 3 3 4 5 6 + 1 10000000: 7 2 3 3 3 3 3 2 3 2 2 3 3 2 3 2 3 3 3 3 2 3 3 2 3 3 3 3 3 2 3 3 4 5 6 + 1 10000000: 11 3 3 3 3 2 2 3 3 3 3 3 3 2 2 3 3 3 3 2 3 3 3 3 3 2 3 2 2 3 3 3 3 4 5 6 + 1 10000000: 14 2 3 3 2 2 3 2 3 2 3 3 2 2 3 2 3 3 3 2 3 2 3 2 3 2 4 5 6 + 1 10000000: 18 3 2 2 2 3 2 3 2 2 2 2 3 2 3 3 2 2 3 2 2 3 3 2 3 3 3 4 5 6 + 1 10000000: 13 3 3 3 3 3 3 3 3 2 3 2 3 3 3 3 3 3 2 3 2 3 2 3 3 3 3 2 3 3 3 3 4 5 6 + 1 10000000: 10 2 2 3 3 3 3 3 2 2 3 2 3 3 2 3 3 2 3 3 2 2 2 3 3 3 2 3 3 3 2 3 4 5 6 + 1 10000000: 14 3 2 3 3 2 3 2 2 3 3 2 2 2 3 3 2 2 3 3 3 3 2 3 3 3 3 3 4 5 6 + 1 10000000: 13 2 3 3 3 3 3 3 2 3 3 2 2 3 3 3 2 3 3 3 2 2 2 3 3 3 3 2 3 2 3 3 3 4 5 6 + 1 10000000: 11 3 2 2 3 3 3 2 3 3 3 2 2 2 3 3 2 2 3 3 3 2 2 3 2 3 4 5 6 + 1 10000000: 19 3 2 3 2 2 3 2 3 3 3 3 3 3 2 2 3 3 3 3 3 2 3 3 3 3 2 3 2 3 2 3 4 5 6 + 1 10000000: 20 3 2 3 2 3 2 3 2 3 3 2 2 2 3 2 3 2 3 2 3 3 3 3 3 2 3 3 2 3 4 5 6 + 1 10000000: 7 2 3 3 3 3 3 3 3 2 2 3 2 2 3 3 3 2 3 3 2 3 2 3 2 3 2 3 3 3 4 5 6 + 1 10000000: 21 3 2 2 3 3 3 2 2 2 2 3 3 2 2 2 2 3 2 3 2 2 4 5 6 + 1 10000000: 22 3 3 3 2 3 3 3 3 2 3 3 2 3 2 3 3 3 3 3 3 2 3 3 2 3 2 2 2 2 3 4 5 6 + 1 10000000: 17 2 2 2 2 3 3 3 3 3 2 3 2 3 3 2 3 3 2 3 3 3 3 3 3 3 3 3 3 3 2 4 5 6 + 1 10000000: 9 2 2 3 2 2 3 2 3 2 2 3 3 3 3 3 3 3 3 2 3 3 3 3 3 2 2 2 2 3 4 5 6 + 1 10000000: 7 2 3 2 3 2 3 3 3 2 3 2 3 3 3 2 2 3 3 2 3 2 3 3 2 3 3 2 3 3 4 5 6 + 1 10000000: 1 2 2 3 2 2 3 2 3 2 3 3 3 3 3 3 3 3 3 3 3 2 2 3 2 2 3 2 3 3 4 5 6 + 12 120000000: 23 24 25 26 27 28 29 30 31 32 33 34 + 1 10000000: 13 2 3 2 3 3 2 3 3 3 2 2 2 3 3 3 3 2 3 3 3 3 3 3 2 2 2 3 2 3 4 5 6 + 1 10000000: 35 3 3 2 3 2 3 2 3 2 3 2 3 2 3 2 2 2 3 3 2 3 3 3 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 13 3 3 3 3 3 2 3 3 3 2 2 3 3 3 3 2 3 2 3 3 3 2 2 2 2 3 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 1 3 3 3 3 2 3 3 3 3 3 3 2 3 2 2 3 3 3 3 2 2 2 3 3 3 3 2 3 2 3 4 5 6 + 1 10000000: 9 2 3 3 3 3 3 3 2 3 3 3 2 3 2 2 3 3 3 3 2 3 2 3 3 3 3 3 3 2 2 3 4 5 6 + 1 10000000: 17 3 2 3 3 3 3 2 2 3 3 3 3 3 3 3 3 2 3 3 3 3 3 3 2 3 2 3 2 2 3 3 3 4 5 6 + 1 10000000: 16 3 3 2 3 2 3 3 3 3 3 3 3 3 2 3 3 3 3 2 3 3 2 3 3 3 3 3 3 3 2 3 2 3 4 5 6 + 1 10000000: 14 2 3 3 2 3 3 3 3 3 3 3 3 2 3 3 2 2 2 3 3 3 2 3 3 2 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 36 3 3 3 3 3 3 3 2 3 3 3 3 2 2 3 3 2 3 3 3 2 3 3 3 2 3 3 2 3 2 3 3 4 5 6 + 1 10000000: 37 3 3 3 3 3 2 3 3 3 3 3 3 2 3 2 2 2 3 2 3 2 3 3 3 3 3 2 3 2 3 2 4 5 6 + 1 10000000: 8 2 3 3 3 3 3 2 3 3 3 2 3 3 3 3 3 3 3 2 2 2 3 2 3 2 3 3 2 3 2 3 4 5 6 + 1 10000000: 13 2 3 3 3 3 3 3 3 3 3 3 3 2 3 2 2 3 3 3 3 2 3 3 3 2 3 2 3 2 3 3 3 2 4 5 6 + 1 10000000: 36 3 3 3 2 3 3 2 3 3 3 3 3 3 3 3 3 3 3 3 2 2 3 3 2 3 3 3 3 3 3 2 3 3 3 3 4 5 6 + 1 10000000: 9 3 3 3 3 2 3 2 3 3 2 2 3 3 2 3 2 2 3 2 3 3 3 3 3 2 3 2 2 2 4 5 6 + 1 10000000: 9 2 2 2 3 2 2 2 2 3 3 3 3 2 3 2 3 3 3 3 3 3 2 3 3 2 2 2 3 3 4 5 6 + 1 10000000: 9 2 2 3 3 3 3 3 3 2 2 2 2 3 3 3 3 2 2 2 3 3 3 3 3 2 3 4 5 6 + 1 10000000: 14 2 2 3 3 3 3 3 3 3 3 2 3 2 3 2 3 3 3 3 3 3 3 3 2 2 2 3 2 2 4 5 6 + 1 10000000: 15 3 2 2 3 3 3 2 3 2 3 3 2 3 3 3 2 2 3 3 3 3 3 2 2 3 2 3 3 2 4 5 6 + 1 10000000: 21 2 3 2 3 3 3 2 3 2 3 3 3 2 2 3 2 3 3 3 3 2 3 2 4 5 6 + 1 10000000: 21 3 3 3 2 2 3 3 3 2 3 3 3 2 2 3 3 2 3 3 3 3 2 2 2 4 5 6 + 1 10000000: 14 2 2 3 2 2 2 2 2 3 3 3 3 3 2 2 2 2 3 3 3 3 2 3 3 4 5 6 + 1 10000000: 11 2 2 3 3 3 3 3 3 2 3 3 3 3 3 2 3 3 3 3 3 2 3 2 3 3 4 5 6 + 1 10000000: 9 2 2 3 3 2 3 3 3 2 2 3 2 3 2 2 2 2 3 2 3 3 3 3 3 2 3 3 3 2 4 5 6 + 1 10000000: 14 2 3 2 3 3 3 3 2 2 2 3 3 2 3 3 3 3 3 3 3 2 2 3 2 3 4 5 6 + 1 10000000: 11 2 2 3 2 3 2 3 3 3 2 3 2 3 3 3 3 2 3 2 3 3 2 2 3 3 2 4 5 6 + 1 10000000: 14 3 2 2 3 3 3 3 3 3 2 3 3 3 3 3 3 3 2 3 3 2 3 2 3 3 2 2 4 5 6 + 1 10000000: 9 3 2 2 3 3 3 3 2 2 3 2 2 2 3 2 3 2 2 3 3 3 3 3 3 2 2 2 4 5 6 + 1 10000000: 13 3 3 3 3 3 3 3 3 2 3 2 2 2 3 2 2 3 3 2 2 2 3 3 2 2 3 2 4 5 6 + 1 10000000: 9 2 2 3 3 3 3 2 3 2 2 3 3 3 3 2 3 2 3 3 3 2 3 3 3 3 2 2 4 5 6 + 1 10000000: 11 3 2 2 3 3 2 3 2 3 3 2 3 3 2 2 2 3 2 4 5 6 + 1 10000000: 13 3 3 3 3 3 3 3 2 3 3 3 2 2 2 3 3 3 2 3 2 3 3 2 2 2 2 3 3 2 4 5 6 + 1 10000000: 13 2 3 3 3 2 2 3 2 3 3 3 3 2 3 2 3 2 3 2 3 3 2 3 2 2 3 2 3 2 4 5 6 + 1 10000000: 9 2 2 2 3 3 3 2 3 3 3 2 2 3 3 2 3 3 3 3 3 3 2 2 3 3 3 3 3 3 3 3 2 3 4 5 6 + 1 10000000: 14 3 3 3 2 3 3 3 2 2 3 3 3 3 3 3 2 2 3 3 2 2 3 3 3 2 2 3 3 3 3 3 4 5 6 + 1 10000000: 13 3 3 3 3 3 3 3 3 3 3 3 3 2 3 3 2 3 3 3 3 3 3 3 2 2 3 3 3 3 3 2 3 3 3 4 5 6 + 1 10000000: 37 2 3 3 3 2 3 3 2 2 3 3 3 3 3 2 3 2 3 2 3 3 2 3 3 3 2 3 3 3 2 3 3 4 5 6 + 1 10000000: 13 2 3 2 3 3 3 2 3 3 3 3 2 3 3 2 3 3 3 3 3 2 2 2 3 2 3 3 3 2 2 3 3 4 5 6 + 1 10000000: 9 2 3 3 3 3 3 3 2 3 3 3 2 2 2 2 3 2 2 3 3 3 3 3 3 3 2 2 3 3 3 3 3 4 5 6 + 1 10000000: 38 3 3 3 3 3 2 3 3 3 2 2 3 3 2 3 3 3 3 3 2 3 3 3 3 2 3 3 3 2 2 3 3 4 5 6 + 1 10000000: 19 3 2 3 2 3 3 3 3 3 3 2 3 2 2 3 2 2 3 2 3 3 3 2 2 3 3 3 3 2 3 4 5 6 + 1 10000000: 37 3 3 3 2 3 3 2 3 2 2 3 2 3 2 3 3 2 2 3 3 3 3 2 2 2 2 3 3 3 3 4 5 6 + 1 10000000: 9 3 2 3 3 3 3 3 2 3 2 3 3 3 3 2 3 3 2 3 3 2 3 3 3 3 3 3 2 2 3 3 2 3 4 5 6 + 1 10000000: 13 3 3 3 2 2 3 3 3 3 3 2 3 3 3 3 3 2 2 2 3 3 3 2 3 3 3 3 2 2 3 2 4 5 6 + 1 10000000: 39 3 2 2 3 2 2 3 3 3 3 2 3 3 3 3 3 3 2 2 3 3 2 3 2 2 3 3 2 3 3 4 5 6 + 1 10000000: 13 3 3 3 3 3 2 3 3 2 3 2 2 2 3 2 3 3 3 3 2 3 3 2 3 3 2 3 3 3 3 2 3 4 5 6 + 1 10000000: 9 2 2 3 2 3 3 3 3 3 3 3 3 3 3 2 3 2 3 3 2 3 2 3 2 3 3 2 2 3 2 3 3 4 5 6 + 1 10000000: 8 2 3 2 3 3 2 3 2 3 3 2 3 3 3 2 3 2 2 2 3 3 3 3 3 3 3 3 2 2 3 3 4 5 6 + 1 10000000: 10 2 2 2 3 3 2 3 3 3 3 2 3 3 2 3 3 2 3 2 3 2 3 2 2 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 13 2 3 3 3 3 3 3 2 3 3 3 2 2 3 3 3 3 2 2 2 3 2 3 3 3 3 3 2 2 3 3 3 4 5 6 + 1 10000000: 9 2 2 3 3 2 3 3 3 3 3 2 3 3 3 3 2 2 3 3 2 3 2 3 2 3 2 3 3 2 4 5 6 + 1 10000000: 11 3 3 3 3 3 3 2 2 2 2 2 3 3 3 3 2 2 2 3 3 2 2 3 3 4 5 6 + 1 10000000: 11 2 3 2 2 3 3 3 3 3 2 3 2 3 3 2 2 2 2 3 3 2 3 3 3 3 2 4 5 6 + 1 10000000: 1 2 3 3 3 3 3 3 2 2 2 2 3 3 2 2 3 2 2 3 2 3 2 2 2 2 4 5 6 + 1 10000000: 11 3 2 2 3 3 3 2 2 2 2 3 2 3 2 2 3 3 3 2 3 3 2 3 2 2 4 5 6 + 1 10000000: 14 2 3 2 2 2 3 2 2 3 3 3 3 2 3 3 3 2 2 3 3 3 2 3 4 5 6 + 1 10000000: 10 2 2 2 2 2 2 3 2 2 3 3 3 3 3 3 3 3 2 3 3 3 3 2 2 3 2 4 5 6 + 1 10000000: 11 3 2 2 2 2 3 2 3 2 3 3 2 3 3 3 3 2 3 2 2 3 2 3 2 3 3 4 5 6 + 1 10000000: 9 2 2 2 3 3 2 3 3 3 2 2 2 3 3 3 3 3 2 3 3 3 3 3 3 3 2 3 4 5 6 + 1 10000000: 11 3 2 2 3 2 2 3 3 3 3 3 2 2 3 3 2 3 3 3 2 3 3 3 2 3 3 3 4 5 6 + 1 10000000: 1 2 2 3 2 3 3 3 3 3 3 2 3 3 2 3 2 3 3 3 3 2 2 2 3 2 3 4 5 6 + 1 10000000: 21 2 3 2 3 2 3 3 3 3 3 3 3 2 3 2 2 2 2 3 2 3 3 3 2 2 4 5 6 + 1 10000000: 7 3 3 3 3 3 3 2 3 3 2 3 2 2 2 3 3 2 3 2 2 3 2 2 2 4 5 6 + 1 10000000: 14 3 2 2 3 2 3 3 3 3 2 3 2 2 2 3 2 3 3 2 3 3 2 2 3 3 4 5 6 + 1 10000000: 21 3 3 3 2 2 3 3 3 3 3 2 2 2 2 2 3 3 2 2 2 3 3 3 2 3 2 3 4 5 6 + 1 10000000: 15 3 2 2 3 2 3 3 2 2 3 3 3 3 3 3 3 2 2 3 3 2 2 2 2 3 3 3 4 5 6 + 1 10000000: 7 3 2 2 3 3 3 3 2 3 3 3 3 3 3 2 3 3 3 3 3 3 2 3 3 3 2 3 4 5 6 + 1 10000000: 21 2 2 2 2 2 3 3 2 2 3 2 2 3 2 3 3 3 2 2 3 3 3 2 3 3 2 3 4 5 6 + 1 10000000: 8 2 3 3 2 3 2 2 3 2 3 3 2 2 3 3 2 2 2 3 3 2 3 2 2 2 3 3 4 5 6 + 1 10000000: 11 3 2 3 3 2 2 2 2 2 3 3 3 3 2 2 3 3 2 2 3 3 2 3 3 3 3 4 5 6 + 1 10000000: 13 3 3 2 2 3 2 3 2 2 2 3 2 3 3 2 3 3 2 2 3 3 3 2 2 3 3 3 4 5 6 + 1 10000000: 21 2 3 2 2 2 3 2 3 3 3 2 2 2 2 3 2 3 3 2 2 3 3 2 3 2 3 3 4 5 6 + 1 10000000: 22 3 2 2 3 2 2 3 3 2 3 3 3 3 2 3 3 3 2 2 3 3 3 3 3 3 2 3 2 3 4 5 6 + 1 10000000: 38 3 3 2 3 2 3 2 3 3 2 2 2 2 3 3 3 2 3 2 2 3 3 3 2 2 3 3 3 3 4 5 6 + 1 10000000: 38 3 2 3 2 3 3 3 3 2 3 3 3 3 2 3 3 2 3 2 2 3 2 2 2 2 2 3 3 3 4 5 6 + 1 10000000: 37 3 3 3 3 2 2 2 3 3 2 3 3 3 2 3 2 2 3 3 2 3 2 3 3 2 2 2 3 3 4 5 6 + 1 10000000: 10 3 3 3 3 3 3 3 3 3 2 3 2 3 3 3 3 3 2 3 2 3 3 2 2 2 3 3 3 3 3 2 2 3 4 5 6 + 1 10000000: 10 3 3 3 3 2 3 3 3 2 2 3 2 3 3 2 3 3 3 3 3 2 3 3 2 3 2 3 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 13 2 3 3 3 3 3 3 2 2 3 3 2 3 3 3 3 3 2 3 2 2 3 3 3 2 2 3 2 2 3 3 4 5 6 + 1 10000000: 35 2 3 3 2 2 3 3 3 3 2 3 3 3 3 2 3 3 3 3 3 3 3 2 3 2 2 3 2 3 2 3 4 5 6 + 1 10000000: 10 2 3 2 3 3 3 3 3 3 3 3 3 2 2 3 3 3 2 3 3 3 3 2 3 2 3 2 2 3 3 2 3 4 5 6 + 1 10000000: 15 2 2 2 3 3 3 3 2 2 2 3 3 2 3 3 3 3 3 3 3 3 3 3 3 2 3 3 3 3 3 3 4 5 6 + 1 10000000: 16 2 3 3 3 3 2 3 2 3 3 3 2 3 3 2 3 3 2 2 3 2 2 3 3 2 3 3 3 3 3 4 5 6 + 1 10000000: 20 3 3 3 3 3 2 3 2 2 2 2 3 3 3 3 2 3 2 3 3 2 3 3 2 3 3 3 3 2 3 3 4 5 6 + 1 10000000: 14 3 3 2 3 3 3 2 3 3 2 3 2 3 3 3 3 3 3 2 3 3 3 3 3 3 2 3 2 2 3 2 3 4 5 6 + 1 10000000: 16 3 3 3 3 2 3 2 3 2 2 2 3 2 2 3 3 3 2 3 2 3 3 2 3 2 3 3 3 3 3 4 5 6 + 1 10000000: 12 3 3 3 2 3 2 2 3 2 3 2 3 3 3 3 3 3 3 3 3 2 3 3 3 3 2 2 2 3 3 3 3 4 5 6 + 1 10000000: 10 3 3 3 2 3 3 3 3 2 3 3 2 3 3 3 3 3 2 3 3 2 3 2 2 2 2 3 3 2 3 4 5 6 + 1 10000000: 21 3 3 2 2 3 3 2 2 3 3 3 3 3 3 3 3 3 3 2 3 2 3 2 2 3 2 3 3 3 3 4 5 6 + 1 10000000: 9 2 2 2 3 2 3 3 3 3 2 2 2 3 3 3 2 3 3 2 3 3 2 2 3 3 3 3 3 3 2 3 4 5 6 + 1 10000000: 14 3 3 3 3 2 3 3 3 3 2 3 2 2 3 2 3 3 3 2 3 3 3 3 3 3 2 3 3 2 3 4 5 6 + 1 10000000: 1 2 2 2 2 2 3 2 3 2 3 3 2 3 3 2 3 3 3 3 3 2 3 3 2 2 3 4 5 6 + 1 10000000: 1 2 2 2 2 3 3 2 3 3 2 2 3 3 3 3 3 2 3 2 2 3 2 3 2 3 4 5 6 + 1 10000000: 12 3 2 2 2 3 3 3 2 2 3 3 2 3 2 3 2 3 2 3 3 3 3 3 2 3 4 5 6 + 1 10000000: 11 3 3 2 3 2 3 3 3 2 3 3 2 2 3 2 3 2 2 2 2 3 3 3 3 2 3 4 5 6 + 1 10000000: 7 3 2 2 3 2 3 3 3 3 3 2 2 2 3 3 2 3 3 3 3 3 2 2 3 3 2 4 5 6 + 1 10000000: 14 2 2 2 2 2 3 3 3 2 2 3 3 2 2 3 3 2 3 3 2 2 2 3 4 5 6 + 1 10000000: 9 3 3 3 3 2 2 3 2 2 2 2 2 3 2 2 2 3 2 3 2 3 3 2 3 2 4 5 6 + 1 10000000: 14 2 3 3 2 2 3 2 3 2 3 3 3 3 2 3 3 2 3 3 2 2 3 3 2 2 2 4 5 6 + 1 10000000: 14 3 3 2 2 3 2 3 2 2 2 3 2 3 3 3 3 2 3 3 3 3 2 3 2 3 2 2 4 5 6 + 1 10000000: 12 2 3 3 3 2 2 3 3 3 3 3 3 3 2 3 2 3 3 3 3 2 3 3 2 3 3 3 4 5 6 + 1 10000000: 12 2 3 3 2 3 2 3 2 3 3 2 3 3 3 3 2 2 3 3 3 3 2 3 2 2 3 3 4 5 6 + 1 10000000: 11 3 2 3 2 2 2 3 2 2 3 2 3 2 2 2 3 3 2 3 2 3 2 3 3 3 4 5 6 + 1 10000000: 21 2 2 2 3 2 2 3 3 3 2 2 3 3 2 3 3 3 3 3 2 2 3 3 2 3 3 4 5 6 + 1 10000000: 14 2 2 2 3 3 2 3 3 2 3 3 2 3 3 3 2 3 3 2 2 3 2 2 3 4 5 6 + 1 10000000: 40 2 2 3 2 2 2 2 2 3 2 2 2 3 2 3 3 3 3 3 3 2 2 2 3 2 4 5 6 + 1 10000000: 9 3 2 2 2 3 2 3 3 3 3 3 3 2 2 2 3 3 3 3 3 2 3 3 3 3 2 3 4 5 6 + 1 10000000: 11 3 2 2 3 2 2 3 2 3 3 2 3 3 3 2 3 2 3 3 2 2 2 3 3 3 2 3 4 5 6 + 1 10000000: 11 2 3 2 3 3 2 3 3 3 3 3 2 3 3 2 3 3 2 2 3 3 2 3 2 3 2 3 4 5 6 + 1 10000000: 14 3 3 2 2 3 3 2 3 3 3 3 3 3 2 2 2 3 3 3 2 3 2 3 3 3 2 3 4 5 6 + 1 10000000: 9 2 3 3 2 3 2 3 3 2 3 3 2 2 2 2 2 2 2 3 3 3 3 2 3 3 3 3 4 5 6 + 1 10000000: 10 3 3 2 3 2 2 2 3 2 2 2 2 3 2 2 3 2 3 2 3 2 2 2 3 3 3 4 5 6 + 1 10000000: 16 3 2 3 2 3 2 2 2 2 3 2 2 3 2 2 2 2 2 2 2 2 2 3 3 4 5 6 + 1 10000000: 10 2 2 2 3 3 3 2 3 3 3 2 3 2 2 2 3 3 2 2 3 2 2 2 3 3 3 3 4 5 6 + 1 10000000: 12 3 3 3 2 3 3 2 2 3 2 2 2 2 2 2 3 3 3 2 3 2 2 2 3 2 3 3 4 5 6 + 1 10000000: 38 3 3 2 3 2 2 3 2 2 3 2 2 3 3 3 3 3 3 2 3 3 2 3 2 3 3 3 2 3 4 5 6 + 1 10000000: 38 3 3 3 3 3 3 2 2 2 2 2 2 2 2 3 2 3 3 3 3 3 3 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 9 2 2 3 2 3 3 3 3 3 3 2 3 3 2 3 3 2 3 2 2 3 3 3 2 2 2 2 2 4 5 6 + 1 10000000: 9 2 2 3 3 3 2 3 3 2 2 2 2 3 3 3 2 3 2 2 3 3 2 3 3 2 3 3 3 4 5 6 + 1 10000000: 10 2 3 3 3 2 3 3 3 2 3 3 2 2 2 3 2 3 2 3 3 3 3 3 2 2 3 2 3 4 5 6 + 1 10000000: 21 3 3 3 2 3 2 3 3 3 2 2 3 3 3 3 3 3 2 3 3 3 3 3 3 2 3 2 2 4 5 6 + 1 10000000: 21 3 3 3 3 3 2 3 3 3 3 3 2 2 3 3 3 2 2 2 3 2 3 3 3 2 3 3 2 4 5 6 + 1 10000000: 21 2 3 2 2 3 3 3 3 3 3 3 3 2 2 3 3 3 2 3 3 3 2 3 3 2 3 3 2 4 5 6 + 1 10000000: 21 3 3 2 2 3 3 3 2 2 3 2 3 3 2 3 3 2 3 3 2 3 2 3 3 3 3 3 3 4 5 6 + 1 10000000: 21 3 2 2 2 3 3 3 3 3 3 2 2 3 2 3 2 2 3 3 3 2 2 3 3 3 3 3 3 4 5 6 + 1 10000000: 11 3 3 2 3 3 3 3 2 2 3 3 2 3 3 3 2 3 3 3 3 2 3 3 2 2 3 3 3 4 5 6 + 1 10000000: 13 3 3 2 3 3 3 3 3 2 3 3 3 3 3 3 3 3 2 2 3 3 3 2 3 2 2 2 3 3 3 3 3 4 5 6 + 1 10000000: 16 2 2 2 3 3 3 3 3 2 3 3 3 3 3 3 3 2 2 2 3 2 2 2 3 3 3 2 3 3 3 4 5 6 + 1 10000000: 36 3 2 3 2 2 2 3 3 3 3 3 2 3 2 3 3 2 3 2 2 2 3 3 3 3 3 3 3 3 3 4 5 6 + 1 10000000: 39 2 2 2 2 3 3 3 3 3 3 3 2 3 3 3 2 3 3 3 2 3 3 2 3 3 3 3 2 2 4 5 6 + 1 10000000: 11 3 2 3 2 3 3 3 2 2 3 3 3 2 3 3 2 3 2 3 3 3 3 2 3 3 3 2 3 3 3 4 5 6 + 1 10000000: 39 2 3 3 3 2 3 2 3 3 3 3 2 3 2 3 2 2 2 2 3 3 3 3 3 3 3 2 2 3 4 5 6 + 1 10000000: 38 3 3 2 2 2 3 3 2 3 3 2 3 2 3 2 3 3 2 2 3 2 3 3 2 2 3 4 5 6 + 1 10000000: 40 2 2 3 3 3 3 2 3 2 3 2 3 3 2 3 3 3 2 3 2 3 3 3 2 2 2 2 3 3 4 5 6 + 1 10000000: 14 3 2 3 2 2 3 3 2 2 2 3 3 2 2 3 3 3 3 3 3 2 3 2 3 3 2 4 5 6 + 1 10000000: 21 2 2 2 3 3 2 3 2 2 2 3 3 3 2 3 3 3 2 3 2 3 2 2 4 5 6 + 1 10000000: 21 2 3 2 2 2 3 2 3 2 2 2 3 3 2 2 2 2 3 2 2 2 3 2 3 4 5 6 + 1 10000000: 11 3 3 3 3 3 3 3 2 3 3 3 2 3 2 2 3 2 2 2 2 3 3 4 5 6 + 1 10000000: 11 3 2 3 2 2 2 2 2 2 2 2 3 3 3 3 3 3 3 2 3 3 2 4 5 6 + 1 10000000: 10 2 3 2 2 3 2 3 3 3 2 3 3 3 2 3 3 3 3 3 2 3 3 2 2 3 3 3 3 4 5 6 + 1 10000000: 8 3 3 2 3 3 2 3 2 3 2 3 3 3 3 3 2 2 3 2 3 2 3 2 3 2 2 3 2 4 5 6 + 1 10000000: 11 3 3 3 2 2 3 2 3 2 3 2 3 3 2 3 3 3 3 3 3 3 2 3 3 3 2 2 3 4 5 6 + 1 10000000: 11 3 3 3 2 2 2 3 3 3 3 3 3 3 3 2 3 2 2 3 3 3 3 3 3 3 3 3 2 4 5 6 + 1 10000000: 20 3 2 3 3 3 2 2 3 3 3 2 3 3 3 3 2 2 2 3 3 2 3 3 2 3 2 3 2 4 5 6 + 1 10000000: 13 3 2 2 3 3 2 3 2 2 3 2 3 2 2 2 3 3 3 2 3 2 3 3 2 3 3 3 3 4 5 6 + 1 10000000: 13 3 3 2 3 3 2 2 3 3 3 2 3 2 3 3 2 3 3 3 3 3 3 3 2 2 3 3 3 4 5 6 + 1 10000000: 13 3 3 3 3 2 3 3 2 3 2 3 2 2 2 2 2 2 3 2 3 2 2 3 3 3 2 3 3 4 5 6 + 1 10000000: 21 3 2 3 3 3 3 2 3 3 3 2 3 3 2 3 3 3 2 3 2 3 2 3 3 3 2 3 2 4 5 6 + 1 10000000: 21 2 2 2 3 2 3 3 3 3 3 3 2 3 2 2 3 2 3 2 3 3 3 3 3 2 3 3 3 4 5 6 + 1 10000000: 1 2 3 3 3 2 3 2 3 3 2 3 3 2 2 3 2 3 3 3 2 3 2 3 2 2 3 3 3 4 5 6 + 1 10000000: 35 2 3 3 3 3 2 3 2 2 2 2 3 3 2 2 3 3 2 2 3 3 3 3 3 3 3 2 2 4 5 6 + 1 10000000: 21 3 3 3 3 3 3 3 2 3 3 3 2 3 2 2 3 3 3 2 2 3 3 3 3 3 2 2 3 4 5 6 + 1 10000000: 14 3 3 3 3 2 3 2 3 3 2 3 3 3 3 3 2 2 3 2 2 2 3 3 3 3 3 3 2 4 5 6 + 1 10000000: 7 3 3 2 3 3 2 3 2 3 3 3 2 3 3 3 3 2 3 3 3 3 3 2 2 3 3 3 3 4 5 6 + 1 10000000: 12 2 3 2 2 3 3 3 3 3 2 3 2 2 3 2 3 3 2 3 3 2 3 3 2 3 2 2 3 4 5 6 + 1 10000000: 12 2 3 3 3 3 3 3 3 2 3 3 3 2 2 2 2 3 3 2 3 2 2 3 3 3 3 3 2 4 5 6 + 1 10000000: 40 2 3 3 3 3 2 3 3 2 2 3 2 2 3 3 2 3 2 2 3 3 2 2 3 3 3 3 2 4 5 6 + 1 10000000: 21 3 3 3 2 3 3 3 3 3 3 3 3 2 3 3 2 2 3 2 3 3 3 2 2 2 3 3 3 4 5 6 + 1 10000000: 14 3 3 3 2 2 3 2 3 3 3 2 2 3 3 3 2 3 2 2 2 3 3 3 2 3 3 3 3 4 5 6 + 1 10000000: 11 2 2 3 3 3 3 3 3 2 3 3 2 2 3 3 3 3 2 3 3 3 2 3 3 2 3 3 2 3 4 5 6 + 1 10000000: 20 3 2 3 2 3 3 3 3 3 3 3 2 3 3 2 2 3 2 2 2 4 5 6 + 1 10000000: 9 2 3 3 3 3 3 2 3 3 3 3 2 2 3 3 3 2 3 2 3 2 2 2 2 3 3 3 3 3 3 4 5 6 + 1 10000000: 41 2 2 2 3 3 2 2 3 2 3 3 3 3 3 3 3 2 3 2 3 2 3 3 3 2 2 3 3 3 2 4 5 6 + 1 10000000: 21 3 3 3 2 3 3 3 3 2 3 3 3 3 3 2 3 2 2 3 3 2 3 3 3 3 3 3 2 3 3 4 5 6 + 1 10000000: 36 3 2 3 3 2 3 2 2 3 3 3 3 2 3 3 3 2 3 3 3 2 3 3 2 2 3 3 3 2 3 4 5 6 + 1 10000000: 10 3 2 2 3 2 2 2 3 2 3 2 3 3 3 3 3 3 2 2 3 3 3 3 3 2 2 2 3 3 4 5 6 + 1 10000000: 13 2 3 3 2 3 3 3 2 3 2 3 2 2 2 3 3 2 2 3 3 3 3 2 3 3 3 2 3 2 3 4 5 6 + 1 10000000: 8 2 3 3 2 3 3 2 3 3 2 3 2 2 2 2 3 3 2 2 3 3 3 3 2 3 2 2 3 4 5 6 + 1 10000000: 10 2 3 3 3 3 3 2 2 2 3 3 2 3 2 3 3 2 2 3 3 3 3 2 3 3 3 2 2 4 5 6 + 1 10000000: 13 3 2 2 3 3 2 2 3 2 3 2 3 3 2 2 2 3 2 3 3 3 3 3 3 2 2 2 3 4 5 6 + 1 10000000: 11 3 3 2 2 2 3 2 3 3 2 3 2 3 3 2 3 3 2 3 3 3 2 3 3 3 3 3 3 4 5 6 + 1 10000000: 1 2 3 3 3 3 3 3 3 3 3 3 3 3 3 2 3 2 2 2 3 2 3 2 2 3 3 2 3 4 5 6 + 1 10000000: 38 3 3 2 3 2 2 2 3 2 3 3 2 3 3 2 2 3 3 3 3 2 2 3 3 2 3 3 3 4 5 6 + 1 10000000: 11 3 3 3 3 3 2 3 3 3 3 3 2 3 2 2 3 3 3 3 3 2 3 3 2 3 3 3 3 2 4 5 6 + 1 10000000: 13 3 3 2 2 3 3 3 2 3 2 3 2 2 2 2 2 3 2 3 2 2 3 2 3 3 2 3 3 4 5 6 + 1 10000000: 11 3 3 3 2 2 2 2 3 3 3 3 2 3 2 3 2 3 2 3 2 3 3 2 3 3 2 2 3 3 4 5 6 + 1 10000000: 21 3 3 2 3 3 3 3 3 2 2 3 3 3 3 2 3 3 3 2 3 2 3 2 3 3 2 3 2 4 5 6 + 1 10000000: 7 3 2 3 2 3 3 3 2 3 3 3 3 2 3 2 3 2 2 3 3 3 3 2 3 3 2 2 3 4 5 6 + 1 10000000: 11 3 2 3 2 2 3 2 2 2 3 3 3 3 2 2 3 3 3 2 3 3 2 2 3 3 3 3 3 3 4 5 6 + 1 10000000: 11 2 2 2 3 3 2 3 2 3 3 2 2 3 3 3 3 3 3 3 3 3 2 2 3 3 3 3 2 3 4 5 6 + 1 10000000: 14 3 3 3 2 2 2 2 2 2 3 3 3 2 3 2 3 3 3 3 3 3 3 3 3 2 3 2 2 4 5 6 + 1 10000000: 21 2 2 3 3 3 3 2 3 2 2 2 2 3 2 3 3 3 3 3 3 2 2 2 3 3 3 2 3 3 4 5 6 + 1 10000000: 13 2 3 3 3 3 3 2 3 2 3 3 3 3 2 2 3 3 3 3 3 3 2 3 2 2 3 2 2 2 3 4 5 6 + 1 10000000: 13 2 3 3 2 2 3 2 2 2 3 2 3 3 2 3 2 2 3 2 2 3 3 3 3 3 3 3 2 3 3 4 5 6 + 1 10000000: 10 2 2 3 2 3 3 2 2 3 2 3 3 2 2 3 3 3 3 2 3 3 3 2 3 3 2 2 2 3 4 5 6 + 1 10000000: 8 2 3 3 3 3 2 3 2 3 3 3 2 3 3 3 2 3 3 2 2 2 2 3 2 3 2 2 3 3 4 5 6 + 1 10000000: 9 2 2 2 3 3 2 2 3 2 3 3 3 2 3 3 2 2 3 2 2 3 3 2 3 3 3 3 2 3 3 4 5 6 + 1 10000000: 7 3 3 2 3 3 2 3 3 2 3 3 3 3 3 2 3 3 2 3 2 2 2 3 3 3 3 3 3 3 2 4 5 6 + 1 10000000: 7 3 3 3 3 3 3 3 2 3 3 3 3 2 3 3 3 2 3 3 3 2 3 3 2 3 2 3 3 2 3 4 5 6 + 1 10000000: 7 2 3 3 3 3 3 3 3 3 3 2 3 3 3 2 2 3 2 3 3 3 3 3 3 3 2 3 2 3 3 4 5 6 + 1 10000000: 36 3 3 3 2 2 3 3 3 2 3 3 3 2 3 2 3 3 2 2 3 3 3 3 3 2 2 3 2 2 4 5 6 + 1 10000000: 7 3 3 3 3 3 3 3 3 2 3 3 3 3 3 3 3 2 2 2 2 2 2 2 3 2 2 3 3 3 4 5 6 + 1 10000000: 11 3 2 2 3 3 2 3 2 3 3 3 2 2 3 3 2 2 3 3 2 3 3 2 3 2 3 3 3 3 4 5 6 + 1 10000000: 21 3 2 3 3 3 2 2 3 2 3 2 3 3 3 3 3 2 3 2 3 3 3 3 2 3 2 3 3 3 4 5 6 + 1 10000000: 10 3 3 3 3 2 3 3 2 3 2 2 3 2 2 3 3 3 2 3 3 2 3 2 2 2 2 3 2 3 4 5 6 + 1 10000000: 21 2 3 3 3 3 3 3 3 2 2 3 3 3 3 3 3 2 3 2 3 3 3 3 2 2 3 3 3 3 3 4 5 6 + 1 10000000: 35 3 3 3 3 2 2 3 3 3 3 3 3 3 3 3 2 3 3 2 3 2 3 2 3 3 2 3 2 3 3 4 5 6 +Locations + 1: 0x206f main.fib :0 s=0 + 2: 0x2096 main.fib :0 s=0 + 3: 0x207a main.fib :0 s=0 + 4: 0x2134 main.main :0 s=0 + 5: 0x2df2f runtime.main :0 s=0 + 6: 0x5da90 runtime.goexit :0 s=0 + 7: 0x2085 main.fib :0 s=0 + 8: 0x2049 main.fib :0 s=0 + 9: 0x2040 main.fib :0 s=0 + 10: 0x204f main.fib :0 s=0 + 11: 0x2080 main.fib :0 s=0 + 12: 0x20a9 main.fib :0 s=0 + 13: 0x2058 main.fib :0 s=0 + 14: 0x20a4 main.fib :0 s=0 + 15: 0x20a1 main.fib :0 s=0 + 16: 0x2097 main.fib :0 s=0 + 17: 0x208a main.fib :0 s=0 + 18: 0x2072 main.fib :0 s=0 + 19: 0x206b main.fib :0 s=0 + 20: 0x2053 main.fib :0 s=0 + 21: 0x209c main.fib :0 s=0 + 22: 0x2092 main.fib :0 s=0 + 23: 0x5eecb runtime.mach_semaphore_signal :0 s=0 + 24: 0x29bef runtime.mach_semrelease :0 s=0 + 25: 0x28f29 runtime.semawakeup :0 s=0 + 26: 0xefae runtime.notewakeup :0 s=0 + 27: 0x32109 runtime.startm :0 s=0 + 28: 0x32468 runtime.wakep :0 s=0 + 29: 0x332ef runtime.resetspinning :0 s=0 + 30: 0x3374d runtime.schedule :0 s=0 + 31: 0x33b09 runtime.goschedImpl :0 s=0 + 32: 0x33ba1 runtime.gopreempt_m :0 s=0 + 33: 0x44511 runtime.newstack :0 s=0 + 34: 0x5b4fe runtime.morestack :0 s=0 + 35: 0x208e main.fib :0 s=0 + 36: 0x206c main.fib :0 s=0 + 37: 0x205e main.fib :0 s=0 + 38: 0x2076 main.fib :0 s=0 + 39: 0x207b main.fib :0 s=0 + 40: 0x20ad main.fib :0 s=0 + 41: 0x2067 main.fib :0 s=0 +Mappings From a3b513a05af8f62294b10836938f78176b332e73 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 15:06:11 -0700 Subject: [PATCH 04/19] Test for printing callstack --- pprof/raw.go | 7 +++++-- pprof/raw_test.go | 40 +++++++++++++++++++++++++++++++++++++++- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/pprof/raw.go b/pprof/raw.go index 5dbb60b..f11b8a8 100644 --- a/pprof/raw.go +++ b/pprof/raw.go @@ -120,12 +120,15 @@ func (p *rawParser) processLine(line string) { } // print prints out the stack traces collected from the raw pprof output. -func (p *rawParser) print(w io.WriteCloser) { +func (p *rawParser) print(w io.Writer) error { for _, r := range p.records { r.Serialize(p.funcName, w) fmt.Fprintln(w) } - w.Close() + if wc, ok := w.(io.WriteCloser); ok { + return wc.Close() + } + return nil } func findStackCollapse() string { diff --git a/pprof/raw_test.go b/pprof/raw_test.go index a447a7a..1b73c68 100644 --- a/pprof/raw_test.go +++ b/pprof/raw_test.go @@ -21,13 +21,14 @@ package pprof import ( + "bytes" "io/ioutil" "reflect" "testing" "time" ) -func TestParse(t *testing.T) { +func parseTestRawData(t *testing.T) *rawParser { rawBytes, err := ioutil.ReadFile("testdata/pprof.raw.txt") if err != nil { t.Fatalf("Failed to read testdata/pprof.raw.txt: %v", err) @@ -38,6 +39,12 @@ func TestParse(t *testing.T) { t.Fatalf("Parse failed: %v", err) } + return parser +} + +func TestParse(t *testing.T) { + parser := parseTestRawData(t) + // line 7 - 249 are stack records in the test file. const expectedNumRecords = 242 if len(parser.records) != expectedNumRecords { @@ -73,6 +80,37 @@ func TestParse(t *testing.T) { } } +func TestParseAndPrint(t *testing.T) { + parser := parseTestRawData(t) + buf := &bytes.Buffer{} + parser.print(buf) + got := buf.Bytes() + + expected1 := `main.fib +main.fib +main.fib +main.fib +main.main +runtime.main +runtime.goexit +1 +` + if !bytes.Contains(got, []byte(expected1)) { + t.Errorf("missing expected stack: %s", expected1) + } + + expected2 := `runtime.schedule +runtime.goschedImpl +runtime.gopreempt_m +runtime.newstack +runtime.morestack +12 +` + if !bytes.Contains(got, []byte(expected2)) { + t.Errorf("missing expected stack: %s", expected2) + } +} + func TestSplitBySpace(t *testing.T) { tests := []struct { s string From d039589de55add4c4ea918f297806069d0b9332a Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 15:40:49 -0700 Subject: [PATCH 05/19] Move flame graph running to a new package Use []byte everywhere for now rather than streams to keep it simple. Still need to add tests for ensuring running commands works --- pprof/raw.go | 35 ++++------------- renderer/flamegraph.go | 77 +++++++++++++++++++++++++++++++++++++ renderer/flamegraph_test.go | 75 ++++++++++++++++++++++++++++++++++++ 3 files changed, 159 insertions(+), 28 deletions(-) create mode 100644 renderer/flamegraph.go create mode 100644 renderer/flamegraph_test.go diff --git a/pprof/raw.go b/pprof/raw.go index f11b8a8..205941b 100644 --- a/pprof/raw.go +++ b/pprof/raw.go @@ -23,11 +23,8 @@ package pprof import ( "bufio" "bytes" - "errors" "fmt" "io" - "os" - "os/exec" "regexp" "strconv" "strings" @@ -63,9 +60,13 @@ func ParseRaw(input []byte) ([]byte, error) { return nil, err } - pr, pw := io.Pipe() - go parser.print(pw) - return CollapseStacks(pr) + // TODO(prashantv): Refactor interfaces so we use streams. + buf := &bytes.Buffer{} + if err := parser.print(buf); err != nil { + return nil, err + } + + return buf.Bytes(), nil } func newRawParser() *rawParser { @@ -131,28 +132,6 @@ func (p *rawParser) print(w io.Writer) error { return nil } -func findStackCollapse() string { - for _, v := range []string{"stackcollapse.pl", "./stackcollapse.pl", "./FlameGraph/stackcollapse.pl"} { - if path, err := exec.LookPath(v); err == nil { - return path - } - } - return "" -} - -// CollapseStacks runs the flamegraph's collapse stacks script. -func CollapseStacks(stacks io.Reader) ([]byte, error) { - stackCollapse := findStackCollapse() - if stackCollapse == "" { - return nil, errors.New("stackcollapse.pl not found") - } - - cmd := exec.Command(stackCollapse) - cmd.Stdin = stacks - cmd.Stderr = os.Stderr - return cmd.Output() -} - // addSample parses a sample that looks like: // 1 10000000: 1 2 3 4 // and creates a stackRecord for it. diff --git a/renderer/flamegraph.go b/renderer/flamegraph.go new file mode 100644 index 0000000..655d867 --- /dev/null +++ b/renderer/flamegraph.go @@ -0,0 +1,77 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package renderer + +import ( + "bytes" + "errors" + "os" + "os/exec" +) + +var errNoPerlScript = errors.New("Cannot find flamegraph scripts in the PATH or current " + + "directory. You can download the script at https://github.com/brendangregg/FlameGraph. " + + "These scripts should be added to your PATH or in the directory where go-torch is executed. " + + "Alternatively, you can run go-torch with the --raw flag.") + +var ( + stackCollapseScripts = []string{"stackcollapse.pl", "./stackcollapse.pl", "./FlameGraph/stackcollapse.pl"} + flameGraphScripts = []string{"flamegraph.pl", "./flamegraph.pl", "./FlameGraph/flamegraph.pl", "flame-graph-gen"} +) + +// findInPath returns the first path that is found in PATH. +func findInPath(paths []string) string { + for _, v := range paths { + if path, err := exec.LookPath(v); err == nil { + return path + } + } + return "" +} + +// runScript runs scriptName with the given arguments, and stdin set to inData. +// It returns the stdout on success. +func runScript(scriptName string, args []string, inData []byte) ([]byte, error) { + cmd := exec.Command(scriptName, args...) + cmd.Stdin = bytes.NewReader(inData) + cmd.Stderr = os.Stderr + return cmd.Output() +} + +// CollapseStacks runs the flamegraph's collapse stacks script. +func CollapseStacks(stacks []byte) ([]byte, error) { + stackCollapse := findInPath(stackCollapseScripts) + if stackCollapse == "" { + return nil, errNoPerlScript + } + + return runScript(stackCollapse, nil, stacks) +} + +// GenerateFlameGraph runs the flamegraph script to generate a flame graph SVG. +func GenerateFlameGraph(graphInput []byte) ([]byte, error) { + flameGraph := findInPath(flameGraphScripts) + if flameGraph == "" { + return nil, errNoPerlScript + } + + return runScript(flameGraph, nil, graphInput) +} diff --git a/renderer/flamegraph_test.go b/renderer/flamegraph_test.go new file mode 100644 index 0000000..1726ef7 --- /dev/null +++ b/renderer/flamegraph_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package renderer + +import ( + "os" + "path/filepath" + "testing" +) + +func TestFindInPatch(t *testing.T) { + const realCmd1 = "ls" + const realCmd2 = "cat" + const fakeCmd1 = "should-not-find-this" + const fakeCmd2 = "not-going-to-exist" + + tests := []struct { + paths []string + expected string + }{ + { + paths: []string{}, + }, + { + paths: []string{realCmd1}, + expected: realCmd1, + }, + { + paths: []string{fakeCmd1, realCmd1}, + expected: realCmd1, + }, + { + paths: []string{fakeCmd1, realCmd1, fakeCmd2, realCmd2}, + expected: realCmd1, + }, + } + + for _, tt := range tests { + got := findInPath(tt.paths) + var gotFile string + if got != "" { + gotFile = filepath.Base(got) + } + if gotFile != tt.expected { + t.Errorf("findInPaths(%v) got %v, want %v", tt.paths, gotFile, tt.expected) + } + + // Verify that the returned path exists. + if got != "" { + _, err := os.Stat(got) + if err != nil { + t.Errorf("returned path %v failed to stat: %v", got, err) + + } + } + } +} From 2258bd8975879d0e3e2577992b74fa98dd5801b5 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 16:48:40 -0700 Subject: [PATCH 06/19] Add invalid cases to pprof tests --- pprof/pprof_test.go | 25 +++++++++++-- pprof/raw.go | 8 +++++ pprof/raw_test.go | 86 ++++++++++++++++++++++++++++++++++++++++----- 3 files changed, 109 insertions(+), 10 deletions(-) diff --git a/pprof/pprof_test.go b/pprof/pprof_test.go index bebe1bf..88110c0 100644 --- a/pprof/pprof_test.go +++ b/pprof/pprof_test.go @@ -30,6 +30,7 @@ func TestGetArgs(t *testing.T) { tests := []struct { opts Options expected []string + wantErr bool }{ { opts: Options{ @@ -72,12 +73,23 @@ func TestGetArgs(t *testing.T) { TimeSeconds: 5}, expected: []string{"/path/to/binaryname", "/path/to/binaryfile"}, }, + { + opts: Options{ + BaseURL: "%-0", // this makes url.Parse fail. + URLSuffix: "/profile", + TimeSeconds: 5, + }, + wantErr: true, + }, } for _, tt := range tests { got, err := getArgs(tt.opts) + if (err != nil) != tt.wantErr { + t.Errorf("wantErr %v got error: %v", tt.wantErr, err) + continue + } if err != nil { - t.Errorf("failed to get pprof args: %v", err) continue } @@ -105,7 +117,16 @@ func TestRunPProfInvalidURL(t *testing.T) { } } -func TestGetPProfRaw(t *testing.T) { +func TestGetPProfRawBadURL(t *testing.T) { + opts := Options{ + BaseURL: "%-0", + } + if _, err := GetRaw(opts); err == nil { + t.Error("expected bad BaseURL to fail") + } +} + +func TestGetPProfRawSuccess(t *testing.T) { opts := Options{ BinaryFile: "testdata/pprof.1.pb.gz", } diff --git a/pprof/raw.go b/pprof/raw.go index 205941b..c7ea5b2 100644 --- a/pprof/raw.go +++ b/pprof/raw.go @@ -138,6 +138,10 @@ func (p *rawParser) print(w io.Writer) error { func (p *rawParser) addSample(line string) { // Parse a sample which looks like: parts := splitBySpace(line) + if len(parts) < 3 { + p.err = fmt.Errorf("malformed sample line: %v", line) + return + } samples, err := strconv.Atoi(parts[0]) if err != nil { @@ -164,6 +168,10 @@ func (p *rawParser) addSample(line string) { // and creates a mapping from funcID to function name. func (p *rawParser) addLocation(line string) { parts := splitBySpace(line) + if len(parts) < 3 { + p.err = fmt.Errorf("malformed location line: %v", line) + return + } funcID := p.toFuncID(strings.TrimSuffix(parts[0], ":")) p.funcName[funcID] = parts[2] } diff --git a/pprof/raw_test.go b/pprof/raw_test.go index 1b73c68..4aa4d86 100644 --- a/pprof/raw_test.go +++ b/pprof/raw_test.go @@ -24,11 +24,12 @@ import ( "bytes" "io/ioutil" "reflect" + "strings" "testing" "time" ) -func parseTestRawData(t *testing.T) *rawParser { +func parseTestRawData(t *testing.T) ([]byte, *rawParser) { rawBytes, err := ioutil.ReadFile("testdata/pprof.raw.txt") if err != nil { t.Fatalf("Failed to read testdata/pprof.raw.txt: %v", err) @@ -39,11 +40,11 @@ func parseTestRawData(t *testing.T) *rawParser { t.Fatalf("Parse failed: %v", err) } - return parser + return rawBytes, parser } func TestParse(t *testing.T) { - parser := parseTestRawData(t) + _, parser := parseTestRawData(t) // line 7 - 249 are stack records in the test file. const expectedNumRecords = 242 @@ -80,11 +81,12 @@ func TestParse(t *testing.T) { } } -func TestParseAndPrint(t *testing.T) { - parser := parseTestRawData(t) - buf := &bytes.Buffer{} - parser.print(buf) - got := buf.Bytes() +func TestParseRawValid(t *testing.T) { + rawBytes, _ := parseTestRawData(t) + got, err := ParseRaw(rawBytes) + if err != nil { + t.Fatalf("ParseRaw failed: %v", err) + } expected1 := `main.fib main.fib @@ -111,6 +113,74 @@ runtime.morestack } } +func testParseRawBad(t *testing.T, errorReason string, contents string) { + _, err := ParseRaw([]byte(contents)) + if err == nil { + t.Errorf("Bad %v should cause error while parsing:%s", errorReason, contents) + } +} + +// Test data for validating that bad input is handled. +const ( + sampleCount = "2" + sampleTime = "10000000" + funcIDLocation = "3" + funcIDSample = "4" + simpleTemplate = ` +Samples: +samples/count cpu/nanoseconds + 2 10000000: 4 5 6 +Locations: + 3: 0xaaaaa funcName :0 s=0 +` +) + +func TestParseRawBadFuncID(t *testing.T) { + { + contents := strings.Replace(simpleTemplate, funcIDSample, "?sample?", -1) + testParseRawBad(t, "funcID in sample", contents) + } + + { + contents := strings.Replace(simpleTemplate, funcIDLocation, "?location?", -1) + testParseRawBad(t, "funcID in location", contents) + } +} + +func TestParseRawBadSample(t *testing.T) { + { + contents := strings.Replace(simpleTemplate, sampleCount, "??", -1) + testParseRawBad(t, "sample count", contents) + } + + { + contents := strings.Replace(simpleTemplate, sampleTime, "??", -1) + testParseRawBad(t, "sample duration", contents) + } +} + +func TestParseRawBadMalformedSample(t *testing.T) { + contents := ` +Samples: +samples/count cpu/nanoseconds + 1 +Locations: + 3: 0xaaaaa funcName :0 s=0 +` + testParseRawBad(t, "malformed sample line", contents) +} + +func TestParseRawBadMalformedLocation(t *testing.T) { + contents := ` +Samples: +samples/count cpu/nanoseconds + 1 10000: 2 +Locations: + 3 +` + testParseRawBad(t, "malformed location line", contents) +} + func TestSplitBySpace(t *testing.T) { tests := []struct { s string From ce791cc2846ee04abc5d9ad00230f17d498ab007 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 17:01:12 -0700 Subject: [PATCH 07/19] Add tests to renderer package --- renderer/flamegraph_test.go | 53 +++++++++++++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/renderer/flamegraph_test.go b/renderer/flamegraph_test.go index 1726ef7..8b43d03 100644 --- a/renderer/flamegraph_test.go +++ b/renderer/flamegraph_test.go @@ -26,6 +26,8 @@ import ( "testing" ) +const testData = "1 2 3 4 5\n" + func TestFindInPatch(t *testing.T) { const realCmd1 = "ls" const realCmd2 = "cat" @@ -73,3 +75,54 @@ func TestFindInPatch(t *testing.T) { } } } + +func TestRunScriptNoInput(t *testing.T) { + out, err := runScript("echo", []string{"1", "2", "3"}, nil) + if err != nil { + t.Fatalf("run echo failed: %v", err) + } + + const want = "1 2 3\n" + if string(out) != want { + t.Errorf("Got unexpected output:\n got %v\n want %v", string(out), want) + } +} + +type scriptFn func([]byte) ([]byte, error) + +func testScriptFound(t *testing.T, sliceToStub []string, f scriptFn) { + // Stub out the scripts that it looks at for the test + origVal := sliceToStub[0] + sliceToStub[0] = "cat" + defer func() { sliceToStub[0] = origVal }() + + out, err := f([]byte(testData)) + if err != nil { + t.Fatalf("Failed to run script: %v", err) + } + + if string(out) != testData { + t.Errorf("Got unexpected output:\n got %v\n want %v", string(out), testData) + } +} + +func testScriptNotFound(t *testing.T, sliceToStub *[]string, f scriptFn) { + origVal := *sliceToStub + *sliceToStub = []string{} + defer func() { *sliceToStub = origVal }() + + _, err := f([]byte(testData)) + if err != errNoPerlScript { + t.Errorf("Unexpected error:\n got %v\n want %v", err, errNoPerlScript) + } +} + +func TestCollapseStacks(t *testing.T) { + testScriptFound(t, stackCollapseScripts, CollapseStacks) + testScriptNotFound(t, &stackCollapseScripts, CollapseStacks) +} + +func TestGenerateFlameGraph(t *testing.T) { + testScriptFound(t, flameGraphScripts, GenerateFlameGraph) + testScriptNotFound(t, &flameGraphScripts, GenerateFlameGraph) +} From 149b2f3ae9f4c4fb2914174c53599556de9ee2df Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 19:14:29 -0700 Subject: [PATCH 08/19] Rewrite main to use pprof and renderer package. Add tests to validate that go-torch runs correctly. --- main.go | 234 +++++++----------------------- main_test.go | 393 ++++++++++++++++++--------------------------------- 2 files changed, 191 insertions(+), 436 deletions(-) diff --git a/main.go b/main.go index 600defc..42fe8e5 100644 --- a/main.go +++ b/main.go @@ -23,226 +23,92 @@ package main import ( - "bytes" - "errors" "fmt" - "net/url" + "io/ioutil" + "log" "os" - "os/exec" - "regexp" "strings" - log "github.com/Sirupsen/logrus" - "github.com/codegangsta/cli" + "github.com/uber/go-torch/pprof" + "github.com/uber/go-torch/renderer" - "github.com/uber/go-torch/graph" - "github.com/uber/go-torch/visualization" + gflags "github.com/jessevdk/go-flags" ) -type torcher struct { - commander +// options are the parameters for go-torch. +type options struct { + PProfOptions pprof.Options + File string `short:"f" long:"file" default:"torch.svg" description:"Output file name (must be .svg)"` + Print bool `short:"p" long:"print" description:"Print the generated svg to stdout instead of writing to file"` + Raw bool `short:"r" long:"raw" description:"Print the raw call graph output to stdout instead of creating a flame graph; use with Brendan Gregg's flame graph perl script (see https://github.com/brendangregg/FlameGraph)"` } -type commander interface { - goTorchCommand(*cli.Context) -} - -type defaultCommander struct { - validator validator - pprofer pprofer - grapher graph.Grapher - visualizer visualization.Visualizer -} - -type validator interface { - validateArgument(string, string, string) error -} - -type defaultValidator struct{} - -type osWrapper interface { - cmdOutput(*exec.Cmd) ([]byte, error) -} - -type defaultOSWrapper struct{} - -type pprofer interface { - runPprofCommand(args ...string) ([]byte, error) -} - -type defaultPprofer struct { - osWrapper -} - -// newTorcher returns a torcher struct with a default commander -func newTorcher() *torcher { - return &torcher{ - commander: newCommander(), +// main is the entry point of the application +func main() { + if err := runWithArgs(os.Args...); err != nil { + log.Fatalf("Failed: %v", err) } } -// newCommander returns a default commander struct with default attributes -func newCommander() commander { - return &defaultCommander{ - validator: new(defaultValidator), - pprofer: newPprofer(), - grapher: graph.NewGrapher(), - visualizer: visualization.NewVisualizer(), +func runWithArgs(args ...string) error { + opts := &options{} + if _, err := gflags.ParseArgs(opts, args); err != nil { + return fmt.Errorf("could not parse options: %v", err) } -} - -func newPprofer() pprofer { - return &defaultPprofer{ - osWrapper: new(defaultOSWrapper), + if err := validateOptions(opts); err != nil { + return fmt.Errorf("invalid options: %v", err) } -} -// main is the entry point of the application -func main() { - t := newTorcher() - t.createAndRunApp() + return runWithOptions(opts) } -// createAndRunApp configures and runs a cli.App -func (t *torcher) createAndRunApp() { - app := cli.NewApp() - app.Name = "go-torch" - app.Usage = "go-torch collects stack traces of a Go application and synthesizes them into into a flame graph" - app.Version = "0.5" - app.Authors = []cli.Author{{Name: "Ben Sandler", Email: "bens@uber.com"}} - app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "url, u", - Value: "http://localhost:8080", - Usage: "base url of your Go program", - }, - cli.StringFlag{ - Name: "suffix, s", - Value: "/debug/pprof/profile", - Usage: "url path of pprof profile", - }, - cli.StringFlag{ - Name: "binaryinput, b", - Value: "", - Usage: "file path of raw binary profile; alternative to having go-torch query pprof endpoint " + - "(binary profile is anything accepted by https://golang.org/cmd/pprof)", - }, - cli.StringFlag{ - Name: "binaryname", - Value: "", - Usage: "file path of the binary that the binaryinput is for, used for pprof inputs", - }, - cli.IntFlag{ - Name: "time, t", - Value: 30, - Usage: "time in seconds to profile for", - }, - cli.StringFlag{ - Name: "file, f", - Value: "torch.svg", - Usage: "ouput file name (must be .svg)", - }, - cli.BoolFlag{ - Name: "print, p", - Usage: "print the generated svg to stdout instead of writing to file", - }, - cli.BoolFlag{ - Name: "raw, r", - Usage: "print the raw call graph output to stdout instead of creating a flame graph; " + - "use with Brendan Gregg's flame graph perl script (see https://github.com/brendangregg/FlameGraph)", - }, +func runWithOptions(opts *options) error { + pprofRawOutput, err := pprof.GetRaw(opts.PProfOptions) + if err != nil { + return fmt.Errorf("could not get raw output from pprof: %v", err) } - app.Action = t.commander.goTorchCommand - app.Run(os.Args) -} -// goTorchCommand executes the 'go-torch' command. -func (com *defaultCommander) goTorchCommand(c *cli.Context) { - outputFile := c.String("file") - binaryName := c.String("binaryname") - binaryInput := c.String("binaryinput") - time := c.Int("time") - stdout := c.Bool("print") - raw := c.Bool("raw") - - err := com.validator.validateArgument(outputFile, `\w+\.svg`, "Output file name must be .svg") + callStacks, err := pprof.ParseRaw(pprofRawOutput) if err != nil { - log.Fatal(err) + return fmt.Errorf("could not parse raw pprof output: %v", err) } - log.Info("Profiling ...") - - var pprofArgs []string - if binaryInput != "" { - if binaryName != "" { - pprofArgs = append(pprofArgs, binaryName) - } - pprofArgs = append(pprofArgs, binaryInput) - } else { - u, err := url.Parse(c.String("url")) - if err != nil { - log.Fatal(err) - } - u.Path = c.String("suffix") - pprofArgs = []string{"-seconds", fmt.Sprint(time), u.String()} + if opts.Raw { + log.Print("Printing raw call graph to stdout") + fmt.Printf("%s", callStacks) + return nil } - out, err := com.pprofer.runPprofCommand(pprofArgs...) + collapsedStacks, err := renderer.CollapseStacks(callStacks) if err != nil { - log.Fatal(err) + return fmt.Errorf("could not collapse stacks: %v", err) } - flamegraphInputBytes, err := newRawParser().Parse(out) + flameGraph, err := renderer.GenerateFlameGraph(collapsedStacks) if err != nil { - log.Fatal(err) - } - flamegraphInput := strings.TrimSpace(string(flamegraphInputBytes)) - if raw { - fmt.Println(flamegraphInput) - log.Info("raw call graph output been printed to stdout") - return + return fmt.Errorf("could not generate flame graph: %v", err) } - if err := com.visualizer.GenerateFlameGraph(flamegraphInput, outputFile, stdout); err != nil { - log.Fatal(err) - } -} -// runPprofCommand runs the `go tool pprof` command to profile an application. -// It returns the output of the underlying command. -func (p *defaultPprofer) runPprofCommand(args ...string) ([]byte, error) { - allArgs := []string{"tool", "pprof", "-raw"} - allArgs = append(allArgs, args...) - - var buf bytes.Buffer - cmd := exec.Command("go", allArgs...) - cmd.Stderr = &buf - out, err := p.osWrapper.cmdOutput(cmd) - if err != nil { - return nil, err + if opts.Print { + log.Print("Printing svg to stdout") + fmt.Printf("%s", flameGraph) + return nil } - // @HACK because 'go tool pprof' doesn't exit on errors with nonzero status codes. - // Ironically, this means that Go's own os/exec package does not detect its errors. - // See issue here https://github.com/golang/go/issues/11510 - if len(out) == 0 { - errText := buf.String() - return nil, errors.New("pprof returned an error. Here is the raw STDERR output:\n" + errText) + log.Printf("Writing svg to %v", opts.File) + if err := ioutil.WriteFile(opts.File, flameGraph, 0666); err != nil { + return fmt.Errorf("could not write output file: %v", err) } - return out, nil -} - -// cmdOutput is a tiny wrapper around cmd.Output to enable test mocking -func (w *defaultOSWrapper) cmdOutput(cmd *exec.Cmd) ([]byte, error) { - return cmd.Output() + return nil } -// validateArgument validates a given command line argument with regex. If the -// argument does not match the expected format, this function returns an error. -func (v *defaultValidator) validateArgument(argument, regex, errorMessage string) error { - match, _ := regexp.MatchString(regex, argument) - if !match { - return errors.New(errorMessage) +func validateOptions(opts *options) error { + if opts.File != "" && !strings.HasSuffix(opts.File, ".svg") { + return fmt.Errorf("output file must end in .svg") + } + if opts.PProfOptions.TimeSeconds < 1 { + return fmt.Errorf("seconds must be an integer greater than 0") } return nil } diff --git a/main_test.go b/main_test.go index a62d00b..85758bd 100644 --- a/main_test.go +++ b/main_test.go @@ -21,294 +21,183 @@ package main import ( - "errors" + "bufio" + "io/ioutil" + "os" + "path/filepath" + "strings" "testing" - "github.com/codegangsta/cli" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" + gflags "github.com/jessevdk/go-flags" ) -func TestCreateAndRunApp(t *testing.T) { - mockCommander := new(mockCommander) - torcher := &torcher{ - commander: mockCommander, - } +const testPProfInputFile = "./pprof/testdata/pprof.1.pb.gz" - var validateContext = func(args mock.Arguments) { - context := args.Get(0).(*cli.Context) - assert.NotNil(t, context) - assert.Equal(t, "go-torch", context.App.Name) +func getDefaultOptions() *options { + opts := &options{} + if _, err := gflags.ParseArgs(opts, nil); err != nil { + panic(err) } - mockCommander.On("goTorchCommand", mock.AnythingOfType("*cli.Context")).Return().Run(validateContext).Once() - - torcher.createAndRunApp() + opts.PProfOptions.BinaryFile = testPProfInputFile + return opts } -func TestCreateAndRunAppDefaultValues(t *testing.T) { - mockCommander := new(mockCommander) - torcher := &torcher{ - commander: mockCommander, +func TestBadArgs(t *testing.T) { + err := runWithArgs("-t", "asd") + if err == nil { + t.Fatalf("expected run with bad arguments to fail") } - validateDefaults := func(args mock.Arguments) { - context := args.Get(0).(*cli.Context) - assert.Equal(t, 30, context.Int("time")) - assert.Equal(t, "http://localhost:8080", context.String("url")) - assert.Equal(t, "/debug/pprof/profile", context.String("suffix")) - assert.Equal(t, "torch.svg", context.String("file")) - assert.Equal(t, "", context.String("binaryinput")) - assert.Equal(t, "", context.String("binaryname")) - assert.Equal(t, false, context.Bool("print")) - assert.Equal(t, false, context.Bool("raw")) - assert.Equal(t, 10, len(context.App.Flags)) + expectedSubstr := []string{ + "could not parse options", + "invalid argument", } - mockCommander.On("goTorchCommand", mock.AnythingOfType( - "*cli.Context")).Return().Run(validateDefaults) - - torcher.createAndRunApp() -} - -func testGoTorchCommand(t *testing.T, url string) { - mockValidator := new(mockValidator) - mockPprofer := new(mockPprofer) - mockGrapher := new(mockGrapher) - mockVisualizer := new(mockVisualizer) - commander := &defaultCommander{ - validator: mockValidator, - pprofer: mockPprofer, - grapher: mockGrapher, - visualizer: mockVisualizer, + for _, substr := range expectedSubstr { + if !strings.Contains(err.Error(), substr) { + t.Errorf("error is missing message: %v", substr) + } } - - samplePprofOutput := []byte("out") - - mockValidator.On("validateArgument", "torch.svg", `\w+\.svg`, - "Output file name must be .svg").Return(nil).Once() - mockPprofer.On("runPprofCommand", []string{"-seconds", "30", "http://localhost/hi"}).Return(samplePprofOutput, nil).Once() - mockGrapher.On("GraphAsText", samplePprofOutput).Return("1;2;3 3", nil).Once() - mockVisualizer.On("GenerateFlameGraph", "1;2;3 3", "torch.svg", false).Return(nil).Once() - - createSampleContext(commander, url) - - mockValidator.AssertExpectations(t) - mockPprofer.AssertExpectations(t) - mockGrapher.AssertExpectations(t) - mockVisualizer.AssertExpectations(t) } -func TestGoTorchCommand(t *testing.T) { - testGoTorchCommand(t, "http://localhost") - - // Trailing slash in url should still work. - testGoTorchCommand(t, "http://localhost/") +func TestMain(t *testing.T) { + os.Args = []string{"--raw", "--binaryinput", testPProfInputFile} + main() + // Test should not fatal. } -func TestGoTorchCommandRawOutput(t *testing.T) { - mockValidator := new(mockValidator) - mockPprofer := new(mockPprofer) - mockGrapher := new(mockGrapher) - mockVisualizer := new(mockVisualizer) - commander := &defaultCommander{ - validator: mockValidator, - pprofer: mockPprofer, - grapher: mockGrapher, - visualizer: mockVisualizer, +func TestInvalidOptions(t *testing.T) { + tests := []struct { + args []string + errorMessage string + }{ + { + args: []string{"--file", "bad.jpg"}, + errorMessage: "must end in .svg", + }, + { + args: []string{"-t", "0"}, + errorMessage: "seconds must be an integer greater than 0", + }, } - samplePprofOutput := []byte("out") - mockValidator.On("validateArgument", "torch.svg", `\w+\.svg`, - "Output file name must be .svg").Return(nil).Once() - mockPprofer.On("runPprofCommand", []string{"-seconds", "30", "http://localhost/hi"}).Return(samplePprofOutput, nil).Once() - mockGrapher.On("GraphAsText", samplePprofOutput).Return("1;2;3 3", nil).Once() - - createSampleContextForRaw(commander) + for _, tt := range tests { + err := runWithArgs(tt.args...) + if err == nil { + t.Errorf("Expected error when running with: %v", tt.args) + continue + } - mockValidator.AssertExpectations(t) - mockPprofer.AssertExpectations(t) - mockGrapher.AssertExpectations(t) - mockVisualizer.AssertExpectations(t) // ensure that mockVisualizer was never called -} - -func TestGoTorchCommandBinaryInput(t *testing.T) { - mockValidator := new(mockValidator) - mockPprofer := new(mockPprofer) - mockGrapher := new(mockGrapher) - mockVisualizer := new(mockVisualizer) - commander := &defaultCommander{ - validator: mockValidator, - pprofer: mockPprofer, - grapher: mockGrapher, - visualizer: mockVisualizer, + if !strings.Contains(err.Error(), tt.errorMessage) { + t.Errorf("Error missing message, got %v want message %v", err.Error(), tt.errorMessage) + } } - - samplePprofOutput := []byte("out") - mockValidator.On("validateArgument", "torch.svg", `\w+\.svg`, - "Output file name must be .svg").Return(nil).Once() - mockPprofer.On("runPprofCommand", []string{"/path/to/binary/file", "/path/to/binary/input"}).Return(samplePprofOutput, nil).Once() - mockGrapher.On("GraphAsText", samplePprofOutput).Return("1;2;3 3", nil).Once() - mockVisualizer.On("GenerateFlameGraph", "1;2;3 3", "torch.svg", false).Return(nil).Once() - - createSampleContextForBinaryInput(commander) - - mockValidator.AssertExpectations(t) - mockPprofer.AssertExpectations(t) - mockGrapher.AssertExpectations(t) - mockVisualizer.AssertExpectations(t) } -func TestValidateArgumentFail(t *testing.T) { - validator := new(defaultValidator) - assert.Error(t, validator.validateArgument("bad bad", `\w+\.svg`, "Message")) -} +func TestRunRaw(t *testing.T) { + opts := getDefaultOptions() + opts.Raw = true -func TestValidateArgumentPass(t *testing.T) { - assert.NotPanics(t, func() { - new(defaultValidator).validateArgument("good.svg", `\w+\.svg`, "Message") - }) -} - -func TestRunPprofCommand(t *testing.T) { - mockOSWrapper := new(mockOSWrapper) - pprofer := defaultPprofer{ - osWrapper: mockOSWrapper, + if err := runWithOptions(opts); err != nil { + t.Fatalf("Run with Raw failed: %v", err) } - - mockOSWrapper.On("cmdOutput", mock.AnythingOfType("*exec.Cmd")).Return([]byte("output"), nil).Once() - - sampleArgs := []string{"-seconds", "15", "http://localhost:8080"} - out, err := pprofer.runPprofCommand(sampleArgs...) - - assert.Equal(t, []byte("output"), out) - assert.NoError(t, err) - mockOSWrapper.AssertExpectations(t) } -func TestRunPprofCommandUnderlyingError(t *testing.T) { - mockOSWrapper := new(mockOSWrapper) - pprofer := defaultPprofer{ - osWrapper: mockOSWrapper, +func getTempFilename(t *testing.T, suffix string) string { + f, err := ioutil.TempFile("", "") + if err != nil { + t.Fatalf("Failed to create temp file: %v", err) } - mockOSWrapper.On("cmdOutput", mock.AnythingOfType("*exec.Cmd")).Return(nil, errors.New("pprof underlying error")).Once() - - sampleArgs := []string{"-seconds", "15", "http://localhost:8080"} - out, err := pprofer.runPprofCommand(sampleArgs...) - - assert.Equal(t, 0, len(out)) - assert.Error(t, err) - mockOSWrapper.AssertExpectations(t) + defer f.Close() + return f.Name() + suffix +} + +func TestRunFile(t *testing.T) { + opts := getDefaultOptions() + opts.File = getTempFilename(t, ".svg") + + withScriptsInPath(t, func() { + if err := runWithOptions(opts); err != nil { + t.Fatalf("Run with Print failed: %v", err) + } + + f, err := os.Open(opts.File) + if err != nil { + t.Errorf("Failed to open output file: %v", err) + } + defer f.Close() + + // Our fake flamegraph scripts just add script names to the output. + reader := bufio.NewReader(f) + line1, err := reader.ReadString('\n') + if err != nil { + t.Errorf("Failed to read line 1 in output file: %v", err) + } + line2, err := reader.ReadString('\n') + if err != nil { + t.Errorf("Failed to read line 2 in output file: %v", err) + } + + if !strings.Contains(line1, "flamegraph.pl") || + !strings.Contains(line2, "stackcollapse.pl") { + t.Errorf("Output file has not been processed by flame graph scripts") + } + }) } -// 'go tool pprof' doesn't exit on errors with nonzero status codes. This test -// ensures that go-torch will detect undrlying errors despite the pprof bug. -// See pprof issue here https://github.com/golang/go/issues/11510 -func TestRunPprofCommandHandlePprofErrorBug(t *testing.T) { - mockOSWrapper := new(mockOSWrapper) - pprofer := defaultPprofer{ - osWrapper: mockOSWrapper, - } - - mockOSWrapper.On("cmdOutput", mock.AnythingOfType("*exec.Cmd")).Return([]byte{}, nil).Once() - - sampleArgs := []string{"-seconds", "15", "http://localhost:8080"} - out, err := pprofer.runPprofCommand(sampleArgs...) +func TestRunBadFile(t *testing.T) { + opts := getDefaultOptions() + opts.File = "/dev/zero/invalid/file" - assert.Equal(t, 0, len(out)) - assert.Error(t, err) - mockOSWrapper.AssertExpectations(t) + withScriptsInPath(t, func() { + if err := runWithOptions(opts); err == nil { + t.Fatalf("Run with bad file expected to fail") + } + }) } -func TestNewTorcher(t *testing.T) { - assert.NotNil(t, newTorcher()) -} +func TestRunPrint(t *testing.T) { + opts := getDefaultOptions() + opts.Print = true -func TestNewCommander(t *testing.T) { - assert.NotNil(t, newCommander()) + withScriptsInPath(t, func() { + if err := runWithOptions(opts); err != nil { + t.Fatalf("Run with Print failed: %v", err) + } + // TODO(prashantv): Verify that output is printed to stdout. + }) } -func createSampleContext(commander *defaultCommander, url string) { - app := cli.NewApp() - app.Name = "go-torch" - app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "url, u", - Value: url, - }, - cli.StringFlag{ - Name: "suffix, s", - Value: "/hi", - }, - cli.IntFlag{ - Name: "time, t", - Value: 30, - }, - cli.StringFlag{ - Name: "file, f", - Value: "torch.svg", - }, +// scriptsPath is used to cache the fake scripts if we've already created it. +var scriptsPath string + +func withScriptsInPath(t *testing.T, f func()) { + oldPath := os.Getenv("PATH") + defer os.Setenv("PATH", oldPath) + + // Create a temporary directory with fake flamegraph scripts if we haven't already. + if scriptsPath == "" { + var err error + scriptsPath, err = ioutil.TempDir("", "go-torch-scripts") + if err != nil { + t.Fatalf("Failed to create temporary scripts dir: %v", err) + } + + // Create scripts in this path. + const scriptContents = `#!/bin/sh + echo $0 + cat + ` + scripts := []string{"stackcollapse.pl", "flamegraph.pl"} + scriptContentsBytes := []byte(scriptContents) + for _, s := range scripts { + scriptFile := filepath.Join(scriptsPath, s) + if err := ioutil.WriteFile(scriptFile, scriptContentsBytes, 0777); err != nil { + t.Errorf("Failed to create script %v: %v", scriptFile, err) + } + } } - app.Action = commander.goTorchCommand - app.Run([]string{"go-torch"}) -} -func createSampleContextForRaw(commander *defaultCommander) { - app := cli.NewApp() - app.Name = "go-torch" - app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "url, u", - Value: "http://localhost", - }, - cli.StringFlag{ - Name: "suffix, s", - Value: "/hi", - }, - cli.IntFlag{ - Name: "time, t", - Value: 30, - }, - cli.StringFlag{ - Name: "file, f", - Value: "torch.svg", - }, - cli.BoolTFlag{ - Name: "raw, r", - }, - } - app.Action = commander.goTorchCommand - app.Run([]string{"go-torch"}) -} - -func createSampleContextForBinaryInput(commander *defaultCommander) { - app := cli.NewApp() - app.Name = "go-torch" - app.Flags = []cli.Flag{ - cli.StringFlag{ - Name: "url, u", - Value: "http://localhost", - }, - cli.StringFlag{ - Name: "suffix, s", - Value: "/hi", - }, - cli.StringFlag{ - Name: "binaryinput, b", - Value: "/path/to/binary/input", - }, - cli.StringFlag{ - Name: "binaryname", - Value: "/path/to/binary/file", - }, - cli.IntFlag{ - Name: "time, t", - Value: 30, - }, - cli.StringFlag{ - Name: "file, f", - Value: "torch.svg", - }, - } - app.Action = commander.goTorchCommand - app.Run([]string{"go-torch"}) + os.Setenv("PATH", scriptsPath+":"+oldPath) + f() } From da1acc69c211d164c6396c3d2f752fa0e7e2670c Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 17:12:57 -0700 Subject: [PATCH 09/19] Delete unused code --- graph/graph.go | 245 ------------- graph/graph_test.go | 550 ---------------------------- graph/mocks.go | 71 ---- mocks.go | 105 ------ raw.go | 173 --------- visualization/mocks.go | 75 ---- visualization/visualization.go | 148 -------- visualization/visualization_test.go | 145 -------- 8 files changed, 1512 deletions(-) delete mode 100644 graph/graph.go delete mode 100644 graph/graph_test.go delete mode 100644 graph/mocks.go delete mode 100644 mocks.go delete mode 100644 raw.go delete mode 100644 visualization/mocks.go delete mode 100644 visualization/visualization.go delete mode 100644 visualization/visualization_test.go diff --git a/graph/graph.go b/graph/graph.go deleted file mode 100644 index e8b8caf..0000000 --- a/graph/graph.go +++ /dev/null @@ -1,245 +0,0 @@ -// Copyright (c) 2015 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package graph transforms a DOT graph text file into the representation -// expected by the visualization package. -// -// The graph is a directed acyclic graph where nodes represent functions and -// directed edges represent how many times a function calls another. -package graph - -import ( - "bytes" - "errors" - "fmt" - "strconv" - "strings" - - log "github.com/Sirupsen/logrus" - ggv "github.com/awalterschulze/gographviz" - "github.com/awalterschulze/gographviz/parser" -) - -var errNoActivity = errors.New("Your application is not doing anything right now. Please try again.") - -// Grapher handles transforming a DOT graph byte array into the -// representation expected by the visualization package. -type Grapher interface { - GraphAsText([]byte) (string, error) -} - -type defaultGrapher struct { - searcher - collectionGetter -} - -type searchArgs struct { - root string - path []ggv.Edge - nodeToOutEdges map[string][]*ggv.Edge - nameToNodes map[string]*ggv.Node - buffer *bytes.Buffer - statusMap map[string]discoveryStatus -} - -type searcher interface { - dfs(args searchArgs) -} - -type defaultSearcher struct { - pathStringer -} - -type collectionGetter interface { - generateNodeToOutEdges(*ggv.Graph) map[string][]*ggv.Edge - getInDegreeZeroNodes(*ggv.Graph) []string -} - -type defaultCollectionGetter struct{} - -type pathStringer interface { - pathAsString([]ggv.Edge, map[string]*ggv.Node) string -} - -type defaultPathStringer struct{} - -// Marking nodes during depth-first search is a standard way of detecting cycles. -// A node is undiscovered before it has been discovered, onstack when it is on the recursion stack, -// and discovered when all of its neighbors have been traversed. A edge terminating at a onstack -// node implies a back edge, which also implies a cycle -// (see: https://en.wikipedia.org/wiki/Cycle_(graph_theory)#Cycle_detection). -type discoveryStatus int - -const ( - undiscovered discoveryStatus = iota - onstack - discovered -) - -// NewGrapher returns a default grapher struct with default attributes -func NewGrapher() Grapher { - return &defaultGrapher{ - searcher: newSearcher(), - collectionGetter: new(defaultCollectionGetter), - } -} - -// newSearcher returns a default searcher struct with a default pathStringer -func newSearcher() *defaultSearcher { - return &defaultSearcher{ - pathStringer: new(defaultPathStringer), - } -} - -// GraphAsText is the standard implementation of Grapher -func (g *defaultGrapher) GraphAsText(dotText []byte) (string, error) { - graphAst, err := parser.ParseBytes(dotText) - if err != nil { - return "", err - } - dag := ggv.NewGraph() // A directed acyclic graph - ggv.Analyse(graphAst, dag) - - if len(dag.Edges.Edges) == 0 { - return "", errNoActivity - } - nodeToOutEdges := g.collectionGetter.generateNodeToOutEdges(dag) - inDegreeZeroNodes := g.collectionGetter.getInDegreeZeroNodes(dag) - nameToNodes := dag.Nodes.Lookup - - buffer := new(bytes.Buffer) - statusMap := make(map[string]discoveryStatus) - - for _, root := range inDegreeZeroNodes { - g.searcher.dfs(searchArgs{ - root: root, - path: nil, - nodeToOutEdges: nodeToOutEdges, - nameToNodes: nameToNodes, - buffer: buffer, - statusMap: statusMap, - }) - } - - return buffer.String(), nil -} - -// generateNodeToOutEdges takes a graph and generates a mapping of nodes to -// edges originating from nodes. -func (c *defaultCollectionGetter) generateNodeToOutEdges(dag *ggv.Graph) map[string][]*ggv.Edge { - nodeToOutEdges := make(map[string][]*ggv.Edge) - for _, edge := range dag.Edges.Edges { - nodeToOutEdges[edge.Src] = append(nodeToOutEdges[edge.Src], edge) - } - return nodeToOutEdges -} - -// getInDegreeZeroNodes takes a graph and returns a list of nodes with -// in-degree of 0. In other words, no edges terminate at these nodes. -func (c *defaultCollectionGetter) getInDegreeZeroNodes(dag *ggv.Graph) []string { - var inDegreeZeroNodes []string - nodeToInDegree := make(map[string]int) - for _, edge := range dag.Edges.Edges { - dst := edge.Dst - nodeToInDegree[dst]++ - } - for _, node := range dag.Nodes.Nodes { - // @HACK This is a hack to fix a bug with gographviz where a cluster - // 'L' is being parsed as a node. This just checks that all node names - // begin with N. - correctPrefix := strings.HasPrefix(node.Name, "N") - if correctPrefix && nodeToInDegree[node.Name] == 0 { - inDegreeZeroNodes = append(inDegreeZeroNodes, node.Name) - } - } - return inDegreeZeroNodes -} - -// dfs performs a depth-first search traversal of the graph starting from a -// given root node. When a node with no outgoing edges is reached, the path -// taken to that node is written to a buffer. -func (s *defaultSearcher) dfs(args searchArgs) { - outEdges := args.nodeToOutEdges[args.root] - if args.statusMap[args.root] == onstack { - log.Warn("The input call graph contains a cycle. This can't be represented in a " + - "flame graph, so this path will be ignored. For your record, the ignored path " + - "is:\n" + strings.TrimSpace(s.pathStringer.pathAsString(args.path, args.nameToNodes))) - return - } - if len(outEdges) == 0 { - args.buffer.WriteString(s.pathStringer.pathAsString(args.path, args.nameToNodes)) - args.statusMap[args.root] = discovered - return - } - args.statusMap[args.root] = onstack - for _, edge := range outEdges { - s.dfs(searchArgs{ - root: edge.Dst, - path: append(args.path, *edge), - nodeToOutEdges: args.nodeToOutEdges, - nameToNodes: args.nameToNodes, - buffer: args.buffer, - statusMap: args.statusMap, - }) - } - args.statusMap[args.root] = discovered -} - -// pathAsString takes a path and a mapping of node names to node structs and -// generates the string representation of the path expected by the -// visualization package. -func (p *defaultPathStringer) pathAsString(path []ggv.Edge, nameToNodes map[string]*ggv.Node) string { - var ( - pathBuffer bytes.Buffer - weightSum int - ) - for _, edge := range path { - // If the function call represented by the edge happened very rarely, - // the edge's weight will not be recorded. The edge's label will always - // be recorded. - if weightStr, ok := edge.Attrs["weight"]; ok { - weight, err := strconv.Atoi(weightStr) - if err != nil { // This should never happen - log.Panic(err) - } - weightSum += weight - } - functionLabel := getFormattedFunctionLabel(nameToNodes[edge.Src]) - pathBuffer.WriteString(functionLabel + ";") - } - if len(path) >= 1 { - lastEdge := path[len(path)-1] - lastFunctionLabel := getFormattedFunctionLabel(nameToNodes[lastEdge.Dst]) - pathBuffer.WriteString(lastFunctionLabel + " ") - } - pathBuffer.WriteString(fmt.Sprint(weightSum)) - pathBuffer.WriteString("\n") - - return pathBuffer.String() -} - -// getFormattedFunctionLabel takes a node and returns a formatted function -// label. -func getFormattedFunctionLabel(node *ggv.Node) string { - label := node.Attrs["tooltip"] - label = strings.Replace(label, `\n`, " ", -1) - label = strings.Replace(label, `"`, "", -1) - return label -} diff --git a/graph/graph_test.go b/graph/graph_test.go deleted file mode 100644 index 233ddde..0000000 --- a/graph/graph_test.go +++ /dev/null @@ -1,550 +0,0 @@ -// Copyright (c) 2015 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package graph - -import ( - "bytes" - "testing" - - ggv "github.com/awalterschulze/gographviz" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestPathAsString(t *testing.T) { - g := testGraphWithTooltipAndWeight() - - eMap := g.Edges.SrcToDsts - path := []ggv.Edge{*eMap["N1"]["N2"], *eMap["N2"]["N3"], *eMap["N3"]["N4"]} - - pathString := new(defaultPathStringer).pathAsString(path, g.Nodes.Lookup) - - assert.Equal(t, "function1;function2;function3;function4 9\n", pathString) -} - -func TestPathAsStringWithEmptyPath(t *testing.T) { - path := []ggv.Edge{} - - pathString := new(defaultPathStringer).pathAsString(path, map[string]*ggv.Node{}) - assert.Equal(t, "0\n", pathString) -} - -func TestPathAsStringWithNoWeightEdges(t *testing.T) { - g := testGraphWithTooltip() - - eMap := g.Edges.SrcToDsts - path := []ggv.Edge{*eMap["N1"]["N2"], *eMap["N2"]["N3"], *eMap["N3"]["N4"]} - - pathString := new(defaultPathStringer).pathAsString(path, g.Nodes.Lookup) - - assert.Equal(t, "function1;function2;function3;function4 0\n", pathString) -} - -func TestDFS(t *testing.T) { - g := testSingleRootGraph() - eMap := g.Edges.SrcToDsts - - nodeToOutEdges := map[string][]*ggv.Edge{ - "N1": {eMap["N1"]["N2"], eMap["N1"]["N3"], eMap["N1"]["N4"]}, - "N2": {eMap["N2"]["N3"]}, - "N4": {eMap["N4"]["N3"]}, - } - - buffer := new(bytes.Buffer) - mockPathStringer := new(mockPathStringer) - anythingType := mock.AnythingOfType("map[string]*gographviz.Node") - pathOne := []ggv.Edge{*eMap["N1"]["N2"], *eMap["N2"]["N3"]} - pathTwo := []ggv.Edge{*eMap["N1"]["N3"]} - pathThree := []ggv.Edge{*eMap["N1"]["N4"], *eMap["N4"]["N3"]} - - mockPathStringer.On("pathAsString", pathOne, anythingType).Return("N1;N2;N3 3\n").Once() - mockPathStringer.On("pathAsString", pathTwo, anythingType).Return("N1;N3 2\n").Once() - mockPathStringer.On("pathAsString", pathThree, anythingType).Return("N1;N4;N3 8\n").Once() - - searcherWithTestStringer := &defaultSearcher{ - pathStringer: mockPathStringer, - } - searcherWithTestStringer.dfs(searchArgs{ - root: "N1", - path: []ggv.Edge{}, - nodeToOutEdges: nodeToOutEdges, - nameToNodes: g.Nodes.Lookup, - buffer: buffer, - statusMap: make(map[string]discoveryStatus), - }) - - correctOutput := "N1;N2;N3 3\nN1;N3 2\nN1;N4;N3 8\n" - actualOutput := buffer.String() - - assert.Equal(t, correctOutput, actualOutput) - mockPathStringer.AssertExpectations(t) -} - -func TestDFSAlmostEmptyGraph(t *testing.T) { - g := ggv.NewGraph() - g.SetName("G") - g.AddNode("G", "N1", nil) - g.SetDir(true) - - nodeToOutEdges := map[string][]*ggv.Edge{} - buffer := new(bytes.Buffer) - - mockPathStringer := new(mockPathStringer) - anythingType := mock.AnythingOfType("map[string]*gographviz.Node") - - mockPathStringer.On("pathAsString", []ggv.Edge{}, anythingType).Return("").Once() - - searcherWithTestStringer := &defaultSearcher{ - pathStringer: mockPathStringer, - } - searcherWithTestStringer.dfs(searchArgs{ - root: "N1", - path: []ggv.Edge{}, - nodeToOutEdges: nodeToOutEdges, - nameToNodes: g.Nodes.Lookup, - buffer: buffer, - statusMap: make(map[string]discoveryStatus), - }) - - correctOutput := "" - actualOutput := buffer.String() - - assert.Equal(t, correctOutput, actualOutput) - mockPathStringer.AssertExpectations(t) -} - -func TestDFSMultipleRootsLeaves(t *testing.T) { - g := testMultiRootGraph() - - eMap := g.Edges.SrcToDsts - - nodeToOutEdges := map[string][]*ggv.Edge{ - "N1": {eMap["N1"]["N2"], eMap["N1"]["N3"]}, - "N4": {eMap["N4"]["N5"], eMap["N4"]["N6"]}, - "N6": {eMap["N6"]["N5"]}, - } - - buffer := new(bytes.Buffer) - mockPathStringer := new(mockPathStringer) - anythingType := mock.AnythingOfType("map[string]*gographviz.Node") - pathOne := []ggv.Edge{*eMap["N1"]["N2"]} - pathTwo := []ggv.Edge{*eMap["N1"]["N3"]} - pathThree := []ggv.Edge{*eMap["N4"]["N5"]} - pathFour := []ggv.Edge{*eMap["N4"]["N6"], *eMap["N6"]["N5"]} - - mockPathStringer.On("pathAsString", pathOne, anythingType).Return("N1;N2 3\n").Once() - mockPathStringer.On("pathAsString", pathTwo, anythingType).Return("N1;N3 2\n").Once() - mockPathStringer.On("pathAsString", pathThree, anythingType).Return("N4;N5 8\n").Once() - mockPathStringer.On("pathAsString", pathFour, anythingType).Return("N4;N6;N5 7\n").Once() - - searcherWithTestStringer := &defaultSearcher{ - pathStringer: mockPathStringer, - } - - searcherWithTestStringer.dfs(searchArgs{ - root: "N1", - path: []ggv.Edge{}, - nodeToOutEdges: nodeToOutEdges, - nameToNodes: g.Nodes.Lookup, - buffer: buffer, - statusMap: make(map[string]discoveryStatus), - }) - searcherWithTestStringer.dfs(searchArgs{ - root: "N4", - path: []ggv.Edge{}, - nodeToOutEdges: nodeToOutEdges, - nameToNodes: g.Nodes.Lookup, - buffer: buffer, - statusMap: make(map[string]discoveryStatus), - }) - - correctOutput := "N1;N2 3\nN1;N3 2\nN4;N5 8\nN4;N6;N5 7\n" - actualOutput := buffer.String() - - assert.Equal(t, correctOutput, actualOutput) - mockPathStringer.AssertExpectations(t) -} - -func TestDFSCyclicGraph(t *testing.T) { - g := testGraphWithCycles() - eMap := g.Edges.SrcToDsts - - nodeToOutEdges := map[string][]*ggv.Edge{ - "N1": {eMap["N1"]["N2"], eMap["N1"]["N4"]}, - "N2": {eMap["N2"]["N3"], eMap["N2"]["N4"]}, - "N3": {eMap["N3"]["N4"], eMap["N3"]["N5"]}, - "N5": {eMap["N5"]["N2"]}, - } - - buffer := new(bytes.Buffer) - mockPathStringer := new(mockPathStringer) - anythingType := mock.AnythingOfType("map[string]*gographviz.Node") - pathOne := []ggv.Edge{*eMap["N1"]["N2"], *eMap["N2"]["N3"], *eMap["N3"]["N4"]} - pathTwo := []ggv.Edge{*eMap["N1"]["N4"]} - pathThree := []ggv.Edge{*eMap["N1"]["N2"], *eMap["N2"]["N4"]} - - cycleOne := []ggv.Edge{*eMap["N1"]["N2"], *eMap["N2"]["N3"], *eMap["N3"]["N5"], - *eMap["N5"]["N2"]} - - mockPathStringer.On("pathAsString", pathOne, anythingType).Return("N1;N2;N3;N4 4\n").Once() - mockPathStringer.On("pathAsString", pathTwo, anythingType).Return("N1;N4 2\n").Once() - mockPathStringer.On("pathAsString", pathThree, anythingType).Return("N2;N4 2\n").Once() - - mockPathStringer.On("pathAsString", cycleOne, anythingType).Return("should not include\n").Once() - - searcherWithTestStringer := &defaultSearcher{ - pathStringer: mockPathStringer, - } - searcherWithTestStringer.dfs(searchArgs{ - root: "N1", - path: []ggv.Edge{}, - nodeToOutEdges: nodeToOutEdges, - nameToNodes: g.Nodes.Lookup, - buffer: buffer, - statusMap: make(map[string]discoveryStatus), - }) - - correctOutput := "N1;N2;N3;N4 4\nN2;N4 2\nN1;N4 2\n" - actualOutput := buffer.String() - - assert.Equal(t, correctOutput, actualOutput) - mockPathStringer.AssertExpectations(t) -} - -func TestGetInDegreeZeroNodes(t *testing.T) { - g := testMultiRootGraph() - - correctInDegreeZeroNodes := []string{"N1", "N4"} - actualInDegreeZeroNodes := new(defaultCollectionGetter).getInDegreeZeroNodes(g) - assert.Equal(t, correctInDegreeZeroNodes, actualInDegreeZeroNodes) -} - -func TestGetInDegreeZeroNodesEmptyGraph(t *testing.T) { - g := ggv.NewGraph() - g.SetName("G") - g.SetDir(true) - - var correctInDegreeZeroNodes []string - actualInDegreeZeroNodes := new(defaultCollectionGetter).getInDegreeZeroNodes(g) - assert.Equal(t, correctInDegreeZeroNodes, actualInDegreeZeroNodes) -} - -func TestGetInDegreeZeroNodesIgnoreClusterNodes(t *testing.T) { - g := testGraphWithClusterNodes() - - correctInDegreeZeroNodes := []string{"N1"} - actualInDegreeZeroNodes := new(defaultCollectionGetter).getInDegreeZeroNodes(g) - assert.Equal(t, correctInDegreeZeroNodes, actualInDegreeZeroNodes) -} - -func TestGenerateNodeToOutEdges(t *testing.T) { - g := testMultiRootGraph() - - eMap := g.Edges.SrcToDsts - - correctNodeToOutEdges := map[string][]*ggv.Edge{ - "N1": {eMap["N1"]["N2"], eMap["N1"]["N3"]}, - "N4": {eMap["N4"]["N5"], eMap["N4"]["N6"]}, - "N6": {eMap["N6"]["N5"]}, - } - actualNodeToOutEdges := new(defaultCollectionGetter).generateNodeToOutEdges(g) - assert.Equal(t, correctNodeToOutEdges, actualNodeToOutEdges) -} - -func TestGenerateNodeToOutEdgesEmptyGraph(t *testing.T) { - g := ggv.NewGraph() - g.SetName("G") - g.SetDir(true) - - correctNodeToOutEdges := make(map[string][]*ggv.Edge) - actualNodeToOutEdges := new(defaultCollectionGetter).generateNodeToOutEdges(g) - assert.Equal(t, correctNodeToOutEdges, actualNodeToOutEdges) -} - -func TestGraphAsText(t *testing.T) { - mockSearcher := new(mockSearcher) - mockCollectionGetter := new(mockCollectionGetter) - grapher := &defaultGrapher{ - searcher: mockSearcher, - collectionGetter: mockCollectionGetter, - } - - graphAsTextInput := []byte(`digraph "unnamed" { - node [style=filled fillcolor="#f8f8f8"] - N1 [tooltip="N1"] - N2 [tooltip="N2"] - N3 [tooltip="N3"] - N4 [tooltip="N4"] - N5 [tooltip="N5"] - N6 [tooltip="N6"] - N1 -> N2 [weight=1] - N1 -> N3 [weight=2] - N4 -> N5 [weight=1] - N4 -> N6 [weight=4] - N6 -> N5 [weight=4] - }`) - - fakeWriteToBuffer := func(args mock.Arguments) { - searchArgs := args.Get(0).(searchArgs) - if searchArgs.root == "N1" { - searchArgs.buffer.WriteString("N1;N2 1\nN1;N3 2\n") - } else { - searchArgs.buffer.WriteString("N4;N5 1\nN4;N6;N5 8\n") - } - } - - mockSearcher.On("dfs", mock.AnythingOfType("searchArgs")).Return().Run(fakeWriteToBuffer).Twice() - mockCollectionGetter.On("generateNodeToOutEdges", - mock.AnythingOfType("*gographviz.Graph")).Return(nil).Once() // We can return nil since the mock dfs will ignore this - mockCollectionGetter.On("getInDegreeZeroNodes", - mock.AnythingOfType("*gographviz.Graph")).Return([]string{"N1", "N4"}).Once() - - correctGraphAsText := "N1;N2 1\nN1;N3 2\nN4;N5 1\nN4;N6;N5 8\n" - - actualGraphAsText, err := grapher.GraphAsText(graphAsTextInput) - assert.NoError(t, err) - assert.Equal(t, correctGraphAsText, actualGraphAsText) - mockSearcher.AssertExpectations(t) -} - -func TestNewGrapher(t *testing.T) { - assert.NotNil(t, NewGrapher()) -} - -func TestNewSearcher(t *testing.T) { - assert.NotNil(t, newSearcher()) -} - -// The returned graph, represented in ascii: -// +----+ +----+ -// | N2 | <-- | N1 | -// +----+ +----+ -// | -// | -// v -// +----+ -// | N3 | -// +----+ -// +----+ -// | N4 | -+ -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N6 | | -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N5 | <+ -// +----+ -func testMultiRootGraph() *ggv.Graph { - g := ggv.NewGraph() - g.SetName("G") - g.SetDir(true) - g.AddNode("G", "N1", nil) - g.AddNode("G", "N2", nil) - g.AddNode("G", "N3", nil) - g.AddNode("G", "N4", nil) - g.AddNode("G", "N5", nil) - g.AddNode("G", "N6", nil) - g.AddEdge("N1", "N2", true, nil) - g.AddEdge("N1", "N3", true, nil) - g.AddEdge("N4", "N5", true, nil) - g.AddEdge("N4", "N6", true, nil) - g.AddEdge("N6", "N5", true, nil) - return g -} - -// The returned graph, represented in ascii: -// +----+ +----+ -// +> | N4 | <-- | N1 | -// | +----+ +----+ -// | ^ | -// | | | -// | | v -// | | +----+ -// | +------- | N2 | <+ -// | +----+ | -// | | | -// | | | -// | v | -// | +----+ | -// +------------ | N3 | | -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N5 | -+ -// +----+ -func testGraphWithCycles() *ggv.Graph { - g := ggv.NewGraph() - g.SetName("G") - g.SetDir(true) - g.AddNode("G", "N1", nil) - g.AddNode("G", "N2", nil) - g.AddNode("G", "N3", nil) - g.AddNode("G", "N4", nil) - g.AddNode("G", "N5", nil) - g.AddEdge("N1", "N2", true, nil) - g.AddEdge("N1", "N4", true, nil) - g.AddEdge("N2", "N3", true, nil) - g.AddEdge("N2", "N4", true, nil) - g.AddEdge("N3", "N4", true, nil) - g.AddEdge("N3", "N5", true, nil) - g.AddEdge("N5", "N2", true, nil) - return g -} - -// The returned graph, represented in ascii: -// +----+ -// | N1 | -+ -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N2 | | -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N3 | | -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N4 | <+ -// +----+ -func testGraphWithTooltipAndWeight() *ggv.Graph { - g := ggv.NewGraph() - g.SetName("G") - g.SetDir(true) - g.AddNode("G", "N1", map[string]string{"tooltip": "function1"}) - g.AddNode("G", "N2", map[string]string{"tooltip": "function2"}) - g.AddNode("G", "N3", map[string]string{"tooltip": "function3"}) - g.AddNode("G", "N4", map[string]string{"tooltip": "function4"}) - g.AddEdge("N1", "N2", true, map[string]string{"weight": "5"}) - g.AddEdge("N2", "N3", true, map[string]string{"weight": "2"}) - g.AddEdge("N3", "N4", true, map[string]string{"weight": "2"}) - g.AddEdge("N1", "N4", true, map[string]string{"weight": "1"}) - return g -} - -// The returned graph, represented in ascii: -// +----+ -// | N1 | -+ -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N2 | | -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N3 | | -// +----+ | -// | | -// | | -// v | -// +----+ | -// | N4 | <+ -// +----+ -func testGraphWithTooltip() *ggv.Graph { - g := ggv.NewGraph() - g.SetName("G") - g.SetDir(true) - g.AddNode("G", "N1", map[string]string{"tooltip": "function1"}) - g.AddNode("G", "N2", map[string]string{"tooltip": "function2"}) - g.AddNode("G", "N3", map[string]string{"tooltip": "function3"}) - g.AddNode("G", "N4", map[string]string{"tooltip": "function4"}) - g.AddEdge("N1", "N2", true, nil) - g.AddEdge("N2", "N3", true, nil) - g.AddEdge("N3", "N4", true, nil) - g.AddEdge("N1", "N4", true, nil) - return g -} - -// The returned graph, represented in ascii: -// +----+ +----+ -// | N4 | <-- | N1 | -+ -// +----+ +----+ | -// | | | -// | | | -// | v | -// | +----+ | -// | | N2 | | -// | +----+ | -// | | | -// | | | -// | v | -// | +----+ | -// +------> | N3 | <+ -// +----+ -func testSingleRootGraph() *ggv.Graph { - g := ggv.NewGraph() - g.SetName("G") - g.SetDir(true) - g.AddNode("G", "N1", nil) - g.AddNode("G", "N2", nil) - g.AddNode("G", "N3", nil) - g.AddNode("G", "N4", nil) - g.AddEdge("N1", "N2", true, nil) - g.AddEdge("N2", "N3", true, nil) - g.AddEdge("N4", "N3", true, nil) - g.AddEdge("N1", "N4", true, nil) - g.AddEdge("N1", "N3", true, nil) - return g -} - -// The returned graph, represented in ascii: -// +----------+ -// | Ignoreme | -// +----------+ -// +----+ +----------+ -// | N2 | <-- | N1 | -// +----+ +----------+ -// | -// | -// v -// +----------+ -// | N3 | -// +----------+ -func testGraphWithClusterNodes() *ggv.Graph { - g := ggv.NewGraph() - g.SetName("G") - g.SetDir(true) - g.AddNode("G", "N1", nil) - g.AddNode("G", "N2", nil) - g.AddNode("G", "N3", nil) - g.AddNode("G", "Ignore me!", nil) - g.AddEdge("N1", "N2", true, nil) - g.AddEdge("N1", "N3", true, nil) - return g -} diff --git a/graph/mocks.go b/graph/mocks.go deleted file mode 100644 index 4ee7a91..0000000 --- a/graph/mocks.go +++ /dev/null @@ -1,71 +0,0 @@ -// Copyright (c) 2015 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package graph - -import ( - ggv "github.com/awalterschulze/gographviz" - "github.com/stretchr/testify/mock" -) - -type mockSearcher struct { - mock.Mock -} - -func (m *mockSearcher) dfs(args searchArgs) { - m.Called(args) -} - -type mockCollectionGetter struct { - mock.Mock -} - -func (m *mockCollectionGetter) generateNodeToOutEdges(_a0 *ggv.Graph) map[string][]*ggv.Edge { - ret := m.Called(_a0) - - var r0 map[string][]*ggv.Edge - if ret.Get(0) != nil { - r0 = ret.Get(0).(map[string][]*ggv.Edge) - } - - return r0 -} -func (m *mockCollectionGetter) getInDegreeZeroNodes(_a0 *ggv.Graph) []string { - ret := m.Called(_a0) - - var r0 []string - if ret.Get(0) != nil { - r0 = ret.Get(0).([]string) - } - - return r0 -} - -type mockPathStringer struct { - mock.Mock -} - -func (m *mockPathStringer) pathAsString(_a0 []ggv.Edge, _a1 map[string]*ggv.Node) string { - ret := m.Called(_a0, _a1) - - r0 := ret.Get(0).(string) - - return r0 -} diff --git a/mocks.go b/mocks.go deleted file mode 100644 index 9f178a7..0000000 --- a/mocks.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright (c) 2015 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package main - -import ( - "os/exec" - - "github.com/codegangsta/cli" - "github.com/stretchr/testify/mock" -) - -type mockVisualizer struct { - mock.Mock -} - -func (m *mockVisualizer) GenerateFlameGraph(_a0 string, _a1 string, _a2 bool) error { - ret := m.Called(_a0, _a1, _a2) - - r0 := ret.Error(0) - - return r0 -} - -type mockGrapher struct { - mock.Mock -} - -func (m *mockGrapher) GraphAsText(_a0 []byte) (string, error) { - ret := m.Called(_a0) - - r0 := ret.Get(0).(string) - r1 := ret.Error(1) - - return r0, r1 -} - -type mockCommander struct { - mock.Mock -} - -func (m *mockCommander) goTorchCommand(_a0 *cli.Context) { - m.Called(_a0) -} - -type mockValidator struct { - mock.Mock -} - -func (m *mockValidator) validateArgument(_a0 string, _a1 string, _a2 string) error { - ret := m.Called(_a0, _a1, _a2) - - r0 := ret.Error(0) - - return r0 -} - -type mockPprofer struct { - mock.Mock -} - -type mockOSWrapper struct { - mock.Mock -} - -func (m *mockOSWrapper) cmdOutput(_a0 *exec.Cmd) ([]byte, error) { - ret := m.Called(_a0) - - var r0 []byte - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - r1 := ret.Error(1) - - return r0, r1 -} - -func (m *mockPprofer) runPprofCommand(args ...string) ([]byte, error) { - ret := m.Called(args) - - var r0 []byte - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - r1 := ret.Error(1) - - return r0, r1 -} diff --git a/raw.go b/raw.go deleted file mode 100644 index 2d3ebd4..0000000 --- a/raw.go +++ /dev/null @@ -1,173 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "errors" - "fmt" - "io" - "os/exec" - "strconv" - "strings" - "time" -) - -type readMode int - -const ( - ignore readMode = iota - samplesHeader - samples - locations - mappings -) - -type funcID int - -type rawParser struct { - funcName map[funcID]string - records []*stackRecord -} - -func newRawParser() *rawParser { - return &rawParser{ - funcName: make(map[funcID]string), - } -} - -func (p *rawParser) Parse(input []byte) ([]byte, error) { - var mode readMode - reader := bufio.NewReader(bytes.NewReader(input)) - - for { - line, err := reader.ReadString('\n') - line = strings.TrimSpace(line) - if err != nil { - if err == io.EOF { - break - } - return nil, err - } - - switch mode { - case ignore: - if strings.HasPrefix(line, "Samples") { - mode = samplesHeader - continue - } - case samplesHeader: - mode = samples - case samples: - if strings.HasPrefix(line, "Locations") { - mode = locations - continue - } - p.addSample(line) - case locations: - if strings.HasPrefix(line, "Mappings") { - mode = mappings - continue - } - p.addLocation(line) - case mappings: - // Nothing to process. - } - } - - pr, pw := io.Pipe() - go p.Print(pw) - return p.CollapseStacks(pr) -} - -func (p *rawParser) Print(w io.WriteCloser) { - for _, r := range p.records { - r.Serialize(p.funcName, w) - fmt.Fprintln(w) - } - w.Close() -} - -func findStackCollapse() string { - for _, v := range []string{"stackcollapse.pl", "./stackcollapse.pl", "./FlameGraph/stackcollapse.pl"} { - if path, err := exec.LookPath(v); err == nil { - return path - } - } - return "" -} - -func (p *rawParser) CollapseStacks(stacks io.Reader) ([]byte, error) { - stackCollapse := findStackCollapse() - if stackCollapse == "" { - return nil, errors.New("stackcollapse.pl not found") - } - - cmd := exec.Command(stackCollapse) - cmd.Stdin = stacks - return cmd.Output() -} - -func (p *rawParser) addSample(line string) { - // Parse a sample which looks like: - // 1 10000000: 1 2 3 4 - parts := splitIgnoreEmpty(line, " ") - - samples, err := strconv.Atoi(parts[0]) - if err != nil { - panic(err) - } - - duration, err := strconv.Atoi(strings.TrimSuffix(parts[1], ":")) - if err != nil { - panic(err) - } - - var stack []funcID - for _, fIDStr := range parts[2:] { - stack = append(stack, toFuncID(fIDStr)) - } - - p.records = append(p.records, &stackRecord{samples, time.Duration(duration), stack}) -} - -func (p *rawParser) addLocation(line string) { - // 292: 0x49dee1 github.com/uber/tchannel/golang.(*Frame).ReadIn :0 s=0 - parts := splitIgnoreEmpty(line, " ") - funcID := toFuncID(strings.TrimSuffix(parts[0], ":")) - p.funcName[funcID] = parts[2] -} - -type stackRecord struct { - samples int - duration time.Duration - stack []funcID -} - -func (r *stackRecord) Serialize(funcName map[funcID]string, w io.Writer) { - // Go backwards through the stack - for _, funcID := range r.stack { - fmt.Fprintln(w, funcName[funcID]) - } - fmt.Fprintln(w, r.samples) -} - -func toFuncID(s string) funcID { - i, err := strconv.Atoi(s) - if err != nil { - panic(err) - } - return funcID(i) -} - -// splitIgnoreEmpty does a strings.Split and then removes all empty strings. -func splitIgnoreEmpty(s string, splitter string) []string { - vals := strings.Split(s, splitter) - var res []string - for _, v := range vals { - if len(v) != 0 { - res = append(res, v) - } - } - - return res -} diff --git a/visualization/mocks.go b/visualization/mocks.go deleted file mode 100644 index 4c5ee03..0000000 --- a/visualization/mocks.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2015 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package visualization - -import ( - "os/exec" - - "github.com/stretchr/testify/mock" -) - -type mockOSWrapper struct { - mock.Mock -} - -func (m *mockOSWrapper) execLookPath(_a0 string) (string, error) { - println(_a0) - ret := m.Called(_a0) - - r0 := ret.Get(0).(string) - r1 := ret.Error(1) - - return r0, r1 -} -func (m *mockOSWrapper) cmdOutput(_a0 *exec.Cmd) ([]byte, error) { - ret := m.Called(_a0) - - var r0 []byte - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - r1 := ret.Error(1) - - return r0, r1 -} - -type mockExecutor struct { - mock.Mock -} - -func (m *mockExecutor) createFile(_a0 string, _a1 []byte) error { - ret := m.Called(_a0, _a1) - - r0 := ret.Error(0) - - return r0 -} -func (m *mockExecutor) runPerlScript(_a0 string) ([]byte, error) { - ret := m.Called(_a0) - - var r0 []byte - if ret.Get(0) != nil { - r0 = ret.Get(0).([]byte) - } - r1 := ret.Error(1) - - return r0, r1 -} diff --git a/visualization/visualization.go b/visualization/visualization.go deleted file mode 100644 index 2343ba5..0000000 --- a/visualization/visualization.go +++ /dev/null @@ -1,148 +0,0 @@ -// Copyright (c) 2015 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -// Package visualization handles the generation of the -// flame graph visualization. -package visualization - -import ( - "errors" - "fmt" - "os" - "os/exec" - "strings" - - log "github.com/Sirupsen/logrus" -) - -var errNoPerlScript = errors.New("Cannot find flamegraph script in the PATH or current " + - "directory. You can download the script at https://github.com/brendangregg/FlameGraph. " + - "Alternatively, you can run go-torch with the --raw flag.") - -// Visualizer takes a graph in the format specified at -// https://github.com/brendangregg/FlameGraph and creates a svg flame graph -// using Brendan Gregg's flame graph perl script -type Visualizer interface { - GenerateFlameGraph(string, string, bool) error -} - -type defaultVisualizer struct { - executor -} - -type osWrapper interface { - execLookPath(string) (string, error) - cmdOutput(*exec.Cmd) ([]byte, error) -} - -type defaultOSWrapper struct{} - -type executor interface { - createFile(string, []byte) error - runPerlScript(string) ([]byte, error) -} - -type defaultExecutor struct { - osWrapper -} - -func newExecutor() executor { - return &defaultExecutor{ - osWrapper: new(defaultOSWrapper), - } -} - -// NewVisualizer returns a visualizer struct with default fileCreator -func NewVisualizer() Visualizer { - return &defaultVisualizer{ - executor: newExecutor(), - } -} - -// GenerateFlameGraph is the standard implementation of Visualizer -func (v *defaultVisualizer) GenerateFlameGraph(graphInput, outputFilePath string, stdout bool) error { - out, err := v.executor.runPerlScript(graphInput) - if err != nil { - return err - } - if stdout { - fmt.Println(string(out)) - log.Info("flame graph has been printed to stdout") - return nil - } - if err = v.executor.createFile(outputFilePath, out); err != nil { - return err - } - log.Info("flame graph has been created as " + outputFilePath) - - return nil -} - -// runPerlScript checks whether the flamegraph script exists in the PATH or current directory and -// then executes it with the graphInput. -func (e *defaultExecutor) runPerlScript(graphInput string) ([]byte, error) { - cwd, err := os.Getwd() - if err != nil { - return nil, err - } - possibilities := []string{"flamegraph.pl", cwd + "/flamegraph.pl", "flame-graph-gen"} - perlScript := "" - for _, path := range possibilities { - perlScript, err = e.osWrapper.execLookPath(path) - // found a valid script - if err == nil { - break - } - } - if err != nil { - return nil, errNoPerlScript - } - cmd := exec.Command(perlScript, os.Stdin.Name()) - cmd.Stdin = strings.NewReader(graphInput) - out, err := e.osWrapper.cmdOutput(cmd) - return out, err -} - -// execLookPath is a tiny wrapper around exec.LookPath to enable test mocking -func (w *defaultOSWrapper) execLookPath(path string) (fullPath string, err error) { - return exec.LookPath(path) -} - -// cmdOutput is a tiny wrapper around cmd.Output to enable test mocking -func (w *defaultOSWrapper) cmdOutput(cmd *exec.Cmd) ([]byte, error) { - return cmd.Output() -} - -// createFile creates a file at a given path with given contents. If a file -// already exists at the path, it will be overwritten and replaced. -func (e *defaultExecutor) createFile(filePath string, fileContents []byte) error { - os.Remove(filePath) - file, err := os.Create(filePath) - if err != nil { - return err - } - defer file.Close() - - _, err = file.Write(fileContents) - if err != nil { - return err - } - return nil -} diff --git a/visualization/visualization_test.go b/visualization/visualization_test.go deleted file mode 100644 index aed1316..0000000 --- a/visualization/visualization_test.go +++ /dev/null @@ -1,145 +0,0 @@ -// Copyright (c) 2015 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package visualization - -import ( - "errors" - "io/ioutil" - "os" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestCreateFile(t *testing.T) { - new(defaultExecutor).createFile(".text.svg", []byte("the contents")) - - // teardown - defer os.Remove(".text.svg") - - actualContents, err := ioutil.ReadFile(".text.svg") - assert.NoError(t, err) - assert.Equal(t, "the contents", string(actualContents)) -} - -func TestCreateFileOverwriteExisting(t *testing.T) { - new(defaultExecutor).createFile(".text.svg", []byte("delete me")) - new(defaultExecutor).createFile(".text.svg", []byte("correct answer")) - - // teardown - defer os.Remove(".text.svg") - - actualContents, err := ioutil.ReadFile(".text.svg") - assert.NoError(t, err) - assert.Equal(t, "correct answer", string(actualContents)) -} - -func TestGenerateFlameGraph(t *testing.T) { - mockExecutor := new(mockExecutor) - visualizer := defaultVisualizer{ - executor: mockExecutor, - } - - graphInput := "N4;N5 1\nN4;N6;N5 8\n" - - mockExecutor.On("runPerlScript", graphInput).Return([]byte(""), nil).Once() - mockExecutor.On("createFile", ".text.svg", mock.AnythingOfType("[]uint8")).Return(nil).Once() - - visualizer.GenerateFlameGraph(graphInput, ".text.svg", false) - - mockExecutor.AssertExpectations(t) -} - -func TestGenerateFlameGraphPrintsToStdout(t *testing.T) { - mockExecutor := new(mockExecutor) - visualizer := defaultVisualizer{ - executor: mockExecutor, - } - graphInput := "N4;N5 1\nN4;N6;N5 8\n" - mockExecutor.On("runPerlScript", graphInput).Return([]byte(""), nil).Once() - visualizer.GenerateFlameGraph(graphInput, ".text.svg", true) - - mockExecutor.AssertNotCalled(t, "createFile") - mockExecutor.AssertExpectations(t) -} - -// Underlying errors can occur in runPerlScript(). This test ensures that errors -// like a missing flamegraph.pl script or malformed input are propagated. -func TestGenerateFlameGraphExecError(t *testing.T) { - mockExecutor := new(mockExecutor) - visualizer := defaultVisualizer{ - executor: mockExecutor, - } - mockExecutor.On("runPerlScript", "").Return(nil, errors.New("bad input")).Once() - - err := visualizer.GenerateFlameGraph("", ".text.svg", false) - assert.Error(t, err) - mockExecutor.AssertNotCalled(t, "createFile") - mockExecutor.AssertExpectations(t) -} - -func TestRunPerlScriptDoesExist(t *testing.T) { - mockOSWrapper := new(mockOSWrapper) - executor := defaultExecutor{ - osWrapper: mockOSWrapper, - } - cwd, err := os.Getwd() - if err != nil { - t.Fatal(err.Error()) - } - mockOSWrapper.On("execLookPath", "flamegraph.pl").Return("", errors.New("DNE")).Once() - mockOSWrapper.On("execLookPath", cwd+"/flamegraph.pl").Return("", errors.New("DNE")).Once() - mockOSWrapper.On("execLookPath", "flame-graph-gen").Return("/somepath/flame-graph-gen", nil).Once() - - mockOSWrapper.On("cmdOutput", mock.AnythingOfType("*exec.Cmd")).Return([]byte("output"), nil).Once() - - out, err := executor.runPerlScript("some graph input") - - assert.Equal(t, []byte("output"), out) - assert.NoError(t, err) - mockOSWrapper.AssertExpectations(t) -} - -func TestRunPerlScriptDoesNotExist(t *testing.T) { - mockOSWrapper := new(mockOSWrapper) - executor := defaultExecutor{ - osWrapper: mockOSWrapper, - } - cwd, err := os.Getwd() - if err != nil { - t.Fatal(err.Error()) - } - mockOSWrapper.On("execLookPath", "flamegraph.pl").Return("", errors.New("DNE")).Once() - mockOSWrapper.On("execLookPath", cwd+"/flamegraph.pl").Return("", errors.New("DNE")).Once() - mockOSWrapper.On("execLookPath", "flame-graph-gen").Return("", errors.New("DNE")).Once() - - out, err := executor.runPerlScript("some graph input") - - assert.Equal(t, 0, len(out)) - assert.Error(t, err) - mockOSWrapper.AssertExpectations(t) -} - -// Smoke test the NewVisualizer method -func TestNewVisualizer(t *testing.T) { - assert.NotNil(t, NewVisualizer()) -} From 0e810532c67a5cdb34a96aeaba1fe4aee907d0dd Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 19:25:08 -0700 Subject: [PATCH 10/19] Rerun godeps --- Godeps/Godeps.json | 31 ++++--------------------------- 1 file changed, 4 insertions(+), 27 deletions(-) diff --git a/Godeps/Godeps.json b/Godeps/Godeps.json index a1cb4fe..b7562f9 100644 --- a/Godeps/Godeps.json +++ b/Godeps/Godeps.json @@ -1,37 +1,14 @@ { "ImportPath": "github.com/uber/go-torch", - "GoVersion": "go1.4.1", + "GoVersion": "go1.5", "Packages": [ "./..." ], "Deps": [ { - "ImportPath": "github.com/Sirupsen/logrus", - "Comment": "v0.8.2-2-g6ba91e2", - "Rev": "6ba91e24c498b49d0363c723e9e2ab2b5b8fd012" - }, - { - "ImportPath": "github.com/awalterschulze/gographviz", - "Rev": "7c3cf72121515513fad9d6c9a090db2aa4f47143" - }, - { - "ImportPath": "github.com/codegangsta/cli", - "Comment": "1.2.0-87-g8ce64f1", - "Rev": "8ce64f19ff08029a69d11b7615c9b591245450ad" - }, - { - "ImportPath": "github.com/stretchr/objx", - "Rev": "cbeaeb16a013161a98496fad62933b1d21786672" - }, - { - "ImportPath": "github.com/stretchr/testify/assert", - "Comment": "v1.0-17-g089c718", - "Rev": "089c7181b8c728499929ff09b62d3fdd8df8adff" - }, - { - "ImportPath": "github.com/stretchr/testify/mock", - "Comment": "v1.0-17-g089c718", - "Rev": "089c7181b8c728499929ff09b62d3fdd8df8adff" + "ImportPath": "github.com/jessevdk/go-flags", + "Comment": "v1-297-g1b89bf7", + "Rev": "1b89bf73cd2c3a911d7b2a279ab085c4a18cf539" } ] } From b0330d776a1b0f768224c196a3544c44f87830c0 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 22:37:43 -0700 Subject: [PATCH 11/19] Rename readMode to readState It represents a state in the parsing state machine. --- pprof/raw.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pprof/raw.go b/pprof/raw.go index c7ea5b2..cabfe2b 100644 --- a/pprof/raw.go +++ b/pprof/raw.go @@ -31,10 +31,10 @@ import ( "time" ) -type readMode int +type readState int const ( - ignore readMode = iota + ignore readState = iota samplesHeader samples locations @@ -48,7 +48,7 @@ type rawParser struct { // err is the first error encountered by the parser. err error - mode readMode + state readState funcName map[funcID]string records []*stackRecord } @@ -95,23 +95,23 @@ func (p *rawParser) parse(input []byte) error { } func (p *rawParser) processLine(line string) { - switch p.mode { + switch p.state { case ignore: if strings.HasPrefix(line, "Samples") { - p.mode = samplesHeader + p.state = samplesHeader return } case samplesHeader: - p.mode = samples + p.state = samples case samples: if strings.HasPrefix(line, "Locations") { - p.mode = locations + p.state = locations return } p.addSample(line) case locations: if strings.HasPrefix(line, "Mappings") { - p.mode = mappings + p.state = mappings return } p.addLocation(line) From 2b5323dbb4cae463f9c86679fcc40480828f15e4 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 22:38:40 -0700 Subject: [PATCH 12/19] Rename raw to parser --- pprof/{raw.go => parser.go} | 0 pprof/{raw_test.go => parser_test.go} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pprof/{raw.go => parser.go} (100%) rename pprof/{raw_test.go => parser_test.go} (100%) diff --git a/pprof/raw.go b/pprof/parser.go similarity index 100% rename from pprof/raw.go rename to pprof/parser.go diff --git a/pprof/raw_test.go b/pprof/parser_test.go similarity index 100% rename from pprof/raw_test.go rename to pprof/parser_test.go From 9e1dd629fd6e108e057d78501db9640153a68fac Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 23:28:06 -0700 Subject: [PATCH 13/19] Add better error handling and tests for errors --- pprof/parser.go | 58 ++++++++++++++++++++++++++------------------ pprof/parser_test.go | 51 ++++++++++++++++++++++++++++++++------ 2 files changed, 79 insertions(+), 30 deletions(-) diff --git a/pprof/parser.go b/pprof/parser.go index cabfe2b..e96dd6d 100644 --- a/pprof/parser.go +++ b/pprof/parser.go @@ -80,20 +80,29 @@ func (p *rawParser) parse(input []byte) error { for { line, err := reader.ReadString('\n') - line = strings.TrimSpace(line) if err != nil { if err == io.EOF { + if p.state < locations { + p.setError(fmt.Errorf("parser ended before processing locations, state: %v", p.state)) + } break } return err } - p.processLine(line) + p.processLine(strings.TrimSpace(line)) } return p.err } +func (p *rawParser) setError(err error) { + if p.err != nil { + return + } + p.err = err +} + func (p *rawParser) processLine(line string) { switch p.state { case ignore: @@ -132,6 +141,16 @@ func (p *rawParser) print(w io.Writer) error { return nil } +func (p *rawParser) parseInt(s string) int { + v, err := strconv.Atoi(s) + if err != nil { + p.setError(err) + return 0 + } + + return v +} + // addSample parses a sample that looks like: // 1 10000000: 1 2 3 4 // and creates a stackRecord for it. @@ -139,21 +158,12 @@ func (p *rawParser) addSample(line string) { // Parse a sample which looks like: parts := splitBySpace(line) if len(parts) < 3 { - p.err = fmt.Errorf("malformed sample line: %v", line) + p.setError(fmt.Errorf("malformed sample line: %v", line)) return } - samples, err := strconv.Atoi(parts[0]) - if err != nil { - p.err = err - return - } - - duration, err := strconv.Atoi(strings.TrimSuffix(parts[1], ":")) - if err != nil { - p.err = err - return - } + samples := p.parseInt(parts[0]) + duration := p.parseInt(strings.TrimSuffix(parts[1], ":")) var stack []funcID for _, fIDStr := range parts[2:] { @@ -169,7 +179,7 @@ func (p *rawParser) addSample(line string) { func (p *rawParser) addLocation(line string) { parts := splitBySpace(line) if len(parts) < 3 { - p.err = fmt.Errorf("malformed location line: %v", line) + p.setError(fmt.Errorf("malformed location line: %v", line)) return } funcID := p.toFuncID(strings.TrimSuffix(parts[0], ":")) @@ -182,22 +192,24 @@ type stackRecord struct { stack []funcID } +func getFunctionName(funcNames map[funcID]string, funcID funcID) string { + if funcName, ok := funcNames[funcID]; ok { + return funcName + } + return fmt.Sprintf("missing-function-%v", funcID) +} + // Serialize serializes a call stack for a given stackRecord given the funcID mapping. -func (r *stackRecord) Serialize(funcName map[funcID]string, w io.Writer) { +func (r *stackRecord) Serialize(funcNames map[funcID]string, w io.Writer) { for _, funcID := range r.stack { - fmt.Fprintln(w, funcName[funcID]) + fmt.Fprintln(w, getFunctionName(funcNames, funcID)) } fmt.Fprintln(w, r.samples) } // toFuncID converts a string like "8" to a funcID. func (p *rawParser) toFuncID(s string) funcID { - i, err := strconv.Atoi(s) - if err != nil { - p.err = fmt.Errorf("failed to parse funcID: %v", err) - return 0 - } - return funcID(i) + return funcID(p.parseInt(s)) } var spaceSplitter = regexp.MustCompile(`\s+`) diff --git a/pprof/parser_test.go b/pprof/parser_test.go index 4aa4d86..e8d033b 100644 --- a/pprof/parser_test.go +++ b/pprof/parser_test.go @@ -113,10 +113,32 @@ runtime.morestack } } -func testParseRawBad(t *testing.T, errorReason string, contents string) { +func TestParseMissingLocation(t *testing.T) { + contents := `Samples: + samples/count cpu/nanoseconds + 2 10000000: 1 2 + Locations: + 1: 0xaaaaa funcName :0 s=0 +` + out, err := ParseRaw([]byte(contents)) + if err != nil { + t.Fatalf("Missing location should not cause an error, got %v", err) + } + + if !bytes.Contains(out, []byte("missing-function-2")) { + t.Errorf("Missing function call stack should show missing-function-2, got: %s", out) + } +} + +func testParseRawBad(t *testing.T, errorReason, errorSubstr, contents string) { _, err := ParseRaw([]byte(contents)) if err == nil { t.Errorf("Bad %v should cause error while parsing:%s", errorReason, contents) + return + } + + if !strings.Contains(err.Error(), errorSubstr) { + t.Errorf("Bad %v error should contain %q, got %v", errorReason, errorSubstr, err) } } @@ -138,27 +160,33 @@ Locations: func TestParseRawBadFuncID(t *testing.T) { { contents := strings.Replace(simpleTemplate, funcIDSample, "?sample?", -1) - testParseRawBad(t, "funcID in sample", contents) + testParseRawBad(t, "funcID in sample", "strconv.ParseInt", contents) } { contents := strings.Replace(simpleTemplate, funcIDLocation, "?location?", -1) - testParseRawBad(t, "funcID in location", contents) + testParseRawBad(t, "funcID in location", "strconv.ParseInt", contents) } } func TestParseRawBadSample(t *testing.T) { { contents := strings.Replace(simpleTemplate, sampleCount, "??", -1) - testParseRawBad(t, "sample count", contents) + testParseRawBad(t, "sample count", "strconv.ParseInt", contents) } { contents := strings.Replace(simpleTemplate, sampleTime, "??", -1) - testParseRawBad(t, "sample duration", contents) + testParseRawBad(t, "sample duration", "strconv.ParseInt", contents) } } +func TestParseRawBadMultipleErrors(t *testing.T) { + contents := strings.Replace(simpleTemplate, sampleCount, "?s?", -1) + contents = strings.Replace(contents, sampleTime, "?t?", -1) + testParseRawBad(t, "sample duration", `strconv.ParseInt: parsing "?s?"`, contents) +} + func TestParseRawBadMalformedSample(t *testing.T) { contents := ` Samples: @@ -167,7 +195,7 @@ samples/count cpu/nanoseconds Locations: 3: 0xaaaaa funcName :0 s=0 ` - testParseRawBad(t, "malformed sample line", contents) + testParseRawBad(t, "malformed sample line", "malformed sample", contents) } func TestParseRawBadMalformedLocation(t *testing.T) { @@ -178,7 +206,16 @@ samples/count cpu/nanoseconds Locations: 3 ` - testParseRawBad(t, "malformed location line", contents) + testParseRawBad(t, "malformed location line", "malformed location", contents) +} + +func TestParseRawBadNoLocations(t *testing.T) { + contents := ` +Samples: +samples/count cpu/nanoseconds + 1 10000: 2 +` + testParseRawBad(t, "no locations", "parser ended before processing locations", contents) } func TestSplitBySpace(t *testing.T) { From 52d253922eb39129c4d0989c192488788485cd74 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 23:37:35 -0700 Subject: [PATCH 14/19] Unexport stackRecord.Serialize --- pprof/parser.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pprof/parser.go b/pprof/parser.go index e96dd6d..599cfc2 100644 --- a/pprof/parser.go +++ b/pprof/parser.go @@ -132,7 +132,7 @@ func (p *rawParser) processLine(line string) { // print prints out the stack traces collected from the raw pprof output. func (p *rawParser) print(w io.Writer) error { for _, r := range p.records { - r.Serialize(p.funcName, w) + r.serialize(p.funcName, w) fmt.Fprintln(w) } if wc, ok := w.(io.WriteCloser); ok { @@ -199,8 +199,8 @@ func getFunctionName(funcNames map[funcID]string, funcID funcID) string { return fmt.Sprintf("missing-function-%v", funcID) } -// Serialize serializes a call stack for a given stackRecord given the funcID mapping. -func (r *stackRecord) Serialize(funcNames map[funcID]string, w io.Writer) { +// serialize serializes a call stack for a given stackRecord given the funcID mapping. +func (r *stackRecord) serialize(funcNames map[funcID]string, w io.Writer) { for _, funcID := range r.stack { fmt.Fprintln(w, getFunctionName(funcNames, funcID)) } From 23c66d2b74ad23209dbeeb0026ff9f821bf45cdd Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Thu, 10 Sep 2015 23:42:06 -0700 Subject: [PATCH 15/19] Move around code in parser (no logic changes) --- pprof/parser.go | 64 ++++++++++++++++++++++++------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/pprof/parser.go b/pprof/parser.go index 599cfc2..9a50c2c 100644 --- a/pprof/parser.go +++ b/pprof/parser.go @@ -141,38 +141,6 @@ func (p *rawParser) print(w io.Writer) error { return nil } -func (p *rawParser) parseInt(s string) int { - v, err := strconv.Atoi(s) - if err != nil { - p.setError(err) - return 0 - } - - return v -} - -// addSample parses a sample that looks like: -// 1 10000000: 1 2 3 4 -// and creates a stackRecord for it. -func (p *rawParser) addSample(line string) { - // Parse a sample which looks like: - parts := splitBySpace(line) - if len(parts) < 3 { - p.setError(fmt.Errorf("malformed sample line: %v", line)) - return - } - - samples := p.parseInt(parts[0]) - duration := p.parseInt(strings.TrimSuffix(parts[1], ":")) - - var stack []funcID - for _, fIDStr := range parts[2:] { - stack = append(stack, p.toFuncID(fIDStr)) - } - - p.records = append(p.records, &stackRecord{samples, time.Duration(duration), stack}) -} - // addLocation parses a location that looks like: // 292: 0x49dee1 github.com/uber/tchannel/golang.(*Frame).ReadIn :0 s=0 // and creates a mapping from funcID to function name. @@ -192,6 +160,27 @@ type stackRecord struct { stack []funcID } +// addSample parses a sample that looks like: +// 1 10000000: 1 2 3 4 +// and creates a stackRecord for it. +func (p *rawParser) addSample(line string) { + // Parse a sample which looks like: + parts := splitBySpace(line) + if len(parts) < 3 { + p.setError(fmt.Errorf("malformed sample line: %v", line)) + return + } + + record := &stackRecord{ + samples: p.parseInt(parts[0]), + duration: time.Duration(p.parseInt(strings.TrimSuffix(parts[1], ":"))), + } + for _, fIDStr := range parts[2:] { + record.stack = append(record.stack, p.toFuncID(fIDStr)) + } + + p.records = append(p.records, record) +} func getFunctionName(funcNames map[funcID]string, funcID funcID) string { if funcName, ok := funcNames[funcID]; ok { return funcName @@ -207,6 +196,17 @@ func (r *stackRecord) serialize(funcNames map[funcID]string, w io.Writer) { fmt.Fprintln(w, r.samples) } +// parseInt converts a string to an int. It stores any errors using setError. +func (p *rawParser) parseInt(s string) int { + v, err := strconv.Atoi(s) + if err != nil { + p.setError(err) + return 0 + } + + return v +} + // toFuncID converts a string like "8" to a funcID. func (p *rawParser) toFuncID(s string) funcID { return funcID(p.parseInt(s)) From 9e24c9dce42116c69ab90ef19fcb6f90c35c54f3 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Tue, 15 Sep 2015 18:34:21 -0700 Subject: [PATCH 16/19] Fix handling of --help Help message should not be printed twice with an error --- main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/main.go b/main.go index 42fe8e5..4549ee9 100644 --- a/main.go +++ b/main.go @@ -53,6 +53,9 @@ func main() { func runWithArgs(args ...string) error { opts := &options{} if _, err := gflags.ParseArgs(opts, args); err != nil { + if flagErr, ok := err.(*gflags.Error); ok && flagErr.Type == gflags.ErrHelp { + os.Exit(0) + } return fmt.Errorf("could not parse options: %v", err) } if err := validateOptions(opts); err != nil { From a9d2e1021024b6e6620a950e1064e0e0be3cf205 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Tue, 15 Sep 2015 20:11:23 -0700 Subject: [PATCH 17/19] Return []*stack.Sample instead of []byte --- pprof/parser.go | 63 ++++++++++++++++++++++++++------------------ pprof/parser_test.go | 41 +++++++++------------------- stack/sample.go | 28 ++++++++++++++++++++ 3 files changed, 79 insertions(+), 53 deletions(-) create mode 100644 stack/sample.go diff --git a/pprof/parser.go b/pprof/parser.go index 9a50c2c..c0f222b 100644 --- a/pprof/parser.go +++ b/pprof/parser.go @@ -29,6 +29,8 @@ import ( "strconv" "strings" "time" + + "github.com/uber/go-torch/stack" ) type readState int @@ -48,30 +50,24 @@ type rawParser struct { // err is the first error encountered by the parser. err error - state readState - funcName map[funcID]string - records []*stackRecord + state readState + funcNames map[funcID]string + records []*stackRecord } // ParseRaw parses the raw pprof output and returns call stacks. -func ParseRaw(input []byte) ([]byte, error) { +func ParseRaw(input []byte) ([]*stack.Sample, error) { parser := newRawParser() if err := parser.parse(input); err != nil { return nil, err } - // TODO(prashantv): Refactor interfaces so we use streams. - buf := &bytes.Buffer{} - if err := parser.print(buf); err != nil { - return nil, err - } - - return buf.Bytes(), nil + return parser.toSamples(), nil } func newRawParser() *rawParser { return &rawParser{ - funcName: make(map[funcID]string), + funcNames: make(map[funcID]string), } } @@ -129,16 +125,30 @@ func (p *rawParser) processLine(line string) { } } -// print prints out the stack traces collected from the raw pprof output. -func (p *rawParser) print(w io.Writer) error { +// toSamples aggregates stack sample counts and returns a list of unique stack samples. +func (p *rawParser) toSamples() []*stack.Sample { + samples := make(map[string]*stack.Sample) for _, r := range p.records { - r.serialize(p.funcName, w) - fmt.Fprintln(w) + funcNames := r.funcNames(p.funcNames) + funcKey := strings.Join(funcNames, ";") + + if sample, ok := samples[funcKey]; ok { + sample.Count += r.samples + continue + } + + samples[funcKey] = &stack.Sample{ + Funcs: funcNames, + Count: r.samples, + } } - if wc, ok := w.(io.WriteCloser); ok { - return wc.Close() + + samplesList := make([]*stack.Sample, 0, len(samples)) + for _, s := range samples { + samplesList = append(samplesList, s) } - return nil + + return samplesList } // addLocation parses a location that looks like: @@ -151,7 +161,7 @@ func (p *rawParser) addLocation(line string) { return } funcID := p.toFuncID(strings.TrimSuffix(parts[0], ":")) - p.funcName[funcID] = parts[2] + p.funcNames[funcID] = parts[2] } type stackRecord struct { @@ -188,12 +198,15 @@ func getFunctionName(funcNames map[funcID]string, funcID funcID) string { return fmt.Sprintf("missing-function-%v", funcID) } -// serialize serializes a call stack for a given stackRecord given the funcID mapping. -func (r *stackRecord) serialize(funcNames map[funcID]string, w io.Writer) { - for _, funcID := range r.stack { - fmt.Fprintln(w, getFunctionName(funcNames, funcID)) +// funcNames returns the function names for this stack sample. +// It returns in parent first order. +func (r *stackRecord) funcNames(funcNames map[funcID]string) []string { + var names []string + for i := len(r.stack) - 1; i >= 0; i-- { + funcID := r.stack[i] + names = append(names, getFunctionName(funcNames, funcID)) } - fmt.Fprintln(w, r.samples) + return names } // parseInt converts a string to an int. It stores any errors using setError. diff --git a/pprof/parser_test.go b/pprof/parser_test.go index e8d033b..5c7f30b 100644 --- a/pprof/parser_test.go +++ b/pprof/parser_test.go @@ -21,12 +21,13 @@ package pprof import ( - "bytes" "io/ioutil" "reflect" "strings" "testing" "time" + + "github.com/uber/go-torch/stack" ) func parseTestRawData(t *testing.T) ([]byte, *rawParser) { @@ -65,9 +66,9 @@ func TestParse(t *testing.T) { // line 250 - 290 are locations (or funcID mappings) const expectedFuncIDs = 41 - if len(parser.funcName) != expectedFuncIDs { + if len(parser.funcNames) != expectedFuncIDs { t.Errorf("Failed to parse func ID mappings, got %v records, expected %v", - len(parser.funcName), expectedFuncIDs) + len(parser.funcNames), expectedFuncIDs) } knownMappings := map[funcID]string{ 1: "main.fib", @@ -75,7 +76,7 @@ func TestParse(t *testing.T) { 34: "runtime.morestack", } for funcID, expected := range knownMappings { - if got := parser.funcName[funcID]; got != expected { + if got := parser.funcNames[funcID]; got != expected { t.Errorf("Unexpected mapping for %v: got %v, want %v", funcID, got, expected) } } @@ -88,28 +89,8 @@ func TestParseRawValid(t *testing.T) { t.Fatalf("ParseRaw failed: %v", err) } - expected1 := `main.fib -main.fib -main.fib -main.fib -main.main -runtime.main -runtime.goexit -1 -` - if !bytes.Contains(got, []byte(expected1)) { - t.Errorf("missing expected stack: %s", expected1) - } - - expected2 := `runtime.schedule -runtime.goschedImpl -runtime.gopreempt_m -runtime.newstack -runtime.morestack -12 -` - if !bytes.Contains(got, []byte(expected2)) { - t.Errorf("missing expected stack: %s", expected2) + if expected := 18; len(got) != expected { + t.Errorf("Expected %v unique stack samples, got %v", expected, got) } } @@ -125,8 +106,12 @@ func TestParseMissingLocation(t *testing.T) { t.Fatalf("Missing location should not cause an error, got %v", err) } - if !bytes.Contains(out, []byte("missing-function-2")) { - t.Errorf("Missing function call stack should show missing-function-2, got: %s", out) + expected := []*stack.Sample{{ + Funcs: []string{"missing-function-2", "funcName"}, + Count: 2, + }} + if !reflect.DeepEqual(out, expected) { + t.Errorf("Missing function call stack should contain missing-function-2\n got %+v\n want %+v", expected, out) } } diff --git a/stack/sample.go b/stack/sample.go new file mode 100644 index 0000000..048f63c --- /dev/null +++ b/stack/sample.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package stack + +// Sample represents the sample count for a specific call stack. +type Sample struct { + // Funcs is parent first. + Funcs []string + Count int +} From 3d8c44b71f060c0d15a38ec1020879618e62f674 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Tue, 15 Sep 2015 20:21:23 -0700 Subject: [PATCH 18/19] Render flame graph input from stack samples --- main.go | 16 ++++++------- main_test.go | 18 ++++----------- renderer/renderer.go | 47 +++++++++++++++++++++++++++++++++++++++ renderer/renderer_test.go | 46 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 22 deletions(-) create mode 100644 renderer/renderer.go create mode 100644 renderer/renderer_test.go diff --git a/main.go b/main.go index 4549ee9..1b5bee7 100644 --- a/main.go +++ b/main.go @@ -76,18 +76,18 @@ func runWithOptions(opts *options) error { return fmt.Errorf("could not parse raw pprof output: %v", err) } - if opts.Raw { - log.Print("Printing raw call graph to stdout") - fmt.Printf("%s", callStacks) - return nil + flameInput, err := renderer.ToFlameInput(callStacks) + if err != nil { + return fmt.Errorf("could not convert stacks to flamegraph input: %v", err) } - collapsedStacks, err := renderer.CollapseStacks(callStacks) - if err != nil { - return fmt.Errorf("could not collapse stacks: %v", err) + if opts.Raw { + log.Print("Printing raw flamegraph input to stdout") + fmt.Printf("%s", flameInput) + return nil } - flameGraph, err := renderer.GenerateFlameGraph(collapsedStacks) + flameGraph, err := renderer.GenerateFlameGraph(flameInput) if err != nil { return fmt.Errorf("could not generate flame graph: %v", err) } diff --git a/main_test.go b/main_test.go index 85758bd..32977de 100644 --- a/main_test.go +++ b/main_test.go @@ -133,13 +133,7 @@ func TestRunFile(t *testing.T) { if err != nil { t.Errorf("Failed to read line 1 in output file: %v", err) } - line2, err := reader.ReadString('\n') - if err != nil { - t.Errorf("Failed to read line 2 in output file: %v", err) - } - - if !strings.Contains(line1, "flamegraph.pl") || - !strings.Contains(line2, "stackcollapse.pl") { + if !strings.Contains(line1, "flamegraph.pl") { t.Errorf("Output file has not been processed by flame graph scripts") } }) @@ -188,13 +182,9 @@ func withScriptsInPath(t *testing.T, f func()) { echo $0 cat ` - scripts := []string{"stackcollapse.pl", "flamegraph.pl"} - scriptContentsBytes := []byte(scriptContents) - for _, s := range scripts { - scriptFile := filepath.Join(scriptsPath, s) - if err := ioutil.WriteFile(scriptFile, scriptContentsBytes, 0777); err != nil { - t.Errorf("Failed to create script %v: %v", scriptFile, err) - } + scriptFile := filepath.Join(scriptsPath, "flamegraph.pl") + if err := ioutil.WriteFile(scriptFile, []byte(scriptContents), 0777); err != nil { + t.Errorf("Failed to create script %v: %v", scriptFile, err) } } diff --git a/renderer/renderer.go b/renderer/renderer.go new file mode 100644 index 0000000..3a280d1 --- /dev/null +++ b/renderer/renderer.go @@ -0,0 +1,47 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package renderer + +import ( + "bytes" + "fmt" + "io" + "strings" + + "github.com/uber/go-torch/stack" +) + +// ToFlameInput convers the given stack samples to flame graph input. +func ToFlameInput(samples []*stack.Sample) ([]byte, error) { + buf := &bytes.Buffer{} + for _, s := range samples { + if err := renderSample(buf, s); err != nil { + return nil, err + } + } + return buf.Bytes(), nil +} + +// renderSample renders a single stack sample as flame graph input. +func renderSample(w io.Writer, s *stack.Sample) error { + _, err := fmt.Fprintf(w, "%s %v\n", strings.Join(s.Funcs, ";"), s.Count) + return err +} diff --git a/renderer/renderer_test.go b/renderer/renderer_test.go new file mode 100644 index 0000000..34b4db3 --- /dev/null +++ b/renderer/renderer_test.go @@ -0,0 +1,46 @@ +// Copyright (c) 2015 Uber Technologies, Inc. +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in +// all copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +// THE SOFTWARE. + +package renderer + +import ( + "reflect" + "testing" + + "github.com/uber/go-torch/stack" +) + +func TestToFlameInput(t *testing.T) { + samples := []*stack.Sample{ + {Funcs: []string{"func1", "func2"}, Count: 10}, + {Funcs: []string{"func3"}, Count: 8}, + {Funcs: []string{"func4", "func5", "func6"}, Count: 3}, + } + expected := "func1;func2 10\nfunc3 8\nfunc4;func5;func6 3\n" + + out, err := ToFlameInput(samples) + if err != nil { + t.Fatalf("ToFlameInput failed: %v", err) + } + + if !reflect.DeepEqual(expected, string(out)) { + t.Errorf("ToFlameInput failed:\n got %s\n want %s", out, expected) + } +} From 0ef4ab4aa948cc36343ba650b705697cc7268b27 Mon Sep 17 00:00:00 2001 From: Prashant Varanasi Date: Tue, 15 Sep 2015 20:58:54 -0700 Subject: [PATCH 19/19] Change logging flags --- main.go | 1 + 1 file changed, 1 insertion(+) diff --git a/main.go b/main.go index 1b5bee7..d1e02cf 100644 --- a/main.go +++ b/main.go @@ -45,6 +45,7 @@ type options struct { // main is the entry point of the application func main() { + log.SetFlags(log.Ltime) if err := runWithArgs(os.Args...); err != nil { log.Fatalf("Failed: %v", err) }