From fb4f7fdb26da9ed0fee6beab280c84b399edaa42 Mon Sep 17 00:00:00 2001 From: Cherry Mui Date: Tue, 1 Nov 2022 11:40:29 -0400 Subject: [PATCH] cmd/compile: use edge weights to decide inlineability in PGO Currently, with PGO, the inliner uses node weights to decide if a function is inlineable (with a larger budget). But the actual inlining is determined by the weight of the call edge. There is a discrepancy that, if a callee node is hot but the call edge is not, it would not inlined, and marking the callee inlineable would of no use. Instead of using two kinds of weights, we just use the edge weights to decide inlineability. If a function is the callee of a hot call edge, its inlineability is determined with a larger threshold. For a function that exceeds the regular inlining budget, it is still inlined only when the call edge is hot, as it would exceed the regular inlining cost for non-hot call sites, even if it is marked inlineable. For #55022. Change-Id: I93fa9919fc6bcbb394e6cfe54ec96a96eede08f7 Reviewed-on: https://go-review.googlesource.com/c/go/+/447015 Run-TryBot: Cherry Mui Reviewed-by: Michael Pratt TryBot-Result: Gopher Robot --- src/cmd/compile/internal/base/debug.go | 1 - src/cmd/compile/internal/inline/inl.go | 25 ++++++------------------- src/cmd/compile/internal/pgo/irgraph.go | 7 ++----- 3 files changed, 8 insertions(+), 25 deletions(-) diff --git a/src/cmd/compile/internal/base/debug.go b/src/cmd/compile/internal/base/debug.go index 682c0dd518118..18422256ab61c 100644 --- a/src/cmd/compile/internal/base/debug.go +++ b/src/cmd/compile/internal/base/debug.go @@ -46,7 +46,6 @@ type DebugFlags struct { WB int `help:"print information about write barriers"` ABIWrap int `help:"print information about ABI wrapper generation"` MayMoreStack string `help:"call named function before all stack growth checks"` - InlineHotFuncThreshold string `help:"threshold percentage for determining functions as hot candidates for inlining"` InlineHotCallSiteThreshold string `help:"threshold percentage for determining call sites as hot candidates for inlining"` InlineHotBudget int `help:"inline budget for hot functions"` PGOInline int `help:"debug profile-guided inlining"` diff --git a/src/cmd/compile/internal/inline/inl.go b/src/cmd/compile/internal/inline/inl.go index 335bb23ecb91f..98bfb4e382ca1 100644 --- a/src/cmd/compile/internal/inline/inl.go +++ b/src/cmd/compile/internal/inline/inl.go @@ -56,9 +56,9 @@ const ( ) var ( - // List of all hot nodes. + // List of all hot callee nodes. // TODO(prattmic): Make this non-global. - candHotNodeMap = make(map[*pgo.IRNode]struct{}) + candHotCalleeMap = make(map[*pgo.IRNode]struct{}) // List of all hot call sites. CallSiteInfo.Callee is always nil. // TODO(prattmic): Make this non-global. @@ -68,9 +68,6 @@ var ( // TODO(prattmic): Make this non-global. inlinedCallSites = make(map[pgo.CallSiteInfo]struct{}) - // Threshold in percentage for hot function inlining. - inlineHotFuncThresholdPercent = float64(2) - // Threshold in percentage for hot callsite inlining. inlineHotCallSiteThresholdPercent = float64(0.1) @@ -80,13 +77,6 @@ var ( // pgoInlinePrologue records the hot callsites from ir-graph. func pgoInlinePrologue(p *pgo.Profile) { - if s, err := strconv.ParseFloat(base.Debug.InlineHotFuncThreshold, 64); err == nil { - inlineHotFuncThresholdPercent = s - if base.Debug.PGOInline > 0 { - fmt.Printf("hot-node-thres=%v\n", inlineHotFuncThresholdPercent) - } - } - if s, err := strconv.ParseFloat(base.Debug.InlineHotCallSiteThreshold, 64); err == nil { inlineHotCallSiteThresholdPercent = s if base.Debug.PGOInline > 0 { @@ -102,10 +92,6 @@ func pgoInlinePrologue(p *pgo.Profile) { for _, f := range list { name := ir.PkgFuncName(f) if n, ok := p.WeightedCG.IRNodes[name]; ok { - nodeweight := pgo.WeightInPercentage(n.Flat, p.TotalNodeWeight) - if nodeweight > inlineHotFuncThresholdPercent { - candHotNodeMap[n] = struct{}{} - } for _, e := range p.WeightedCG.OutEdges[n] { if e.Weight != 0 { edgeweightpercent := pgo.WeightInPercentage(e.Weight, p.TotalEdgeWeight) @@ -113,6 +99,7 @@ func pgoInlinePrologue(p *pgo.Profile) { csi := pgo.CallSiteInfo{Line: e.CallSite, Caller: n.AST} if _, ok := candHotEdgeMap[csi]; !ok { candHotEdgeMap[csi] = struct{}{} + candHotCalleeMap[e.Dst] = struct{}{} } } } @@ -122,7 +109,7 @@ func pgoInlinePrologue(p *pgo.Profile) { }) if base.Debug.PGOInline > 0 { fmt.Printf("hot-cg before inline in dot format:") - p.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent) + p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent) } } @@ -139,7 +126,7 @@ func pgoInlineEpilogue(p *pgo.Profile) { }) // Print the call-graph after inlining. This is a debugging feature. fmt.Printf("hot-cg after inline in dot:") - p.PrintWeightedCallGraphDOT(inlineHotFuncThresholdPercent, inlineHotCallSiteThresholdPercent) + p.PrintWeightedCallGraphDOT(inlineHotCallSiteThresholdPercent) } } @@ -270,7 +257,7 @@ func CanInline(fn *ir.Func, profile *pgo.Profile) { budget := int32(inlineMaxBudget) if profile != nil { if n, ok := profile.WeightedCG.IRNodes[ir.PkgFuncName(fn)]; ok { - if _, ok := candHotNodeMap[n]; ok { + if _, ok := candHotCalleeMap[n]; ok { budget = int32(inlineHotMaxBudget) if base.Debug.PGOInline > 0 { fmt.Printf("hot-node enabled increased budget=%v for func=%v\n", budget, ir.PkgFuncName(fn)) diff --git a/src/cmd/compile/internal/pgo/irgraph.go b/src/cmd/compile/internal/pgo/irgraph.go index 56cfebf85e3ff..f0932b51ea275 100644 --- a/src/cmd/compile/internal/pgo/irgraph.go +++ b/src/cmd/compile/internal/pgo/irgraph.go @@ -150,7 +150,7 @@ func New(profileFile string) *Profile { p := &Profile{ NodeMap: make(map[NodeMapKey]*Weights), ProfileGraph: g, - WeightedCG: &IRGraph{ + WeightedCG: &IRGraph{ IRNodes: make(map[string]*IRNode), }, } @@ -347,7 +347,7 @@ func WeightInPercentage(value int64, total int64) float64 { } // PrintWeightedCallGraphDOT prints IRGraph in DOT format. -func (p *Profile) PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold float64) { +func (p *Profile) PrintWeightedCallGraphDOT(edgeThreshold float64) { fmt.Printf("\ndigraph G {\n") fmt.Printf("forcelabels=true;\n") @@ -383,9 +383,6 @@ func (p *Profile) PrintWeightedCallGraphDOT(nodeThreshold float64, edgeThreshold if n, ok := p.WeightedCG.IRNodes[name]; ok { nodeweight := WeightInPercentage(n.Flat, p.TotalNodeWeight) color := "black" - if nodeweight > nodeThreshold { - color = "red" - } if ast.Inl != nil { fmt.Printf("\"%v\" [color=%v,label=\"%v,freq=%.2f,inl_cost=%d\"];\n", ir.PkgFuncName(ast), color, ir.PkgFuncName(ast), nodeweight, ast.Inl.Cost) } else {