perfschema: support query cpu/memory/mutex/block/allocs/goroutines flamegraph by SQL (#12986)

go.mod

@@ -20,6 +20,7 @@ require (
 	github.com/golang/protobuf v1.2.0
 	github.com/golang/snappy v0.0.1
 	github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c
+	github.com/google/pprof v0.0.0-20190930153522-6ce02741cba3
 	github.com/google/uuid v1.1.1
 	github.com/gorilla/context v1.1.1 // indirect
 	github.com/gorilla/mux v1.6.2
@@ -68,7 +69,6 @@ require (
 	golang.org/x/crypto v0.0.0-20190909091759-094676da4a83 // indirect
 	golang.org/x/net v0.0.0-20190909003024-a7b16738d86b
 	golang.org/x/sync v0.0.0-20190423024810-112230192c58 // indirect
-	golang.org/x/sys v0.0.0-20190909082730-f460065e899a // indirect
 	golang.org/x/text v0.3.0
 	golang.org/x/tools v0.0.0-20190130214255-bb1329dc71a0
 	google.golang.org/genproto v0.0.0-20190108161440-ae2f86662275 // indirect
@@ -78,4 +78,6 @@ require (
 	sourcegraph.com/sourcegraph/appdash-data v0.0.0-20151005221446-73f23eafcf67
 )
 
+replace github.com/google/pprof => github.com/lonng/pprof v0.0.0-20191012154247-04dfd648ce8d
+
 go 1.13

go.sum

@@ -12,7 +12,9 @@ github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d h1:rQ
 github.com/blacktear23/go-proxyprotocol v0.0.0-20180807104634-af7a81e8dd0d/go.mod h1:VKt7CNAQxpFpSDz3sXyj9hY/GbVsQCr0sB3w59nE7lU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20171208011716-f6d7a1f6fbf3/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJI=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w=
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
@@ -98,6 +100,8 @@ github.com/grpc-ecosystem/grpc-gateway v1.5.1 h1:3scN4iuXkNOyP98jF55Lv8a9j1o/Iwv
 github.com/grpc-ecosystem/grpc-gateway v1.5.1/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6 h1:UDMh68UUwekSh5iP2OMhRRZJiiBccgV7axzUG8vi56c=
+github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
@@ -116,6 +120,8 @@ github.com/kr/pty v1.0.0/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/lonng/pprof v0.0.0-20191012154247-04dfd648ce8d h1:6Ike9EBxOFsCMMih14rQJmb7WPWdgRu4C0OLl6oRHwE=
+github.com/lonng/pprof v0.0.0-20191012154247-04dfd648ce8d/go.mod h1:0vjxLpmyJvBwQbIQuxhHxmogQFpJvB9doVHvxFFfyoY=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
@@ -275,8 +281,8 @@ golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5h
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20190909082730-f460065e899a h1:mIzbOulag9/gXacgxKlFVwpCOWSfBT3/pDyyCwGA9as=
-golang.org/x/sys v0.0.0-20190909082730-f460065e899a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47 h1:/XfQ9z7ib8eEJX2hdgFTZJ/ntt0swNk5oYBziWeTCvY=
+golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM=

infoschema/perfschema/const.go

@@ -37,6 +37,12 @@ var perfSchemaTables = []string{
 	tableStagesHistory,
 	tableStagesHistoryLong,
 	tableEventsStatementsSummaryByDigest,
+	tableTiDBProfileCPU,
+	tableTiDBProfileMemory,
+	tableTiDBProfileMutex,
+	tableTiDBAllocsProfile,
+	tableTiDBProfileBlock,
+	tableTiDBGoroutine,
 }
 
 // tableGlobalStatus contains the column name definitions for table global_status, same as MySQL.
@@ -388,3 +394,55 @@ const tableEventsStatementsSummaryByDigest = "CREATE TABLE if not exists events_
 	"FIRST_SEEN TIMESTAMP(6) NOT NULL," +
 	"LAST_SEEN TIMESTAMP(6) NOT NULL," +
 	"QUERY_SAMPLE_TEXT LONGTEXT DEFAULT NULL);"
+
+// tableTiDBProfileCPU contains the columns name definitions for table events_cpu_profile_graph
+const tableTiDBProfileCPU = "CREATE TABLE IF NOT EXISTS " + tableNameTiDBProfileCPU + " (" +
+	"FUNCTION VARCHAR(512) NOT NULL," +
+	"PERCENT_ABS VARCHAR(8) NOT NULL," +
+	"PERCENT_REL VARCHAR(8) NOT NULL," +
+	"ROOT_CHILD INT(8) NOT NULL," +
+	"DEPTH INT(8) NOT NULL," +
+	"FILE VARCHAR(512) NOT NULL);"
+
+// tableTiDBProfileMemory contains the columns name definitions for table events_memory_profile_graph
+const tableTiDBProfileMemory = "CREATE TABLE IF NOT EXISTS " + tableNameTiDBProfileMemory + " (" +
+	"FUNCTION VARCHAR(512) NOT NULL," +
+	"PERCENT_ABS VARCHAR(8) NOT NULL," +
+	"PERCENT_REL VARCHAR(8) NOT NULL," +
+	"ROOT_CHILD INT(8) NOT NULL," +
+	"DEPTH INT(8) NOT NULL," +
+	"FILE VARCHAR(512) NOT NULL);"
+
+// tableTiDBProfileMutex contains the columns name definitions for table events_mutex_profile_graph
+const tableTiDBProfileMutex = "CREATE TABLE IF NOT EXISTS " + tableNameTiDBProfileMutex + " (" +
+	"FUNCTION VARCHAR(512) NOT NULL," +
+	"PERCENT_ABS VARCHAR(8) NOT NULL," +
+	"PERCENT_REL VARCHAR(8) NOT NULL," +
+	"ROOT_CHILD INT(8) NOT NULL," +
+	"DEPTH INT(8) NOT NULL," +
+	"FILE VARCHAR(512) NOT NULL);"
+
+// tableTiDBAllocsProfile contains the columns name definitions for table events_allocs_profile_graph
+const tableTiDBAllocsProfile = "CREATE TABLE IF NOT EXISTS " + tableNameTiDBProfileAllocs + " (" +
+	"FUNCTION VARCHAR(512) NOT NULL," +
+	"PERCENT_ABS VARCHAR(8) NOT NULL," +
+	"PERCENT_REL VARCHAR(8) NOT NULL," +
+	"ROOT_CHILD INT(8) NOT NULL," +
+	"DEPTH INT(8) NOT NULL," +
+	"FILE VARCHAR(512) NOT NULL);"
+
+// tableTiDBProfileBlock contains the columns name definitions for table events_block_profile_graph
+const tableTiDBProfileBlock = "CREATE TABLE IF NOT EXISTS " + tableNameTiDBProfileBlock + " (" +
+	"FUNCTION VARCHAR(512) NOT NULL," +
+	"PERCENT_ABS VARCHAR(8) NOT NULL," +
+	"PERCENT_REL VARCHAR(8) NOT NULL," +
+	"ROOT_CHILD INT(8) NOT NULL," +
+	"DEPTH INT(8) NOT NULL," +
+	"FILE VARCHAR(512) NOT NULL);"
+
+// tableTiDBGoroutine contains the columns name definitions for table events_goroutine
+const tableTiDBGoroutine = "CREATE TABLE IF NOT EXISTS " + tableNameTiDBGoroutines + " (" +
+	"FUNCTION VARCHAR(512) NOT NULL," +
+	"ID INT(8) NOT NULL," +
+	"STATE VARCHAR(16) NOT NULL," +
+	"LOCATION VARCHAR(512));"

infoschema/perfschema/tables.go

@@ -21,11 +21,18 @@ import (
 	"github.com/pingcap/tidb/sessionctx"
 	"github.com/pingcap/tidb/table"
 	"github.com/pingcap/tidb/types"
+	"github.com/pingcap/tidb/util/profile"
 	"github.com/pingcap/tidb/util/stmtsummary"
 )
 
 const (
 	tableNameEventsStatementsSummaryByDigest = "events_statements_summary_by_digest"
+	tableNameTiDBProfileCPU                  = "tidb_profile_cpu"
+	tableNameTiDBProfileMemory               = "tidb_profile_memory"
+	tableNameTiDBProfileMutex                = "tidb_profile_mutex"
+	tableNameTiDBProfileAllocs               = "tidb_profile_allocs"
+	tableNameTiDBProfileBlock                = "tidb_profile_block"
+	tableNameTiDBGoroutines                  = "tidb_goroutines"
 )
 
 // perfSchemaTable stands for the fake table all its data is in the memory.
@@ -90,6 +97,21 @@ func (vt *perfSchemaTable) getRows(ctx sessionctx.Context, cols []*table.Column)
 	switch vt.meta.Name.O {
 	case tableNameEventsStatementsSummaryByDigest:
 		fullRows = stmtsummary.StmtSummaryByDigestMap.ToDatum()
+	case tableNameTiDBProfileCPU:
+		fullRows, err = (&profile.Collector{}).ProfileGraph("cpu")
+	case tableNameTiDBProfileMemory:
+		fullRows, err = (&profile.Collector{}).ProfileGraph("heap")
+	case tableNameTiDBProfileMutex:
+		fullRows, err = (&profile.Collector{}).ProfileGraph("mutex")
+	case tableNameTiDBProfileAllocs:
+		fullRows, err = (&profile.Collector{}).ProfileGraph("allocs")
+	case tableNameTiDBProfileBlock:
+		fullRows, err = (&profile.Collector{}).ProfileGraph("block")
+	case tableNameTiDBGoroutines:
+		fullRows, err = (&profile.Collector{}).Goroutines()
+	}
+	if err != nil {
+		return
 	}
 	if len(cols) == len(vt.cols) {
 		return

planner/core/common_plans.go

@@ -34,8 +34,8 @@ import (
 	driver "github.com/pingcap/tidb/types/parser_driver"
 	"github.com/pingcap/tidb/util/chunk"
 	"github.com/pingcap/tidb/util/kvcache"
-	"github.com/pingcap/tidb/util/plancodec"
 	"github.com/pingcap/tidb/util/ranger"
+	"github.com/pingcap/tidb/util/texttree"
 )
 
 var planCacheCounter = metrics.PlanCacheCounter.WithLabelValues("prepare")
@@ -599,7 +599,7 @@ func (e *Explain) explainPlanInRowFormat(p PhysicalPlan, taskType, indent string
 	e.explainedPlans[p.ID()] = true
 
 	// For every child we create a new sub-tree rooted by it.
-	childIndent := e.getIndent4Child(indent, isLastChild)
+	childIndent := texttree.Indent4Child(indent, isLastChild)
 	for i, child := range p.Children() {
 		if e.explainedPlans[child.ID()] {
 			continue
@@ -624,7 +624,7 @@ func (e *Explain) prepareOperatorInfo(p PhysicalPlan, taskType string, indent st
 	operatorInfo := p.ExplainInfo()
 	count := string(strconv.AppendFloat([]byte{}, p.statsInfo().RowCount, 'f', 2, 64))
 	explainID := p.ExplainID().String()
-	row := []string{e.prettyIdentifier(explainID, indent, isLastChild), count, taskType, operatorInfo}
+	row := []string{texttree.PrettyIdentifier(explainID, indent, isLastChild), count, taskType, operatorInfo}
 	if e.Analyze {
 		runtimeStatsColl := e.ctx.GetSessionVars().StmtCtx.RuntimeStatsColl
 		// There maybe some mock information for cop task to let runtimeStatsColl.Exists(p.ExplainID()) is true.
@@ -647,53 +647,6 @@ func (e *Explain) prepareOperatorInfo(p PhysicalPlan, taskType string, indent st
 	e.Rows = append(e.Rows, row)
 }
 
-func (e *Explain) prettyIdentifier(id, indent string, isLastChild bool) string {
-	if len(indent) == 0 {
-		return id
-	}
-
-	indentBytes := []rune(indent)
-	for i := len(indentBytes) - 1; i >= 0; i-- {
-		if indentBytes[i] != plancodec.TreeBody {
-			continue
-		}
-
-		// Here we attach a new node to the current sub-tree by changing
-		// the closest TreeBody to a:
-		// 1. TreeLastNode, if this operator is the last child.
-		// 2. TreeMiddleNode, if this operator is not the last child..
-		if isLastChild {
-			indentBytes[i] = plancodec.TreeLastNode
-		} else {
-			indentBytes[i] = plancodec.TreeMiddleNode
-		}
-		break
-	}
-
-	// Replace the TreeGap between the TreeBody and the node to a
-	// TreeNodeIdentifier.
-	indentBytes[len(indentBytes)-1] = plancodec.TreeNodeIdentifier
-	return string(indentBytes) + id
-}
-
-func (e *Explain) getIndent4Child(indent string, isLastChild bool) string {
-	if !isLastChild {
-		return string(append([]rune(indent), plancodec.TreeBody, plancodec.TreeGap))
-	}
-
-	// If the current node is the last node of the current operator tree, we
-	// need to end this sub-tree by changing the closest TreeBody to a TreeGap.
-	indentBytes := []rune(indent)
-	for i := len(indentBytes) - 1; i >= 0; i-- {
-		if indentBytes[i] == plancodec.TreeBody {
-			indentBytes[i] = plancodec.TreeGap
-			break
-		}
-	}
-
-	return string(append(indentBytes, plancodec.TreeBody, plancodec.TreeGap))
-}
-
 func (e *Explain) prepareDotInfo(p PhysicalPlan) {
 	buffer := bytes.NewBufferString("")
 	buffer.WriteString(fmt.Sprintf("\ndigraph %s {\n", p.ExplainID()))

util/plancodec/codec.go

@@ -22,23 +22,7 @@ import (
 
 	"github.com/golang/snappy"
 	"github.com/pingcap/errors"
-)
-
-const (
-	// TreeBody indicates the current operator sub-tree is not finished, still
-	// has child operators to be attached on.
-	TreeBody = '│'
-	// TreeMiddleNode indicates this operator is not the last child of the
-	// current sub-tree rooted by its parent.
-	TreeMiddleNode = '├'
-	// TreeLastNode indicates this operator is the last child of the current
-	// sub-tree rooted by its parent.
-	TreeLastNode = '└'
-	// TreeGap is used to represent the gap between the branches of the tree.
-	TreeGap = ' '
-	// TreeNodeIdentifier is used to replace the TreeGap once we need to attach
-	// a node to a sub-tree.
-	TreeNodeIdentifier = '─'
+	"github.com/pingcap/tidb/util/texttree"
 )
 
 const (
@@ -147,8 +131,8 @@ func (pd *planDecoder) initPlanTreeIndents() {
 		for i := 0; i < len(indent)-2; i++ {
 			indent[i] = ' '
 		}
-		indent[len(indent)-2] = TreeLastNode
-		indent[len(indent)-1] = TreeNodeIdentifier
+		indent[len(indent)-2] = texttree.TreeLastNode
+		indent[len(indent)-1] = texttree.TreeNodeIdentifier
 	}
 }
 
@@ -167,11 +151,11 @@ func (pd *planDecoder) fillIndent(parentIndex, childIndex int) {
 	}
 	idx := depth*2 - 2
 	for i := childIndex - 1; i > parentIndex; i-- {
-		if pd.indents[i][idx] == TreeLastNode {
-			pd.indents[i][idx] = TreeMiddleNode
+		if pd.indents[i][idx] == texttree.TreeLastNode {
+			pd.indents[i][idx] = texttree.TreeMiddleNode
 			break
 		}
-		pd.indents[i][idx] = TreeBody
+		pd.indents[i][idx] = texttree.TreeBody
 	}
 }