Skip to content

Commit

Permalink
[#8] impl command service, goroutine dump
Browse files Browse the repository at this point in the history
  • Loading branch information
dwkang committed Dec 1, 2021
1 parent 2ed7f84 commit a3e7ece
Show file tree
Hide file tree
Showing 6 changed files with 846 additions and 12 deletions.
31 changes: 19 additions & 12 deletions agent.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,16 +32,16 @@ func log(srcFile string) *logrus.Entry {
}

type Agent struct {
config Config
startTime int64
sequence int64
agentGrpc *agentGrpc
spanGrpc *spanGrpc
statGrpc *statGrpc
spanBuffer []*span
spanChan chan *span
wg sync.WaitGroup
sampler traceSampler
config Config
startTime int64
sequence int64
agentGrpc *agentGrpc
spanGrpc *spanGrpc
statGrpc *statGrpc
cmdGrpc *cmdGrpc
spanChan chan *span
wg sync.WaitGroup
sampler traceSampler

exceptionIdCache *lru.Cache
exceptionIdGen int32
Expand Down Expand Up @@ -87,6 +87,11 @@ func NewAgent(config *Config) (*Agent, error) {
return nil, err
}

agent.cmdGrpc, err = newCommandGrpc(&agent)
if err != nil {
return nil, err
}

agent.exceptionIdGen = 0
agent.exceptionIdCache, err = lru.New(cacheSize)
if err != nil {
Expand Down Expand Up @@ -119,7 +124,9 @@ func NewAgent(config *Config) (*Agent, error) {
go agent.sendPingWorker()
go agent.sendSpanWorker()
go collectStats(&agent)
agent.wg.Add(3)
go commandService(&agent)

agent.wg.Add(4)

return &agent, nil
}
Expand Down Expand Up @@ -222,7 +229,7 @@ func (agent *Agent) sendPingWorker() {
stream = agent.agentGrpc.newPingStreamWithRetry()
}

time.Sleep(5 * time.Second)
time.Sleep(60 * time.Second)
}

stream.close()
Expand Down
113 changes: 113 additions & 0 deletions command.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package pinpoint

import (
pb "github.com/pinpoint-apm/pinpoint-go-agent/protobuf"
"os"
"path/filepath"
"runtime/pprof"
"time"
)

var gDump *GoroutineDump

func commandService(agent *Agent) {
log("cmd").Info("command service goroutine start")
defer agent.wg.Done()

cmdStream := agent.cmdGrpc.newCommandStreamWithRetry()

for true {
if !agent.enable {
break
}

err := cmdStream.sendCommandMessage()
if err != nil {
log("cmd").Errorf("fail to sendCommandMessage(): %v", err)
cmdStream.close()
cmdStream = agent.cmdGrpc.newCommandStreamWithRetry()
continue
}

for true {
err = cmdStream.recvCommandRequest()
if err != nil {
log("cmd").Errorf("fail to recvCommandRequest(): %v", err)
break
}

cmdReq := cmdStream.cmdReq
reqId := cmdReq.GetRequestId()
log("cmd").Debugf("command service request: %v", cmdReq)

switch cmdReq.Command.(type) {
case *pb.PCmdRequest_CommandEcho:
msg := cmdReq.GetCommandEcho().GetMessage()
agent.cmdGrpc.sendEcho(reqId, msg)
break
case *pb.PCmdRequest_CommandActiveThreadCount:
atcStream := agent.cmdGrpc.newActiveThreadCountStream(reqId)
go sendActiveThreadCount(atcStream)
break
case *pb.PCmdRequest_CommandActiveThreadDump:
limit := cmdReq.GetCommandActiveThreadDump().GetLimit()
threadName := cmdReq.GetCommandActiveThreadDump().GetThreadName()
localId := cmdReq.GetCommandActiveThreadDump().GetLocalTraceId()
agent.cmdGrpc.sendActiveThreadDump(reqId, limit, threadName, localId, gDump)
break
case *pb.PCmdRequest_CommandActiveThreadLightDump:
limit := cmdReq.GetCommandActiveThreadLightDump().GetLimit()
gDump = dumpGoroutine()
agent.cmdGrpc.sendActiveThreadLightDump(reqId, limit, gDump)
break
case nil:
// The field is not set.
default:
}
}

if err != nil {
cmdStream.close()
cmdStream = agent.cmdGrpc.newCommandStreamWithRetry()
}
}

cmdStream.close()
log("cmd").Info("command service goroutine finish")
}

func sendActiveThreadCount(s *activeThreadCountStream) {
for true {
err := s.sendActiveThreadCount()
if err != nil {
log("cmd").Errorf("fail to sendActiveThreadCount(): %d, %v", s.reqId, err)
break
}
time.Sleep(1 * time.Second)
}
s.close()
}

func dumpGoroutine() *GoroutineDump {
//var b bytes.Buffer
//f := bufio.NewWriter(&b)

fn := filepath.Join("/tmp", "goroutine.pprof")
f, err := os.Create(fn)
if err != nil {
log("cmd").Errorf("fail to os.Create(): %v", err)
}

if mp := pprof.Lookup("goroutine"); mp != nil {
mp.WriteTo(f, 2)
}
f.Close()

dump, err := loadProfile(fn)
if err != nil {
log("cmd").Errorf("fail to dumpGoroutine(): %v", err)
return nil
}

return dump
}
9 changes: 9 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,10 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dR
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e h1:fY5BOSpyZCqRo5OhCuC+XN+r/bBCmeuuJtjz+bCNIf8=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
Expand Down Expand Up @@ -107,13 +111,17 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a h1:jmAp/2PZAScNd62lTD3Mcb0Ey9FvIIJtLohPhtxZJ+Q=
github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8=
github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE=
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 h1:mV02weKRL81bEnm8A0HT1/CAelMQDBuQIfLw8n+d6xI=
github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
Expand Down Expand Up @@ -273,6 +281,7 @@ golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200321134203-328b4cd54aae h1:3tcmuaB7wwSZtelmiv479UjUB+vviwABz7a133ZwOKQ=
Expand Down
Loading

0 comments on commit a3e7ece

Please sign in to comment.