Merge pull request #4954 from pachyderm/collect-sidecar-dump

Collect sidecar goroutine dumps in debug dump
pachyderm · Jun 8, 2020 · 0a2998d · 0a2998d
2 parents 2c7247a + f6dfbd4
commit 0a2998d
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 10 deletions.
diff --git a/src/server/cmd/pachd/main.go b/src/server/cmd/pachd/main.go
@@ -260,6 +260,7 @@ func doSidecarMode(config interface{}) (retErr error) {
 			path.Join(env.EtcdPrefix, env.PPSEtcdPrefix),
 			env.PPSWorkerPort,
 			clusterID,
+			nil,
 		))
 		return nil
 	}); err != nil {
@@ -492,6 +493,7 @@ func doFullMode(config interface{}) (retErr error) {
 				path.Join(env.EtcdPrefix, env.PPSEtcdPrefix),
 				env.PPSWorkerPort,
 				clusterID,
+				nil,
 			))
 			return nil
 		}); err != nil {

diff --git a/src/server/cmd/worker/main.go b/src/server/cmd/worker/main.go
@@ -96,7 +96,7 @@ func do(config interface{}) error {
 
 	workerserver.RegisterWorkerServer(server.Server, workerInstance.APIServer)
 	versionpb.RegisterAPIServer(server.Server, version.NewAPIServer(version.Version, version.APIServerOptions{}))
-	debugclient.RegisterDebugServer(server.Server, debugserver.NewDebugServer(env.PodName, env.GetEtcdClient(), env.PPSEtcdPrefix, env.PPSWorkerPort, ""))
+	debugclient.RegisterDebugServer(server.Server, debugserver.NewDebugServer(env.PodName, env.GetEtcdClient(), env.PPSEtcdPrefix, env.PPSWorkerPort, "", pachClient))
 
 	// Put our IP address into etcd, so pachd can discover us
 	key := path.Join(env.PPSEtcdPrefix, workerserver.WorkerEtcdPrefix, workerRcName, env.PPSWorkerIP)

diff --git a/src/server/debug/server/server.go b/src/server/debug/server/server.go
@@ -9,6 +9,7 @@ import (
 
 	etcd "github.com/coreos/etcd/clientv3"
 	"github.com/gogo/protobuf/types"
+	"github.com/pachyderm/pachyderm/src/client"
 	"github.com/pachyderm/pachyderm/src/client/debug"
 	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
 	"github.com/pachyderm/pachyderm/src/client/pkg/grpcutil"
@@ -20,13 +21,14 @@ const (
 )
 
 // NewDebugServer creates a new server that serves the debug api over GRPC
-func NewDebugServer(name string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16, clusterID string) debug.DebugServer {
+func NewDebugServer(name string, etcdClient *etcd.Client, etcdPrefix string, workerGrpcPort uint16, clusterID string, sidecarClient *client.APIClient) debug.DebugServer {
 	return &debugServer{
 		name:           name,
 		etcdClient:     etcdClient,
 		etcdPrefix:     etcdPrefix,
 		workerGrpcPort: workerGrpcPort,
 		clusterID:      clusterID,
+		sidecarClient:  sidecarClient,
 	}
 }
 
@@ -36,6 +38,7 @@ type debugServer struct {
 	etcdPrefix     string
 	workerGrpcPort uint16
 	clusterID      string
+	sidecarClient  *client.APIClient
 }
 
 func (s *debugServer) Dump(request *debug.DumpRequest, server debug.Debug_DumpServer) error {
@@ -48,21 +51,20 @@ func (s *debugServer) Dump(request *debug.DumpRequest, server debug.Debug_DumpSe
 		if _, err := fmt.Fprintf(w, "== %s ==\n\n", s.name); err != nil {
 			return err
 		}
+	} else {
+		if _, err := fmt.Fprintf(w, "== pachd ==\n\n"); err != nil {
+			return err
+		}
 	}
 	if err := profile.WriteTo(w, 2); err != nil {
 		return err
 	}
-	if !request.Recursed {
-		request.Recursed = true
-		cs, err := workerserver.Clients(server.Context(), "", s.etcdClient, s.etcdPrefix, s.workerGrpcPort)
-		if err != nil {
-			return err
-		}
-		for _, c := range cs {
+	if request.Recursed {
+		if s.sidecarClient != nil {
 			if _, err := fmt.Fprintf(w, "\n"); err != nil {
 				return err
 			}
-			dumpC, err := c.Dump(
+			dumpC, err := s.sidecarClient.DebugClient.Dump(
 				server.Context(),
 				request,
 			)
@@ -73,6 +75,27 @@ func (s *debugServer) Dump(request *debug.DumpRequest, server debug.Debug_DumpSe
 				return err
 			}
 		}
+		return nil
+	}
+	request.Recursed = true
+	cs, err := workerserver.Clients(server.Context(), "", s.etcdClient, s.etcdPrefix, s.workerGrpcPort)
+	if err != nil {
+		return err
+	}
+	for _, c := range cs {
+		if _, err := fmt.Fprintf(w, "\n"); err != nil {
+			return err
+		}
+		dumpC, err := c.Dump(
+			server.Context(),
+			request,
+		)
+		if err != nil {
+			return err
+		}
+		if err := grpcutil.WriteFromStreamingBytesClient(dumpC, w); err != nil {
+			return err
+		}
 	}
 	return nil
 }