Increase gRPC request timeout to 45 seconds for sending snapshots

Signed-off-by: Nishant Totla <nishanttotla@gmail.com>

manager/state/raft/transport/peer.go

@@ -133,7 +133,13 @@ func (p *peer) resolveAddr(ctx context.Context, id uint64) (string, error) {
 }
 
 func (p *peer) sendProcessMessage(ctx context.Context, m raftpb.Message) error {
-	ctx, cancel := context.WithTimeout(ctx, p.tr.config.SendTimeout)
+	timeout := p.tr.config.SendTimeout
+	// adjust timeout to be higher for when a snapshot is being sent. This
+	// is to accommodate for the fact that snapshots can be large.
+	if m.Type == raftpb.MsgSnap {
+		timeout = 45 * time.Second
+	}
+	ctx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 	_, err := api.NewRaftClient(p.conn()).ProcessRaftMessage(ctx, &api.ProcessRaftMessageRequest{Message: &m})
 	if grpc.Code(err) == codes.NotFound && grpc.ErrorDesc(err) == membership.ErrMemberRemoved.Error() {