[FAB-7035] Configure keepalive params for peer

Currently all of the various client/server
gRPC keepalive parameters are hard-coded.
This change makes the following settings
configurable:

- minimum permitted interval for client
 pings on the peer endpoint
- keepalive interval and timeout for
 connections to other peers
- keepalive interval and timeout for
 connections to orderer nodes
- minimum permitted interval for client
 pings to the events endpoint

Change-Id: I17b54a68cc2d974f22efd72e1c7892b284ac6506
Signed-off-by: Gari Singh <gari.r.singh@gmail.com>

Author: mastersingh24

core/comm/config.go

@@ -119,6 +119,12 @@ func SetMaxSendMsgSize(size int) {
 	maxSendMsgSize = size
 }
 
+// DefaultKeepaliveOptions returns sane default keepalive settings for gRPC
+// servers and clients
+func DefaultKeepaliveOptions() *KeepaliveOptions {
+	return keepaliveOptions
+}
+
 // ServerKeepaliveOptions returns gRPC keepalive options for server.  If
 // opts is nil, the default keepalive options are returned
 func ServerKeepaliveOptions(ka *KeepaliveOptions) []grpc.ServerOption {

core/comm/config_test.go

@@ -17,6 +17,7 @@ func TestConfig(t *testing.T) {
 	// check the defaults
 	assert.EqualValues(t, maxRecvMsgSize, MaxRecvMsgSize())
 	assert.EqualValues(t, maxSendMsgSize, MaxSendMsgSize())
+	assert.EqualValues(t, keepaliveOptions, DefaultKeepaliveOptions())
 	assert.EqualValues(t, false, TLSEnabled())
 	assert.EqualValues(t, true, configurationCached)
 

core/deliverservice/deliveryclient.go

@@ -20,6 +20,7 @@ import (
 	"github.com/hyperledger/fabric/gossip/util"
 	"github.com/hyperledger/fabric/protos/orderer"
 	"github.com/op/go-logging"
+	"github.com/spf13/viper"
 	"golang.org/x/net/context"
 	"google.golang.org/grpc"
 )
@@ -226,7 +227,16 @@ func DefaultConnectionFactory(channelID string) func(endpoint string) (*grpc.Cli
 		dialOpts = append(dialOpts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(comm.MaxRecvMsgSize()),
 			grpc.MaxCallSendMsgSize(comm.MaxSendMsgSize())))
 		// set the keepalive options
-		dialOpts = append(dialOpts, comm.ClientKeepaliveOptions(nil)...)
+		kaOpts := comm.DefaultKeepaliveOptions()
+		if viper.IsSet("peer.keepalive.deliveryClient.interval") {
+			kaOpts.ClientInterval = viper.GetDuration(
+				"peer.keepalive.deliveryClient.interval")
+		}
+		if viper.IsSet("peer.keepalive.deliveryClient.timeout") {
+			kaOpts.ClientTimeout = viper.GetDuration(
+				"peer.keepalive.deliveryClient.timeout")
+		}
+		dialOpts = append(dialOpts, comm.ClientKeepaliveOptions(kaOpts)...)
 
 		if comm.TLSEnabled() {
 			creds, err := comm.GetCredentialSupport().GetDeliverServiceCredentials(channelID)

core/peer/config.go

@@ -153,5 +153,11 @@ func GetServerConfig() (comm.ServerConfig, error) {
 			secureOptions.ServerRootCAs = [][]byte{rootCert}
 		}
 	}
+	// get the default keepalive options
+	serverConfig.KaOpts = comm.DefaultKeepaliveOptions()
+	// check to see if minInterval is set for the env
+	if viper.IsSet("peer.keepalive.minInterval") {
+		serverConfig.KaOpts.ServerMinInterval = viper.GetDuration("peer.keepalive.minInterval")
+	}
 	return serverConfig, nil
 }

core/peer/config_test.go

@@ -9,7 +9,9 @@ import (
 	"net"
 	"path/filepath"
 	"testing"
+	"time"
 
+	"github.com/hyperledger/fabric/core/comm"
 	"github.com/spf13/viper"
 	"github.com/stretchr/testify/assert"
 )
@@ -123,6 +125,14 @@ func TestGetServerConfig(t *testing.T) {
 	assert.Equal(t, false, sc.SecOpts.UseTLS,
 		"ServerConfig.SecOpts.UseTLS should be false")
 
+	// keepalive options
+	assert.Equal(t, comm.DefaultKeepaliveOptions(), sc.KaOpts,
+		"ServerConfig.KaOpts should be set to default values")
+	viper.Set("peer.keepalive.minInterval", "2m")
+	sc, _ = GetServerConfig()
+	assert.Equal(t, time.Duration(2)*time.Minute, sc.KaOpts.ServerMinInterval,
+		"ServerConfig.KaOpts.ServerMinInterval should be set to 2 min")
+
 	// good config with TLS
 	viper.Set("peer.tls.enabled", true)
 	viper.Set("peer.tls.cert.file", filepath.Join("testdata", "Org1-server1-cert.pem"))

peer/node/start.go

@@ -207,7 +207,14 @@ func serve(args []string) error {
 		dialOpts = append(dialOpts, grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(comm.MaxRecvMsgSize()),
 			grpc.MaxCallSendMsgSize(comm.MaxSendMsgSize())))
 		// set the keepalive options
-		dialOpts = append(dialOpts, comm.ClientKeepaliveOptions(nil)...)
+		kaOpts := comm.DefaultKeepaliveOptions()
+		if viper.IsSet("peer.keepalive.client.interval") {
+			kaOpts.ClientInterval = viper.GetDuration("peer.keepalive.client.interval")
+		}
+		if viper.IsSet("peer.keepalive.client.timeout") {
+			kaOpts.ClientTimeout = viper.GetDuration("peer.keepalive.client.timeout")
+		}
+		dialOpts = append(dialOpts, comm.ClientKeepaliveOptions(kaOpts)...)
 
 		if comm.TLSEnabled() {
 			comm.GetCredentialSupport().ClientCert = peerServer.ServerCertificate()
@@ -388,6 +395,12 @@ func createEventHubServer(serverConfig comm.ServerConfig) (comm.GRPCServer, erro
 		return nil, fmt.Errorf("failed to listen: %v", err)
 	}
 
+	// set the keepalive options
+	serverConfig.KaOpts = comm.DefaultKeepaliveOptions()
+	if viper.IsSet("peer.events.keepalive.minInterval") {
+		serverConfig.KaOpts.ServerMinInterval = viper.GetDuration("peer.events.keepalive.minInterval")
+	}
+
 	grpcServer, err := comm.NewGRPCServerFromListener(lis, serverConfig)
 	if err != nil {
 		logger.Errorf("Failed to return new GRPC server: %s", err)

sampleconfig/core.yaml

@@ -93,6 +93,33 @@ peer:
     # current setting
     gomaxprocs: -1
 
+    # Keepalive settings for peer server and clients
+    keepalive:
+        # MinInterval is the minimum permitted time between client pings.
+        # If clients send pings more frequently, the peer server will
+        # disconnect them
+        minInterval: 60s
+        # Client keepalive settings for communicating with other peer nodes
+        client:
+            # Interval is the time between pings to peer nodes.  This must
+            # greater than or equal to the minInterval specified by peer
+            # nodes
+            interval: 60s
+            # Timeout is the duration the client waits for a response from
+            # peer nodes before closing the connection
+            timeout: 20s
+        # DeliveryClient keepalive settings for communication with ordering
+        # nodes.
+        deliveryClient:
+            # Interval is the time between pings to ordering nodes.  This must
+            # greater than or equal to the minInterval specified by ordering
+            # nodes.
+            interval: 60s
+            # Timeout is the duration the client waits for a response from
+            # ordering nodes before closing the connection
+            timeout: 20s
+
+
     # Gossip related configuration
     gossip:
         # Bootstrap set to initialize gossip with.
@@ -219,6 +246,13 @@ peer:
         # time and the client's time as specified in a registration event
         timewindow: 15m
 
+        # Keepalive settings for peer server and clients
+        keepalive:
+            # MinInterval is the minimum permitted time in seconds which clients
+            # can send keepalive pings.  If clients send pings more frequently,
+            # the events server will disconnect them
+            minInterval: 60s
+
     # TLS Settings
     # Note that peer-chaincode connections through chaincodeListenAddress is
     # not mutual TLS auth. See comments on chaincodeListenAddress for more info