Make keep alive configurable and opt-in (#371)

internal/client.go

@@ -405,6 +405,20 @@ type (
 		TLS                *tls.Config
 		DisableHealthCheck bool
 		HealthCheckTimeout time.Duration
+		// Enables keep alive ping from client to the server, which can help detect abruptly closed connections faster.
+		EnableKeepAliveCheck bool
+		// After a duration of this time if the client doesn't see any activity it
+		// pings the server to see if the transport is still alive.
+		// If set below 10s, a minimum value of 10s will be used instead.
+		KeepAliveTime time.Duration
+		// After having pinged for keepalive check, the client waits for a duration
+		// of Timeout and if no activity is seen even after that the connection is
+		// closed.
+		KeepAliveTimeout time.Duration
+		// If true, client sends keepalive pings even with no active RPCs. If false,
+		// when there are no active RPCs, Time and Timeout will be ignored and no
+		// keepalive pings will be sent.
+		KeepAlivePermitWithoutStream bool
 	}
 
 	// StartWorkflowOptions configuration parameters for starting a workflow execution.

internal/grpc_dialer.go

@@ -79,23 +79,25 @@ func dial(params dialParameters) (*grpc.ClientConn, error) {
 	}
 	cp.Backoff.BaseDelay = retryPollOperationInitialInterval
 	cp.Backoff.MaxDelay = retryPollOperationMaxInterval
-
-	// gRPC utilizes keep alive mechanism to detect dead connections in case if server didn't close them
-	// gracefully. Client would ping the server periodically and expect replies withing the specified timeout.
-	// Learn more by reading https://github.com/grpc/grpc/blob/master/doc/keepalive.md
-	var kap = keepalive.ClientParameters{
-		Time:                30 * time.Second,
-		Timeout:             15 * time.Second,
-		PermitWithoutStream: true,
-	}
-
-	return grpc.Dial(params.HostPort,
+	opts := []grpc.DialOption{
 		grpcSecurityOptions,
 		grpc.WithChainUnaryInterceptor(params.RequiredInterceptors...),
 		grpc.WithDefaultServiceConfig(params.DefaultServiceConfig),
 		grpc.WithConnectParams(cp),
-		grpc.WithKeepaliveParams(kap),
-	)
+	}
+
+	if params.UserConnectionOptions.EnableKeepAliveCheck {
+		// gRPC utilizes keep alive mechanism to detect dead connections in case if server didn't close them
+		// gracefully. Client would ping the server periodically and expect replies withing the specified timeout.
+		// Learn more by reading https://github.com/grpc/grpc/blob/master/doc/keepalive.md
+		var kap = keepalive.ClientParameters{
+			Time:                params.UserConnectionOptions.KeepAliveTime,
+			Timeout:             params.UserConnectionOptions.KeepAliveTimeout,
+			PermitWithoutStream: params.UserConnectionOptions.KeepAlivePermitWithoutStream,
+		}
+		opts = append(opts, grpc.WithKeepaliveParams(kap))
+	}
+	return grpc.Dial(params.HostPort, opts...)
 }
 
 func requiredInterceptors(metricScope tally.Scope, headersProvider HeadersProvider) []grpc.UnaryClientInterceptor {