Skip to content

Commit

Permalink
backport of commit aaf892a
Browse files Browse the repository at this point in the history
  • Loading branch information
kyhavlov committed Oct 15, 2022
1 parent f7a408f commit 2c77ae7
Show file tree
Hide file tree
Showing 6 changed files with 227 additions and 21 deletions.
2 changes: 1 addition & 1 deletion .changelog/14800.txt
@@ -1,3 +1,3 @@
```release-note:improvement
connect: Added gateway options to Envoy proxy config for enabling tcp keepalives on upstream connections to mesh gateways in remote datacenters.
connect: Added gateway options to Envoy proxy config for enabling tcp keepalives on terminating gateway upstreams and mesh gateways in remote datacenters.
```
5 changes: 3 additions & 2 deletions agent/xds/clusters.go
Expand Up @@ -982,8 +982,9 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op
useEDS = false
}

// Set TCP keepalive settings on the upstream gateway cluster if enabled.
if opts.isRemote && cfg.TcpKeepaliveEnable {
// TCP keepalive settings can be enabled for terminating gateway upstreams or remote mesh gateways.
remoteUpstream := opts.isRemote || snap.Kind == structs.ServiceKindTerminatingGateway
if remoteUpstream && cfg.TcpKeepaliveEnable {
cluster.UpstreamConnectionOptions = &envoy_cluster_v3.UpstreamConnectionOptions{
TcpKeepalive: &envoy_core_v3.TcpKeepalive{},
}
Expand Down
22 changes: 18 additions & 4 deletions agent/xds/clusters_test.go
Expand Up @@ -412,10 +412,10 @@ func TestClustersFromSnapshot(t *testing.T) {
name: "mesh-gateway-tcp-keepalives",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotMeshGateway(t, "default", func(ns *structs.NodeService) {
ns.Proxy.Config["envoy_mesh_gateway_tcp_enable_keepalive"] = true
ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_time"] = 120
ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_interval"] = 60
ns.Proxy.Config["envoy_mesh_gateway_tcp_keepalive_probes"] = 7
ns.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 120
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 60
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 7
}, nil)
},
},
Expand Down Expand Up @@ -663,6 +663,20 @@ func TestClustersFromSnapshot(t *testing.T) {
name: "terminating-gateway-lb-config",
create: proxycfg.TestConfigSnapshotTerminatingGatewayLBConfigNoHashPolicies,
},
{
name: "terminating-gateway-tcp-keepalives",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
return proxycfg.TestConfigSnapshotTerminatingGateway(t, true, func(ns *structs.NodeService) {
if ns.Proxy.Config == nil {
ns.Proxy.Config = map[string]interface{}{}
}
ns.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 133
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 27
ns.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 5
}, nil)
},
},
{
name: "ingress-multiple-listeners-duplicate-service",
create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService,
Expand Down
10 changes: 5 additions & 5 deletions agent/xds/config.go
Expand Up @@ -119,12 +119,12 @@ type GatewayConfig struct {
// connection to this upstream. Defaults to 5000 (5 seconds) if not set.
ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"`

// TCP keepalive settings for connections between remote mesh gateways.
// TCP keepalive settings for remote gateway upstreams (mesh gateways and terminating gateway upstreams).
// See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive
TcpKeepaliveEnable bool `mapstructure:"envoy_mesh_gateway_tcp_enable_keepalive"`
TcpKeepaliveTime int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_time"`
TcpKeepaliveInterval int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_interval"`
TcpKeepaliveProbes int `mapstructure:"envoy_mesh_gateway_tcp_keepalive_probes"`
TcpKeepaliveEnable bool `mapstructure:"envoy_gateway_remote_tcp_enable_keepalive"`
TcpKeepaliveTime int `mapstructure:"envoy_gateway_remote_tcp_keepalive_time"`
TcpKeepaliveInterval int `mapstructure:"envoy_gateway_remote_tcp_keepalive_interval"`
TcpKeepaliveProbes int `mapstructure:"envoy_gateway_remote_tcp_keepalive_probes"`
}

// ParseGatewayConfig returns the GatewayConfig parsed from an opaque map. If an
Expand Down
@@ -0,0 +1,190 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "api.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "api.altdomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {

},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {

},
"tlsCertificates": [
{
"certificateChain": {
"filename": "api.cert.pem"
},
"privateKey": {
"filename": "api.key.pem"
}
}
],
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
}
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "cache.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "cache.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {

},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "db.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "db.mydomain",
"portValue": 8081
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {

},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "web.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {

},
"transportSocket": {
"name": "tls",
"typedConfig": {
"@type": "type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext",
"commonTlsContext": {
"tlsParams": {

},
"validationContext": {
"trustedCa": {
"filename": "ca.cert.pem"
}
}
}
}
},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 5,
"keepaliveTime": 133,
"keepaliveInterval": 27
}
}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}
19 changes: 10 additions & 9 deletions website/content/docs/connect/proxies/envoy.mdx
Expand Up @@ -352,24 +352,25 @@ will continue to be supported.
addressed by a hostname, such as a managed database. It also applies to mesh gateways,
such as when gateways in other Consul datacenters are behind a load balancer that is addressed by a hostname.

- `envoy_mesh_gateway_tcp_enable_keepalive` - Enables TCP keepalive settings on mesh gateway upstream connections
to remote datacenters. Defaults to `false`. Must be one of `true` or `false`. Details for this feature are available in
the [Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
- `envoy_gateway_remote_tcp_enable_keepalive` - Enables TCP keepalive settings on remote
upstream connections for mesh and terminating gateways. Defaults to `false`. Must be one
of `true` or `false`. Details for this feature are available in the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).

- `envoy_mesh_gateway_tcp_keepalive_time` - The number of seconds a connection needs to
- `envoy_gateway_remote_tcp_keepalive_time` - The number of seconds a connection needs to
be idle before keep-alive probes start being sent. For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to mesh gateway upstream connections to remote datacenters.
This option only applies to remote upstream connections for mesh and terminating gateways.

- `envoy_mesh_gateway_tcp_keepalive_interval` - The number of seconds between keep-alive probes.
- `envoy_gateway_remote_tcp_keepalive_interval` - The number of seconds between keep-alive probes.
For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to mesh gateway upstream connections to remote datacenters.
This option only applies to remote upstream connections for mesh and terminating gateways.

- `envoy_mesh_gateway_tcp_keepalive_probes` - Maximum number of keepalive probes to send without
- `envoy_gateway_remote_tcp_keepalive_probes` - Maximum number of keepalive probes to send without
response before deciding the connection is dead. For more information, see the
[Envoy documentation](https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive).
This option only applies to mesh gateway upstream connections to remote datacenters.
This option only applies to remote upstream connections for mesh and terminating gateways.

## Advanced Configuration

Expand Down

0 comments on commit 2c77ae7

Please sign in to comment.