Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Backport of Add TCP keepalive settings to proxy config for mesh gateways into release/1.11.x #14998

Merged
merged 1 commit into from Oct 15, 2022
+407 −3
Merged

Backport of Add TCP keepalive settings to proxy config for mesh gateways into release/1.11.x #14998

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions .changelog/14800.txt
View file
Open in desktop
@@ -0,0 +1,3 @@
```release-note:improvement
connect: Added gateway options to Envoy proxy config for enabling tcp keepalives on terminating gateway upstreams and mesh gateways in remote datacenters.
```
17 changes: 17 additions & 0 deletions agent/xds/clusters.go
View file
Open in desktop
Expand Up @@ -953,6 +953,23 @@ func (s *ResourceGenerator) makeGatewayCluster(snap *proxycfg.ConfigSnapshot, op
useEDS = false
}

// TCP keepalive settings can be enabled for terminating gateway upstreams or remote mesh gateways.
remoteUpstream := opts.isRemote || snap.Kind == structs.ServiceKindTerminatingGateway
if remoteUpstream && cfg.TcpKeepaliveEnable {
cluster.UpstreamConnectionOptions = &envoy_cluster_v3.UpstreamConnectionOptions{
TcpKeepalive: &envoy_core_v3.TcpKeepalive{},
}
if cfg.TcpKeepaliveTime != 0 {
cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveTime = makeUint32Value(cfg.TcpKeepaliveTime)
}
if cfg.TcpKeepaliveInterval != 0 {
cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveInterval = makeUint32Value(cfg.TcpKeepaliveInterval)
}
if cfg.TcpKeepaliveProbes != 0 {
cluster.UpstreamConnectionOptions.TcpKeepalive.KeepaliveProbes = makeUint32Value(cfg.TcpKeepaliveProbes)
}
}

// If none of the service instances are addressed by a hostname we provide the endpoint IP addresses via EDS
if useEDS {
cluster.ClusterDiscoveryType = &envoy_cluster_v3.Cluster_Type{Type: envoy_cluster_v3.Cluster_EDS}
Expand Down
23 changes: 23 additions & 0 deletions agent/xds/clusters_test.go
View file
Open in desktop
Expand Up @@ -406,6 +406,17 @@ func TestClustersFromSnapshot(t *testing.T) {
}
},
},
{
name: "mesh-gateway-tcp-keepalives",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
snap := proxycfg.TestConfigSnapshotMeshGateway(t)
snap.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true
snap.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 120
snap.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 60
snap.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 7
return snap
},
},
{
name: "ingress-with-service-max-connections",
create: proxycfg.TestConfigSnapshotIngress,
Expand Down Expand Up @@ -787,6 +798,18 @@ func TestClustersFromSnapshot(t *testing.T) {
}
},
},
{
name: "terminating-gateway-tcp-keepalives",
create: func(t testinf.T) *proxycfg.ConfigSnapshot {
snap := proxycfg.TestConfigSnapshotTerminatingGateway(t)
snap.Proxy.Config = map[string]interface{}{}
snap.Proxy.Config["envoy_gateway_remote_tcp_enable_keepalive"] = true
snap.Proxy.Config["envoy_gateway_remote_tcp_keepalive_time"] = 133
snap.Proxy.Config["envoy_gateway_remote_tcp_keepalive_interval"] = 27
snap.Proxy.Config["envoy_gateway_remote_tcp_keepalive_probes"] = 5
return snap
},
},
{
name: "ingress-multiple-listeners-duplicate-service",
create: proxycfg.TestConfigSnapshotIngress_MultipleListenersDuplicateService,
Expand Down
7 changes: 7 additions & 0 deletions agent/xds/config.go
View file
Open in desktop
Expand Up @@ -118,6 +118,13 @@ type GatewayConfig struct {
// ConnectTimeoutMs is the number of milliseconds to timeout making a new
// connection to this upstream. Defaults to 5000 (5 seconds) if not set.
ConnectTimeoutMs int `mapstructure:"connect_timeout_ms"`

// TCP keepalive settings for remote gateway upstreams (mesh gateways and terminating gateway upstreams).
// See: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/address.proto#envoy-v3-api-msg-config-core-v3-tcpkeepalive
TcpKeepaliveEnable bool `mapstructure:"envoy_gateway_remote_tcp_enable_keepalive"`
TcpKeepaliveTime int `mapstructure:"envoy_gateway_remote_tcp_keepalive_time"`
TcpKeepaliveInterval int `mapstructure:"envoy_gateway_remote_tcp_keepalive_interval"`
TcpKeepaliveProbes int `mapstructure:"envoy_gateway_remote_tcp_keepalive_probes"`
}

// ParseGatewayConfig returns the GatewayConfig parsed from an opaque map. If an
Expand Down
2 changes: 1 addition & 1 deletion ...t/xds/testdata/clusters/ingress-with-defaults-service-max-connections.envoy-1-20-x.golden
View file
Open in desktop
Expand Up @@ -16,7 +16,7 @@
},
"connectTimeout": "33s",
"circuitBreakers": {
"thresholds":[
"thresholds": [
{
"maxConnections": 2048,
"maxPendingRequests": 512,
Expand Down
2 changes: 1 addition & 1 deletion ...data/clusters/ingress-with-overwrite-defaults-service-max-connections.envoy-1-20-x.golden
View file
Open in desktop
Expand Up @@ -16,7 +16,7 @@
},
"connectTimeout": "33s",
"circuitBreakers": {
"thresholds":[
"thresholds": [
{
"maxConnections": 4096,
"maxPendingRequests": 2048
Expand Down
2 changes: 1 addition & 1 deletion agent/xds/testdata/clusters/ingress-with-service-max-connections.envoy-1-20-x.golden
View file
Open in desktop
Expand Up @@ -16,7 +16,7 @@
},
"connectTimeout": "33s",
"circuitBreakers": {
"thresholds":[
"thresholds": [
{
"maxConnections": 4096
}
Expand Down
143 changes: 143 additions & 0 deletions agent/xds/testdata/clusters/mesh-gateway-tcp-keepalives.envoy-1-20-x.golden
View file
Open in desktop
@@ -0,0 +1,143 @@
{
"versionInfo": "00000001",
"resources": [
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "bar.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {

}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "dc2.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {

},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 7,
"keepaliveTime": 120,
"keepaliveInterval": 60
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc4.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-west-2.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "HEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {

},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 7,
"keepaliveTime": 120,
"keepaliveInterval": 60
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "dc6.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "LOGICAL_DNS",
"connectTimeout": "5s",
"loadAssignment": {
"clusterName": "dc6.internal.11111111-2222-3333-4444-555555555555.consul",
"endpoints": [
{
"lbEndpoints": [
{
"endpoint": {
"address": {
"socketAddress": {
"address": "123.us-east-1.elb.notaws.com",
"portValue": 443
}
}
},
"healthStatus": "UNHEALTHY",
"loadBalancingWeight": 1
}
]
}
]
},
"dnsRefreshRate": "10s",
"dnsLookupFamily": "V4_ONLY",
"outlierDetection": {

},
"upstreamConnectionOptions": {
"tcpKeepalive": {
"keepaliveProbes": 7,
"keepaliveTime": 120,
"keepaliveInterval": 60
}
}
},
{
"@type": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"name": "foo.default.dc1.internal.11111111-2222-3333-4444-555555555555.consul",
"type": "EDS",
"edsClusterConfig": {
"edsConfig": {
"ads": {

},
"resourceApiVersion": "V3"
}
},
"connectTimeout": "5s",
"outlierDetection": {

}
}
],
"typeUrl": "type.googleapis.com/envoy.config.cluster.v3.Cluster",
"nonce": "00000001"
}