Introduce configurable values for protocol detection (#11536)

This change allows users to configure protocol detection timeout values
(outbound and inbound). Certain environments may find that protocol
detection inhibits debugging and makes it harder to reason with a
client's behaviour. In such cases (and not only) it may be deseriable to
change the default protocol detection timeout to a higher value than the
default 10s.

Through this change, users may configure their timeout values either
with install-time settings or through annotations; this follows our
usual proxy configuration model. The proxy uses different timeout values
for the inbound and outbound stacks (even though they use the same
default value) and this change respects that by adding two separate
fields.

Signed-off-by: Matei David <matei@buoyant.io>

charts/linkerd-control-plane/README.md

@@ -229,6 +229,8 @@ Kubernetes: `>=1.21.0-0`
 | proxy.await | bool | `true` | If set, the application container will not start until the proxy is ready |
 | proxy.cores | int | `0` | The `cpu.limit` and `cores` should be kept in sync. The value of `cores` must be an integer and should typically be set by rounding up from the limit. E.g. if cpu.limit is '1500m', cores should be 2. |
 | proxy.defaultInboundPolicy | string | "all-unauthenticated" | The default allow policy to use when no `Server` selects a pod.  One of: "all-authenticated", "all-unauthenticated", "cluster-authenticated", "cluster-unauthenticated", "deny" |
+| proxy.disableInboundProtocolDetectTimeout | bool | `false` | When set to true, disables the protocol detection timeout on the inbound side of the proxy by setting it to a very high value |
+| proxy.disableOutboundProtocolDetectTimeout | bool | `false` | When set to true, disables the protocol detection timeout on the outbound side of the proxy by setting it to a very high value |
 | proxy.enableExternalProfiles | bool | `false` | Enable service profiles for non-Kubernetes services |
 | proxy.image.name | string | `"cr.l5d.io/linkerd/proxy"` | Docker image for the proxy |
 | proxy.image.pullPolicy | string | imagePullPolicy | Pull policy for the proxy container image |

charts/linkerd-control-plane/values.yaml

@@ -115,6 +115,12 @@ proxy:
   # -- Maximum time allowed before an unused inbound discovery result
   # is evicted from the cache
   inboundDiscoveryCacheUnusedTimeout: "90s"
+  # -- When set to true, disables the protocol detection timeout on the
+  # outbound side of the proxy by setting it to a very high value
+  disableOutboundProtocolDetectTimeout: false
+  # -- When set to true, disables the protocol detection timeout on the inbound
+  # side of the proxy by setting it to a very high value
+  disableInboundProtocolDetectTimeout: false
   image:
     # -- Docker image for the proxy
     name: cr.l5d.io/linkerd/proxy

charts/partials/templates/_proxy.tpl

@@ -57,6 +57,14 @@ env:
 - name: LINKERD2_PROXY_INBOUND_DISCOVERY_IDLE_TIMEOUT
   value: {{.Values.proxy.inboundDiscoveryCacheUnusedTimeout | quote}}
 {{ end -}}
+{{ if .Values.proxy.DisableOutboundProtocolDetectTimeout -}}
+- name: LINKERD2_PROXY_OUTBOUND_DETECT_TIMEOUT
+  value: "365d"
+{{ end -}}
+{{ if .Values.proxy.DisableInboundProtocolDetectTimeout -}}
+- name: LINKERD2_PROXY_INBOUND_DETECT_TIMEOUT
+  value: "365d"
+{{ end -}}
 - name: LINKERD2_PROXY_CONTROL_LISTEN_ADDR
   value: 0.0.0.0:{{.Values.proxy.ports.control}}
 - name: LINKERD2_PROXY_ADMIN_LISTEN_ADDR

cli/cmd/doc.go

@@ -252,6 +252,14 @@ func generateAnnotationsDocs() []annotationDoc {
 			Name:        k8s.ProxyInboundDiscoveryCacheUnusedTimeout,
 			Description: "Maximum time allowed before an unused inbound discovery result is evicted from the cache. Defaults to `90s`",
 		},
+		{
+			Name:        k8s.ProxyDisableOutboundProtocolDetectTimeout,
+			Description: "When set to true, disables the protocol detection timeout on the outbound side of the proxy by setting it to a very high value",
+		},
+		{
+			Name:        k8s.ProxyDisableInboundProtocolDetectTimeout,
+			Description: "When set to true, disables the protocol detection timeout on the inbound side of the proxy by setting it to a very high value",
+		},
 		{
 			Name:        k8s.ProxyWaitBeforeExitSecondsAnnotation,
 			Description: "The proxy sidecar will stay alive for at least the given period after receiving SIGTERM signal from Kubernetes but no longer than pod's `terminationGracePeriodSeconds`. Defaults to `0`",

pkg/charts/linkerd2/values.go

@@ -94,29 +94,31 @@ type (
 	Proxy struct {
 		Capabilities *Capabilities `json:"capabilities"`
 		// This should match .Resources.CPU.Limit, but must be a whole number
-		Cores                               int64            `json:"cores,omitempty"`
-		EnableExternalProfiles              bool             `json:"enableExternalProfiles"`
-		Image                               *Image           `json:"image"`
-		LogLevel                            string           `json:"logLevel"`
-		LogFormat                           string           `json:"logFormat"`
-		SAMountPath                         *VolumeMountPath `json:"saMountPath"`
-		Ports                               *Ports           `json:"ports"`
-		Resources                           *Resources       `json:"resources"`
-		UID                                 int64            `json:"uid"`
-		WaitBeforeExitSeconds               uint64           `json:"waitBeforeExitSeconds"`
-		IsGateway                           bool             `json:"isGateway"`
-		IsIngress                           bool             `json:"isIngress"`
-		RequireIdentityOnInboundPorts       string           `json:"requireIdentityOnInboundPorts"`
-		OutboundConnectTimeout              string           `json:"outboundConnectTimeout"`
-		InboundConnectTimeout               string           `json:"inboundConnectTimeout"`
-		OutboundDiscoveryCacheUnusedTimeout string           `json:"outboundDiscoveryCacheUnusedTimeout"`
-		InboundDiscoveryCacheUnusedTimeout  string           `json:"inboundDiscoveryCacheUnusedTimeout"`
-		PodInboundPorts                     string           `json:"podInboundPorts"`
-		OpaquePorts                         string           `json:"opaquePorts"`
-		Await                               bool             `json:"await"`
-		DefaultInboundPolicy                string           `json:"defaultInboundPolicy"`
-		AccessLog                           string           `json:"accessLog"`
-		ShutdownGracePeriod                 string           `json:"shutdownGracePeriod"`
+		Cores                                int64            `json:"cores,omitempty"`
+		EnableExternalProfiles               bool             `json:"enableExternalProfiles"`
+		Image                                *Image           `json:"image"`
+		LogLevel                             string           `json:"logLevel"`
+		LogFormat                            string           `json:"logFormat"`
+		SAMountPath                          *VolumeMountPath `json:"saMountPath"`
+		Ports                                *Ports           `json:"ports"`
+		Resources                            *Resources       `json:"resources"`
+		UID                                  int64            `json:"uid"`
+		WaitBeforeExitSeconds                uint64           `json:"waitBeforeExitSeconds"`
+		IsGateway                            bool             `json:"isGateway"`
+		IsIngress                            bool             `json:"isIngress"`
+		RequireIdentityOnInboundPorts        string           `json:"requireIdentityOnInboundPorts"`
+		OutboundConnectTimeout               string           `json:"outboundConnectTimeout"`
+		InboundConnectTimeout                string           `json:"inboundConnectTimeout"`
+		OutboundDiscoveryCacheUnusedTimeout  string           `json:"outboundDiscoveryCacheUnusedTimeout"`
+		InboundDiscoveryCacheUnusedTimeout   string           `json:"inboundDiscoveryCacheUnusedTimeout"`
+		DisableOutboundProtocolDetectTimeout bool             `json:"disableOutboundProtocolDetectTimeout"`
+		DisableInboundProtocolDetectTimeout  bool             `json:"disableInboundProtocolDetectTimeout"`
+		PodInboundPorts                      string           `json:"podInboundPorts"`
+		OpaquePorts                          string           `json:"opaquePorts"`
+		Await                                bool             `json:"await"`
+		DefaultInboundPolicy                 string           `json:"defaultInboundPolicy"`
+		AccessLog                            string           `json:"accessLog"`
+		ShutdownGracePeriod                  string           `json:"shutdownGracePeriod"`
 	}
 
 	// ProxyInit contains the fields to set the proxy-init container

pkg/charts/linkerd2/values_test.go

@@ -123,15 +123,17 @@ func TestNewValues(t *testing.T) {
 					Request: "",
 				},
 			},
-			UID:                                 2102,
-			WaitBeforeExitSeconds:               0,
-			OutboundConnectTimeout:              "1000ms",
-			InboundConnectTimeout:               "100ms",
-			OpaquePorts:                         "25,587,3306,4444,5432,6379,9300,11211",
-			Await:                               true,
-			DefaultInboundPolicy:                "all-unauthenticated",
-			OutboundDiscoveryCacheUnusedTimeout: "5s",
-			InboundDiscoveryCacheUnusedTimeout:  "90s",
+			UID:                                  2102,
+			WaitBeforeExitSeconds:                0,
+			OutboundConnectTimeout:               "1000ms",
+			InboundConnectTimeout:                "100ms",
+			OpaquePorts:                          "25,587,3306,4444,5432,6379,9300,11211",
+			Await:                                true,
+			DefaultInboundPolicy:                 "all-unauthenticated",
+			OutboundDiscoveryCacheUnusedTimeout:  "5s",
+			InboundDiscoveryCacheUnusedTimeout:   "90s",
+			DisableOutboundProtocolDetectTimeout: false,
+			DisableInboundProtocolDetectTimeout:  false,
 		},
 		ProxyInit: &ProxyInit{
 			IptablesMode:        "legacy",

pkg/inject/inject.go

@@ -75,6 +75,8 @@ var (
 		k8s.ProxyShutdownGracePeriodAnnotation,
 		k8s.ProxyOutboundDiscoveryCacheUnusedTimeout,
 		k8s.ProxyInboundDiscoveryCacheUnusedTimeout,
+		k8s.ProxyDisableOutboundProtocolDetectTimeout,
+		k8s.ProxyDisableInboundProtocolDetectTimeout,
 	}
 	// ProxyAlphaConfigAnnotations is the list of all alpha configuration
 	// (config.alpha prefix) that can be applied to a pod or namespace.
@@ -954,6 +956,24 @@ func (conf *ResourceConfig) applyAnnotationOverrides(values *l5dcharts.Values) {
 		}
 	}
 
+	if override, ok := annotations[k8s.ProxyDisableOutboundProtocolDetectTimeout]; ok {
+		value, err := strconv.ParseBool(override)
+		if err == nil {
+			values.Proxy.DisableOutboundProtocolDetectTimeout = value
+		} else {
+			log.Warnf("unrecognised value used on pod annotation %s: %s", k8s.ProxyDisableOutboundProtocolDetectTimeout, err.Error())
+		}
+	}
+
+	if override, ok := annotations[k8s.ProxyDisableInboundProtocolDetectTimeout]; ok {
+		value, err := strconv.ParseBool(override)
+		if err == nil {
+			values.Proxy.DisableInboundProtocolDetectTimeout = value
+		} else {
+			log.Warnf("unrecognised value used on pod annotation %s: %s", k8s.ProxyDisableInboundProtocolDetectTimeout, err.Error())
+		}
+	}
+
 	if override, ok := annotations[k8s.ProxyShutdownGracePeriodAnnotation]; ok {
 		duration, err := time.ParseDuration(override)
 		if err != nil {