Support for resources opting-out of tap (#2807)

Support for resources opting out of tap

Implements the `linkerd inject --disable-tap` flag (although hidden pending #2811) and the config override annotation `config.linkerd.io/disable-tap`.
Fixes #2778

Signed-off-by: Alejandro Pedraza <alejandro@buoyant.io>

cli/cmd/inject.go

@@ -103,6 +103,14 @@ sub-folders, or coming from stdin.`,
 		&options.disableIdentity, "disable-identity", options.disableIdentity,
 		"Disables resources from participating in TLS identity",
 	)
+
+	flags.BoolVar(
+		&options.disableTap, "disable-tap", options.disableTap,
+		"Disables resources from from being tapped",
+	)
+	// hidden till support on the proxy-side (#2811) lands
+	flags.MarkHidden("disable-tap")
+
 	flags.BoolVar(
 		&options.ignoreCluster, "ignore-cluster", options.ignoreCluster,
 		"Ignore the current Kubernetes cluster when checking for existing cluster configuration (default false)",
@@ -378,15 +386,19 @@ func (options *proxyConfigOptions) overrideConfigs(configs *cfg.All, overrideAnn
 
 	if options.disableIdentity {
 		configs.Global.IdentityContext = nil
-		overrideAnnotations[k8s.ProxyDisableIdentityAnnotation] = "true"
+		overrideAnnotations[k8s.ProxyDisableIdentityAnnotation] = strconv.FormatBool(true)
+	}
+
+	if options.disableTap {
+		overrideAnnotations[k8s.ProxyDisableTapAnnotation] = strconv.FormatBool(true)
 	}
 
 	// keep track of this option because its true/false value results in different
 	// values being assigned to the LINKERD2_PROXY_DESTINATION_PROFILE_SUFFIXES
 	// env var. Its annotation is added only if its value is true.
 	configs.Proxy.DisableExternalProfiles = !options.enableExternalProfiles
 	if options.enableExternalProfiles {
-		overrideAnnotations[k8s.ProxyEnableExternalProfilesAnnotation] = "true"
+		overrideAnnotations[k8s.ProxyEnableExternalProfilesAnnotation] = strconv.FormatBool(true)
 	}
 
 	if options.proxyCPURequest != "" {

cli/cmd/root.go

@@ -195,6 +195,7 @@ type proxyConfigOptions struct {
 	// ignoreCluster is not validated by validate().
 	ignoreCluster   bool
 	disableIdentity bool
+	disableTap      bool
 }
 
 func (options *proxyConfigOptions) validate() error {

cli/cmd/testdata/inject_emojivoto_deployment_config_overrides.golden.yml

@@ -14,6 +14,7 @@ spec:
     metadata:
       annotations:
         config.linkerd.io/admin-port: "9998"
+        config.linkerd.io/disable-tap: "true"
         config.linkerd.io/proxy-cpu-limit: "1"
         config.linkerd.io/proxy-cpu-request: "0.5"
         config.linkerd.io/proxy-memory-limit: 256Mi
@@ -71,6 +72,8 @@ spec:
               fieldPath: metadata.namespace
         - name: LINKERD2_PROXY_DESTINATION_CONTEXT
           value: ns:$(_pod_ns)
+        - name: LINKERD2_PROXY_TAP_DISABLED
+          value: "true"
         - name: LINKERD2_PROXY_IDENTITY_DIR
           value: /var/run/linkerd/identity/end-entity
         - name: LINKERD2_PROXY_IDENTITY_TRUST_ANCHORS

cli/cmd/testdata/inject_emojivoto_deployment_config_overrides.input.yml

@@ -14,6 +14,7 @@ spec:
     metadata:
       annotations:
         config.linkerd.io/admin-port: "9998"
+        config.linkerd.io/disable-tap: "true"
         config.linkerd.io/proxy-cpu-limit: "1"
         config.linkerd.io/proxy-cpu-request: "0.5"
         config.linkerd.io/proxy-memory-limit: 256Mi

controller/tap/server.go

@@ -61,6 +61,7 @@ func (s *server) TapByResource(req *public.TapByResourceRequest, stream pb.Tap_T
 	}
 
 	pods := []*corev1.Pod{}
+	foundDisabledPods := false
 	for _, object := range objects {
 		podsFor, err := s.k8sAPI.GetPodsFor(object, false)
 		if err != nil {
@@ -69,14 +70,23 @@ func (s *server) TapByResource(req *public.TapByResourceRequest, stream pb.Tap_T
 
 		for _, pod := range podsFor {
 			if pkgK8s.IsMeshed(pod, s.controllerNamespace) {
-				pods = append(pods, pod)
+				if pkgK8s.IsTapDisabled(pod) {
+					foundDisabledPods = true
+				} else {
+					pods = append(pods, pod)
+				}
 			}
 		}
 	}
 
 	if len(pods) == 0 {
-		return status.Errorf(codes.NotFound, "no pods found for %s/%s",
-			req.GetTarget().GetResource().GetType(), req.GetTarget().GetResource().GetName())
+		resType := req.GetTarget().GetResource().GetType()
+		resName := req.GetTarget().GetResource().GetName()
+		if foundDisabledPods {
+			return status.Errorf(codes.NotFound,
+				"all pods found for %s/%s have tapping disabled", resType, resName)
+		}
+		return status.Errorf(codes.NotFound, "no pods found for %s/%s", resType, resName)
 	}
 
 	log.Infof("Tapping %d pods for target: %+v", len(pods), *req.Target.Resource)

controller/tap/server_test.go

@@ -146,6 +146,39 @@ status:
 					},
 				},
 			},
+			{
+				msg: "rpc error: code = NotFound desc = all pods found for pod/emojivoto-meshed-tap-disabled have tapping disabled",
+				k8sRes: []string{`
+apiVersion: v1
+kind: Pod
+metadata:
+  name: emojivoto-meshed-tap-disabled
+  namespace: emojivoto
+  labels:
+    app: emoji-svc
+    linkerd.io/control-plane-ns: controller-ns
+  annotations:
+    config.linkerd.io/disable-tap: "true"
+    linkerd.io/proxy-version: testinjectversion
+status:
+  phase: Running
+`,
+				},
+				req: public.TapByResourceRequest{
+					Target: &public.ResourceSelection{
+						Resource: &public.Resource{
+							Namespace: "emojivoto",
+							Type:      pkgK8s.Pod,
+							Name:      "emojivoto-meshed-tap-disabled",
+						},
+					},
+					Match: &public.TapByResourceRequest_Match{
+						Match: &public.TapByResourceRequest_Match_All{
+							All: &public.TapByResourceRequest_Match_Seq{},
+						},
+					},
+				},
+			},
 			{
 				// indicates we will accept EOF, in addition to the deadline exceeded message
 				eofOk: true,

pkg/inject/inject.go

@@ -63,6 +63,8 @@ const (
 
 	identityAPIPort = 8080
 
+	envTapDisabled = "LINKERD2_PROXY_TAP_DISABLED"
+
 	proxyInitResourceRequestCPU    = "10m"
 	proxyInitResourceRequestMemory = "10Mi"
 	proxyInitResourceLimitCPU      = "100m"
@@ -514,6 +516,15 @@ func (conf *ResourceConfig) injectPodSpec(patch *Patch) {
 		}
 	}
 
+	if conf.tapDisabled() {
+		sidecar.Env = append(sidecar.Env,
+			corev1.EnvVar{
+				Name:  envTapDisabled,
+				Value: "true",
+			},
+		)
+	}
+
 	if saVolumeMount != nil {
 		sidecar.VolumeMounts = []corev1.VolumeMount{*saVolumeMount}
 	}
@@ -774,6 +785,16 @@ func (conf *ResourceConfig) identityContext() *config.IdentityContext {
 	return conf.configs.GetGlobal().GetIdentityContext()
 }
 
+func (conf *ResourceConfig) tapDisabled() bool {
+	if override := conf.getOverride(k8s.ProxyDisableTapAnnotation); override != "" {
+		value, err := strconv.ParseBool(override)
+		if err == nil && value {
+			return true
+		}
+	}
+	return false
+}
+
 func (conf *ResourceConfig) proxyResourceRequirements() corev1.ResourceRequirements {
 	resources := corev1.ResourceRequirements{
 		Requests: corev1.ResourceList{},

pkg/k8s/labels.go

@@ -7,6 +7,7 @@ package k8s
 
 import (
 	"fmt"
+	"strconv"
 
 	"github.com/linkerd/linkerd2/pkg/version"
 	appsv1 "k8s.io/api/apps/v1"
@@ -154,6 +155,9 @@ const (
 	// ProxyDisableIdentityAnnotation can be used to disable identity on the injected proxy.
 	ProxyDisableIdentityAnnotation = ProxyConfigAnnotationsPrefix + "/disable-identity"
 
+	// ProxyDisableTapAnnotation can be used to disable tap on the injected proxy.
+	ProxyDisableTapAnnotation = ProxyConfigAnnotationsPrefix + "/disable-tap"
+
 	// IdentityModeDefault is assigned to IdentityModeAnnotation to
 	// use the control plane's default identity scheme.
 	IdentityModeDefault = "default"
@@ -288,3 +292,15 @@ func GetPodLabels(ownerKind, ownerName string, pod *corev1.Pod) map[string]strin
 func IsMeshed(pod *corev1.Pod, controllerNS string) bool {
 	return pod.Labels[ControllerNSLabel] == controllerNS
 }
+
+// IsTapDisabled returns true if the pod has an annotation for explicitly
+// disabling tap
+func IsTapDisabled(pod *corev1.Pod) bool {
+	if valStr := pod.Annotations[ProxyDisableTapAnnotation]; valStr != "" {
+		valBool, err := strconv.ParseBool(valStr)
+		if err == nil && valBool {
+			return true
+		}
+	}
+	return false
+}

test/inject/inject_test.go

@@ -55,6 +55,7 @@ func TestInjectParams(t *testing.T) {
 		"--manual",
 		"--linkerd-namespace=fake-ns",
 		"--disable-identity",
+		"--disable-tap",
 		"--ignore-cluster",
 		"--proxy-version=proxy-version",
 		"--proxy-image=proxy-image",

test/inject/testdata/injected_params.golden

@@ -14,6 +14,7 @@ spec:
         config.linkerd.io/admin-port: "789"
         config.linkerd.io/control-port: "123"
         config.linkerd.io/disable-identity: "true"
+        config.linkerd.io/disable-tap: "true"
         config.linkerd.io/enable-external-profiles: "true"
         config.linkerd.io/image-pull-policy: Never
         config.linkerd.io/inbound-port: "678"
@@ -75,6 +76,8 @@ spec:
               fieldPath: metadata.namespace
         - name: LINKERD2_PROXY_DESTINATION_CONTEXT
           value: ns:$(_pod_ns)
+        - name: LINKERD2_PROXY_TAP_DISABLED
+          value: "true"
         - name: LINKERD2_PROXY_IDENTITY_DISABLED
           value: disabled
         image: proxy-image:proxy-version

test/tap/tap_test.go

@@ -112,6 +112,23 @@ func TestCliTap(t *testing.T) {
 		}
 	})
 
+	t.Run("tap a disabled deployment", func(t *testing.T) {
+		out, stderr, err := TestHelper.LinkerdRun("tap", "deploy/t4", "--namespace", prefixedNs)
+		if out != "" {
+			t.Fatalf("Unexpected output: %s", out)
+		}
+		if err == nil {
+			t.Fatal("Expected an error, got none")
+		}
+		if stderr == "" {
+			t.Fatal("Expected an error, got none")
+		}
+		expectedErr := "Error: all pods found for deployment/t4 have tapping disabled"
+		if errs := strings.Split(stderr, "\n"); errs[0] != expectedErr {
+			t.Fatalf("Expected [%s], got: %s", expectedErr, errs[0])
+		}
+	})
+
 	t.Run("tap a service call", func(t *testing.T) {
 		events, err := tap("deploy/gateway", "--to", "svc/t2-svc", "--namespace", prefixedNs)
 		if err != nil {

test/tap/testdata/tap_application.yaml

@@ -1,6 +1,7 @@
 # slow_cooker --http-> gateway --grpc-> t1
 #                              --grpc-> t2 always-error
 #                              --http-> t3
+#                              --http-> t4 tap-disabled
 #
 
 ### t1 terminates gRPC requests
@@ -118,7 +119,47 @@ spec:
     port: 8080
     targetPort: 8080
 
-### gateway broadcasts requests to t1, t2, and t3
+# t4 terminates HTTP/1.1 requests, but tap is disabled
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+  name: t4
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: t4
+  template:
+    metadata:
+      annotations:
+        config.linkerd.io/disable-tap: "true"
+      labels:
+        app: t4
+    spec:
+      containers:
+      - name: t4
+        image: buoyantio/bb:v0.0.5
+        args:
+        - terminus
+        - "--h1-server-port=8080"
+        - "--response-text=t4"
+        ports:
+        - containerPort: 8080
+
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: t4-svc
+spec:
+  selector:
+    app: t4
+  ports:
+  - name: http
+    port: 8080
+
+### gateway broadcasts requests to t1, t2, t3 and t4
 ---
 apiVersion: apps/v1beta1
 kind: Deployment
@@ -143,6 +184,7 @@ spec:
         - "--grpc-downstream-server=t1-svc:9090"
         - "--grpc-downstream-server=t2-svc:9090"
         - "--h1-downstream-server=http://t3-svc:8080"
+        - "--h1-downstream-server=http://t4-svc:8080"
         ports:
         - containerPort: 8080
 ---