Tie Activator health to the readiness of the metrics socket.

Fixes: knative#3508
Fixes: knative#2857

Gopkg.toml

@@ -24,8 +24,8 @@ required = [
 
 [[override]]
   name = "github.com/knative/pkg"
-  # HEAD as of 2019-03-23
-  revision = "a5a847fd5cd410d96324766e51f656227dfe72a2"
+  # HEAD as of 2019-03-25
+  revision = "fe25685384ea3e4f267355f5940960d62e2b7b93"
 
 [[override]]
   name = "go.uber.org/zap"

cmd/activator/main.go

@@ -260,6 +260,7 @@ func main() {
 		GetService:    serviceGetter,
 	}
 	ah = activatorhandler.NewRequestEventHandler(reqChan, ah)
+	ah = &activatorhandler.HealthHandler{HealthCheck: statSink.Status, NextHandler: ah}
 	ah = &activatorhandler.ProbeHandler{NextHandler: ah}
 
 	// Watch the logging config map and dynamically update logging levels.

config/activator.yaml

@@ -51,6 +51,16 @@ spec:
           # and seeing k8s logs in addition to ours is not useful.
         - "-logtostderr=false"
         - "-stderrthreshold=FATAL"
+        readinessProbe:
+          httpGet:
+            # The path does not matter, we look for kubelet probe headers.
+            path: /healthz
+            port: 8080
+        livenessProbe:
+          httpGet:
+            # The path does not matter, we look for kubelet probe headers.
+            path: /healthz
+            port: 8080
         resources:
           # Request 2x what we saw running e2e
           requests:

pkg/activator/handler/healthz_handler.go

@@ -0,0 +1,44 @@
+/*
+Copyright 2019 The Knative Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package handler
+
+import (
+	"net/http"
+	"strings"
+)
+
+func isKubeletProbe(r *http.Request) bool {
+	// Since K8s 1.8, prober requests have
+	//   User-Agent = "kube-probe/{major-version}.{minor-version}".
+	return strings.HasPrefix(r.Header.Get("User-Agent"), "kube-probe/")
+}
+
+// HealthHandler handles responding to kubelet probes with a provided health check.
+type HealthHandler struct {
+	HealthCheck func() error
+	NextHandler http.Handler
+}
+
+func (h *HealthHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if isKubeletProbe(r) {
+		if err := h.HealthCheck(); err != nil {
+			http.Error(w, err.Error(), http.StatusInternalServerError)
+		} else {
+			w.WriteHeader(http.StatusOK)
+		}
+		return
+	}
+
+	h.NextHandler.ServeHTTP(w, r)
+}

pkg/activator/handler/healthz_handler_test.go

@@ -0,0 +1,75 @@
+/*
+Copyright 2019 The Knative Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package handler
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestHealthHandler(t *testing.T) {
+	examples := []struct {
+		name           string
+		headers        http.Header
+		passed         bool
+		expectedStatus int
+		check          func() error
+	}{{
+		name:           "forward non-kubelet request",
+		passed:         true,
+		expectedStatus: http.StatusOK,
+	}, {
+		name:           "kubelet probe success",
+		headers:        mapToHeader(map[string]string{"User-Agent": "kube-probe/something"}),
+		passed:         false,
+		expectedStatus: http.StatusOK,
+		check:          func() error { return nil },
+	}, {
+		name:           "kubelet probe failure",
+		headers:        mapToHeader(map[string]string{"User-Agent": "kube-probe/something"}),
+		passed:         false,
+		expectedStatus: http.StatusInternalServerError,
+		check:          func() error { return errors.New("not ready") },
+	}}
+
+	for _, e := range examples {
+		t.Run(e.name, func(t *testing.T) {
+			wasPassed := false
+			baseHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				wasPassed = true
+				w.WriteHeader(http.StatusOK)
+			})
+			handler := HealthHandler{HealthCheck: e.check, NextHandler: baseHandler}
+
+			resp := httptest.NewRecorder()
+			req := httptest.NewRequest("GET", "http://example.com", nil)
+			req.Header = e.headers
+
+			handler.ServeHTTP(resp, req)
+
+			if wasPassed != e.passed {
+				if !e.passed {
+					t.Error("Request got passed to the next handler unexpectedly")
+				} else {
+					t.Error("Request was not passed to the next handler as expected")
+				}
+			}
+
+			if resp.Code != e.expectedStatus {
+				t.Errorf("Unexpected response status. Want %d, got %d", e.expectedStatus, resp.Code)
+			}
+		})
+	}
+}