Automated cherry pick of #70753, #70676 and #70971 upstream release 1.10

cluster/gce/manifests/kube-apiserver.manifest

@@ -34,11 +34,19 @@
       "httpGet": {
         "host": "127.0.0.1",
         "port": 8080,
-        "path": "/healthz"
+        "path": "/healthz?exclude=etcd"
       },
       "initialDelaySeconds": {{liveness_probe_initial_delay}},
       "timeoutSeconds": 15
     },
+    "readinessProbe": {
+      "httpGet": {
+        "host": "127.0.0.1",
+        "port": 8080,
+        "path": "/healthz"
+      },
+      "timeoutSeconds": 15
+    },
     "ports":[
       { "name": "https",
         "containerPort": {{secure_port}},

cmd/cloud-controller-manager/app/BUILD

@@ -21,6 +21,7 @@ go_library(
         "//vendor/github.com/spf13/cobra:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//vendor/k8s.io/client-go/informers:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes:go_default_library",
         "//vendor/k8s.io/client-go/rest:go_default_library",

cmd/cloud-controller-manager/app/controllermanager.go

@@ -29,6 +29,7 @@ import (
 
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	restclient "k8s.io/client-go/rest"
@@ -115,15 +116,23 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error {
 		glog.Errorf("unable to register configz: %c", err)
 	}
 
+	// Setup any healthz checks we will want to use.
+	var checks []healthz.HealthzChecker
+	var electionChecker *leaderelection.HealthzAdaptor
+	if c.Generic.ComponentConfig.LeaderElection.LeaderElect {
+		electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20)
+		checks = append(checks, electionChecker)
+	}
+
 	// Start the controller manager HTTP server
 	stopCh := make(chan struct{})
 	if c.Generic.SecureServing != nil {
-		if err := genericcontrollermanager.Serve(&c.Generic, c.Generic.SecureServing.Serve, stopCh); err != nil {
+		if err := genericcontrollermanager.Serve(&c.Generic, c.Generic.SecureServing.Serve, stopCh, checks...); err != nil {
 			return err
 		}
 	}
 	if c.Generic.InsecureServing != nil {
-		if err := genericcontrollermanager.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh); err != nil {
+		if err := genericcontrollermanager.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh, checks...); err != nil {
 			return err
 		}
 	}
@@ -187,6 +196,8 @@ func Run(c *cloudcontrollerconfig.CompletedConfig) error {
 				glog.Fatalf("leaderelection lost")
 			},
 		},
+		WatchDog: electionChecker,
+		Name:     "cloud-controller-manager",
 	})
 	panic("unreachable")
 }

cmd/controller-manager/app/serve.go

@@ -17,12 +17,11 @@ limitations under the License.
 package app
 
 import (
+	"github.com/prometheus/client_golang/prometheus"
 	"net/http"
 	goruntime "runtime"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
-
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 	genericfilters "k8s.io/apiserver/pkg/server/filters"
@@ -37,9 +36,9 @@ type serveFunc func(handler http.Handler, shutdownTimeout time.Duration, stopCh
 
 // Serve creates a base handler chain for a controller manager. It runs the
 // the chain with the given serveFunc.
-func Serve(c *CompletedConfig, serveFunc serveFunc, stopCh <-chan struct{}) error {
+func Serve(c *CompletedConfig, serveFunc serveFunc, stopCh <-chan struct{}, checks ...healthz.HealthzChecker) error {
 	mux := mux.NewPathRecorderMux("controller-manager")
-	healthz.InstallHandler(mux)
+	healthz.InstallHandler(mux, checks...)
 	if c.ComponentConfig.EnableProfiling {
 		routes.Profiling{}.Install(mux)
 		if c.ComponentConfig.EnableContentionProfiling {

cmd/kube-controller-manager/app/BUILD

@@ -116,6 +116,7 @@ go_library(
         "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/uuid:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
+        "//vendor/k8s.io/apiserver/pkg/server/healthz:go_default_library",
         "//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
         "//vendor/k8s.io/client-go/discovery:go_default_library",
         "//vendor/k8s.io/client-go/discovery/cached:go_default_library",

cmd/kube-controller-manager/app/controllermanager.go

@@ -35,6 +35,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/uuid"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/client-go/informers"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/leaderelection"
@@ -121,16 +122,23 @@ func Run(c *config.CompletedConfig) error {
 		glog.Errorf("unable to register configz: %c", err)
 	}
 
+	// Setup any healthz checks we will want to use.
+	var checks []healthz.HealthzChecker
+	var electionChecker *leaderelection.HealthzAdaptor
+	if c.Generic.ComponentConfig.LeaderElection.LeaderElect {
+		electionChecker = leaderelection.NewLeaderHealthzAdaptor(time.Second * 20)
+		checks = append(checks, electionChecker)
+	}
+
 	// Start the controller manager HTTP server
 	stopCh := make(chan struct{})
 	if c.Generic.SecureServing != nil {
-		if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.SecureServing.Serve, stopCh); err != nil {
+		if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.SecureServing.Serve, stopCh, checks...); err != nil {
 			return err
 		}
 	}
 	if c.Generic.InsecureServing != nil {
-		if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh); err != nil {
-			return err
+		if err := genericcontrollerconfig.Serve(&c.Generic, c.Generic.InsecureServing.Serve, stopCh, checks...); err != nil {
 		}
 	}
 
@@ -205,6 +213,8 @@ func Run(c *config.CompletedConfig) error {
 				glog.Fatalf("leaderelection lost")
 			},
 		},
+		WatchDog: electionChecker,
+		Name:     "kube-controller-manager",
 	})
 	panic("unreachable")
 }

staging/src/k8s.io/apiserver/pkg/server/healthz/BUILD

@@ -10,6 +10,9 @@ go_test(
     name = "go_default_test",
     srcs = ["healthz_test.go"],
     embed = [":go_default_library"],
+    deps = [
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
 )
 
 go_library(
@@ -19,7 +22,10 @@ go_library(
         "healthz.go",
     ],
     importpath = "k8s.io/apiserver/pkg/server/healthz",
-    deps = ["//vendor/github.com/golang/glog:go_default_library"],
+    deps = [
+        "//vendor/github.com/golang/glog:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+    ],
 )
 
 filegroup(

staging/src/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -24,6 +24,8 @@ import (
 	"sync"
 
 	"github.com/golang/glog"
+
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 // HealthzChecker is a named healthz checker.
@@ -71,7 +73,7 @@ func InstallHandler(mux mux, checks ...HealthzChecker) {
 		checks = []HealthzChecker{PingHealthz}
 	}
 
-	glog.V(5).Info("Installing healthz checkers:", strings.Join(checkerNames(checks...), ", "))
+	glog.V(5).Info("Installing healthz checkers:", formatQuoted(checkerNames(checks...)...))
 
 	mux.Handle("/healthz", handleRootHealthz(checks...))
 	for _, check := range checks {
@@ -100,12 +102,28 @@ func (c *healthzCheck) Check(r *http.Request) error {
 	return c.check(r)
 }
 
+// getExcludedChecks extracts the health check names to be excluded from the query param
+func getExcludedChecks(r *http.Request) sets.String {
+	checks, found := r.URL.Query()["exclude"]
+	if found {
+		return sets.NewString(checks...)
+	}
+	return sets.NewString()
+}
+
 // handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
 func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		failed := false
+		excluded := getExcludedChecks(r)
 		var verboseOut bytes.Buffer
 		for _, check := range checks {
+			// no-op the check if we've specified we want to exclude the check
+			if excluded.Has(check.Name()) {
+				excluded.Delete(check.Name())
+				fmt.Fprintf(&verboseOut, "[+]%v excluded: ok\n", check.Name())
+				continue
+			}
 			if err := check.Check(r); err != nil {
 				// don't include the error since this endpoint is public.  If someone wants more detail
 				// they should have explicit permission to the detailed checks.
@@ -116,6 +134,11 @@ func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc {
 				fmt.Fprintf(&verboseOut, "[+]%v ok\n", check.Name())
 			}
 		}
+		if excluded.Len() > 0 {
+			fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %v\n", formatQuoted(excluded.List()...))
+			glog.Warningf("cannot exclude some health checks, no health checks are installed matching %v",
+				formatQuoted(excluded.List()...))
+		}
 		// always be verbose on failure
 		if failed {
 			http.Error(w, fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)
@@ -146,14 +169,20 @@ func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
 
 // checkerNames returns the names of the checks in the same order as passed in.
 func checkerNames(checks ...HealthzChecker) []string {
-	if len(checks) > 0 {
-		// accumulate the names of checks for printing them out.
-		checkerNames := make([]string, 0, len(checks))
-		for _, check := range checks {
-			// quote the Name so we can disambiguate
-			checkerNames = append(checkerNames, fmt.Sprintf("%q", check.Name()))
-		}
-		return checkerNames
+	// accumulate the names of checks for printing them out.
+	checkerNames := make([]string, 0, len(checks))
+	for _, check := range checks {
+		checkerNames = append(checkerNames, check.Name())
 	}
-	return nil
+	return checkerNames
+}
+
+// formatQuoted returns a formatted string of the health check names,
+// preserving the order passed in.
+func formatQuoted(names ...string) string {
+	quoted := make([]string, 0, len(names))
+	for _, name := range names {
+		quoted = append(quoted, fmt.Sprintf("%q", name))
+	}
+	return strings.Join(quoted, ",")
 }

staging/src/k8s.io/apiserver/pkg/server/healthz/healthz_test.go

@@ -21,8 +21,11 @@ import (
 	"fmt"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"reflect"
 	"testing"
+
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 func TestInstallHandler(t *testing.T) {
@@ -50,6 +53,10 @@ func TestMulitipleChecks(t *testing.T) {
 		addBadCheck      bool
 	}{
 		{"/healthz?verbose", "[+]ping ok\nhealthz check passed\n", http.StatusOK, false},
+		{"/healthz?exclude=dontexist", "ok", http.StatusOK, false},
+		{"/healthz?exclude=bad", "ok", http.StatusOK, true},
+		{"/healthz?verbose=true&exclude=bad", "[+]ping ok\n[+]bad excluded: ok\nhealthz check passed\n", http.StatusOK, true},
+		{"/healthz?verbose=true&exclude=dontexist", "[+]ping ok\nwarn: some health checks cannot be excluded: no matches for \"dontexist\"\nhealthz check passed\n", http.StatusOK, false},
 		{"/healthz/ping", "ok", http.StatusOK, false},
 		{"/healthz", "ok", http.StatusOK, false},
 		{"/healthz?verbose", "[+]ping ok\n[-]bad failed: reason withheld\nhealthz check failed\n", http.StatusInternalServerError, true},
@@ -103,10 +110,72 @@ func TestCheckerNames(t *testing.T) {
 	for _, tc := range testCases {
 		result := checkerNames(tc.have...)
 		t.Run(tc.desc, func(t *testing.T) {
-			if reflect.DeepEqual(tc.want, result) {
+			if !reflect.DeepEqual(tc.want, result) {
 				t.Errorf("want %#v, got %#v", tc.want, result)
 			}
 		})
 	}
+}
+
+func TestFormatQuoted(t *testing.T) {
+	n1 := "n1"
+	n2 := "n2"
+	testCases := []struct {
+		desc     string
+		names    []string
+		expected string
+	}{
+		{"empty", []string{}, ""},
+		{"single name", []string{n1}, "\"n1\""},
+		{"two names", []string{n1, n2}, "\"n1\",\"n2\""},
+		{"two names, reverse order", []string{n2, n1}, "\"n2\",\"n1\""},
+	}
+	for _, tc := range testCases {
+		result := formatQuoted(tc.names...)
+		t.Run(tc.desc, func(t *testing.T) {
+			if result != tc.expected {
+				t.Errorf("expected %#v, got %#v", tc.expected, result)
+			}
+		})
+	}
+}
+
+func TestGetExcludedChecks(t *testing.T) {
+	type args struct {
+		r *http.Request
+	}
+	tests := []struct {
+		name string
+		r    *http.Request
+		want sets.String
+	}{
+		{"Should have no excluded health checks",
+			createGetRequestWithUrl("/healthz?verbose=true"),
+			sets.NewString(),
+		},
+		{"Should extract out the ping health check",
+			createGetRequestWithUrl("/healthz?exclude=ping"),
+			sets.NewString("ping"),
+		},
+		{"Should extract out ping and log health check",
+			createGetRequestWithUrl("/healthz?exclude=ping&exclude=log"),
+			sets.NewString("ping", "log"),
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := getExcludedChecks(tt.r); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("getExcludedChecks() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}
 
+func createGetRequestWithUrl(rawUrlString string) *http.Request {
+	url, _ := url.Parse(rawUrlString)
+	return &http.Request{
+		Method: http.MethodGet,
+		Proto:  "HTTP/1.1",
+		URL:    url,
+	}
 }

staging/src/k8s.io/client-go/tools/leaderelection/BUILD

@@ -8,12 +8,16 @@ load(
 
 go_library(
     name = "go_default_library",
-    srcs = ["leaderelection.go"],
+    srcs = [
+        "healthzadaptor.go",
+        "leaderelection.go",
+    ],
     importpath = "k8s.io/client-go/tools/leaderelection",
     deps = [
         "//vendor/github.com/golang/glog:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/runtime:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
         "//vendor/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",
@@ -22,13 +26,17 @@ go_library(
 
 go_test(
     name = "go_default_test",
-    srcs = ["leaderelection_test.go"],
+    srcs = [
+        "healthzadaptor_test.go",
+        "leaderelection_test.go",
+    ],
     embed = [":go_default_library"],
     deps = [
         "//vendor/k8s.io/api/core/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
         "//vendor/k8s.io/apimachinery/pkg/runtime:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/clock:go_default_library",
         "//vendor/k8s.io/client-go/kubernetes/typed/core/v1/fake:go_default_library",
         "//vendor/k8s.io/client-go/testing:go_default_library",
         "//vendor/k8s.io/client-go/tools/leaderelection/resourcelock:go_default_library",