Add metrics to monitor the kubelet http server

pkg/kubelet/server/BUILD

@@ -23,6 +23,7 @@ go_library(
         "//pkg/kubelet/apis/resourcemetrics/v1alpha1:go_default_library",
         "//pkg/kubelet/container:go_default_library",
         "//pkg/kubelet/prober:go_default_library",
+        "//pkg/kubelet/server/metrics:go_default_library",
         "//pkg/kubelet/server/portforward:go_default_library",
         "//pkg/kubelet/server/remotecommand:go_default_library",
         "//pkg/kubelet/server/stats:go_default_library",
@@ -105,6 +106,7 @@ filegroup(
     name = "all-srcs",
     srcs = [
         ":package-srcs",
+        "//pkg/kubelet/server/metrics:all-srcs",
         "//pkg/kubelet/server/portforward:all-srcs",
         "//pkg/kubelet/server/remotecommand:all-srcs",
         "//pkg/kubelet/server/stats:all-srcs",

pkg/kubelet/server/metrics/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["metrics.go"],
+    importpath = "k8s.io/kubernetes/pkg/kubelet/server/metrics",
+    deps = [
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+)

pkg/kubelet/server/metrics/metrics.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"sync"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	kubeletSubsystem = "kubelet"
+)
+
+var (
+	// HTTPRequests tracks the number of the http requests received since the server started.
+	HTTPRequests = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Subsystem: kubeletSubsystem,
+			Name:      "http_requests_total",
+			Help:      "Number of the http requests received since the server started",
+		},
+		// server_type aims to differentiate the readonly server and the readwrite server.
+		// long_running marks whether the request is long-running or not.
+		// Currently, long-running requests include exec/attach/portforward/debug.
+		[]string{"method", "path", "host", "server_type", "long_running"},
+	)
+	// HTTPRequestsDuration tracks the duration in seconds to serve http requests.
+	HTTPRequestsDuration = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Subsystem: kubeletSubsystem,
+			Name:      "http_requests_duration_seconds",
+			Help:      "Duration in seconds to serve http requests",
+			// Use DefBuckets for now, will customize the buckets if necessary.
+			Buckets: prometheus.DefBuckets,
+		},
+		[]string{"method", "path", "host", "server_type", "long_running"},
+	)
+	// HTTPInflightRequests tracks the number of the inflight http requests.
+	HTTPInflightRequests = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Subsystem: kubeletSubsystem,
+			Name:      "http_inflight_requests",
+			Help:      "Number of the inflight http requests",
+		},
+		[]string{"method", "path", "host", "server_type", "long_running"},
+	)
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	registerMetrics.Do(func() {
+		prometheus.MustRegister(HTTPRequests)
+		prometheus.MustRegister(HTTPRequestsDuration)
+		prometheus.MustRegister(HTTPInflightRequests)
+	})
+}
+
+// SinceInSeconds gets the time since the specified start in seconds.
+func SinceInSeconds(start time.Time) float64 {
+	return time.Since(start).Seconds()
+}

pkg/kubelet/server/server.go

@@ -63,6 +63,7 @@ import (
 	"k8s.io/kubernetes/pkg/kubelet/apis/resourcemetrics/v1alpha1"
 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
 	"k8s.io/kubernetes/pkg/kubelet/prober"
+	servermetrics "k8s.io/kubernetes/pkg/kubelet/server/metrics"
 	"k8s.io/kubernetes/pkg/kubelet/server/portforward"
 	remotecommandserver "k8s.io/kubernetes/pkg/kubelet/server/remotecommand"
 	"k8s.io/kubernetes/pkg/kubelet/server/stats"
@@ -807,6 +808,33 @@ func (s *Server) getPortForward(request *restful.Request, response *restful.Resp
 	proxyStream(response.ResponseWriter, request.Request, url)
 }
 
+// trimURLPath trims a URL path.
+// For paths in the format of "/metrics/xxx", "metrics/xxx" is returned;
+// For all other paths, the first part of the path is returned.
+func trimURLPath(path string) string {
+	parts := strings.SplitN(strings.TrimPrefix(path, "/"), "/", 3)
+	if len(parts) == 0 {
+		return path
+	}
+
+	if parts[0] == "metrics" && len(parts) > 1 {
+		return fmt.Sprintf("%s/%s", parts[0], parts[1])
+
+	}
+	return parts[0]
+}
+
+// isLongRunningRequest determines whether the request is long-running or not.
+func isLongRunningRequest(path string) bool {
+	longRunningRequestPaths := []string{"exec", "attach", "portforward", "debug"}
+	for _, p := range longRunningRequestPaths {
+		if p == path {
+			return true
+		}
+	}
+	return false
+}
+
 // ServeHTTP responds to HTTP requests on the Kubelet.
 func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	defer httplog.NewLogged(req, &w).StacktraceWhen(
@@ -820,6 +848,27 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 			http.StatusSwitchingProtocols,
 		),
 	).Log()
+
+	// monitor http requests
+	var serverType string
+	if s.auth == nil {
+		serverType = "readonly"
+	} else {
+		serverType = "readwrite"
+	}
+
+	method, path, host := req.Method, trimURLPath(req.URL.Path), req.URL.Host
+
+	longRunning := strconv.FormatBool(isLongRunningRequest(path))
+
+	servermetrics.HTTPRequests.WithLabelValues(method, path, host, serverType, longRunning).Inc()
+
+	servermetrics.HTTPInflightRequests.WithLabelValues(method, path, host, serverType, longRunning).Inc()
+	defer servermetrics.HTTPInflightRequests.WithLabelValues(method, path, host, serverType, longRunning).Dec()
+
+	startTime := time.Now()
+	defer servermetrics.HTTPRequestsDuration.WithLabelValues(method, path, host, serverType, longRunning).Observe(servermetrics.SinceInSeconds(startTime))
+
 	s.restfulCont.ServeHTTP(w, req)
 }
 

pkg/kubelet/server/server_test.go

@@ -1666,3 +1666,24 @@ func TestDebuggingDisabledHandlers(t *testing.T) {
 	assert.Equal(t, http.StatusOK, resp.StatusCode)
 
 }
+
+func TestTrimURLPath(t *testing.T) {
+	tests := []struct {
+		path, expected string
+	}{
+		{"", ""},
+		{"//", ""},
+		{"/pods", "pods"},
+		{"pods", "pods"},
+		{"pods/", "pods"},
+		{"good/", "good"},
+		{"pods/probes", "pods"},
+		{"metrics", "metrics"},
+		{"metrics/resource", "metrics/resource"},
+		{"metrics/hello", "metrics/hello"},
+	}
+
+	for _, test := range tests {
+		assert.Equal(t, test.expected, trimURLPath(test.path), fmt.Sprintf("path is: %s", test.path))
+	}
+}