Add metrics-server latency measurement: part 1

Signed-off-by: sanwishe <jiang.mingzhi35@zte.com.cn>

clusterloader2/pkg/measurement/common/metrics_server_prometheus.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package common
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/klog"
+	"k8s.io/perf-tests/clusterloader2/pkg/errors"
+	"k8s.io/perf-tests/clusterloader2/pkg/measurement"
+	measurementutil "k8s.io/perf-tests/clusterloader2/pkg/measurement/util"
+	"k8s.io/perf-tests/clusterloader2/pkg/util"
+)
+
+const (
+	metricsServerPrometheusMeasurementName = "MetricsServerPrometheus"
+
+	metricsServerLatencyQuery = "histogram_quantile(%v, sum(rate(apiserver_request_duration_seconds_bucket{group=metrics.k8s.io,resource=pods,scope=cluster}[%v])) by (le))"
+)
+
+var (
+	desiredMsPercentiles = []float64{0.5, 0.9, 0.99}
+)
+
+func init() {
+	create := func() measurement.Measurement { return CreatePrometheusMeasurement(&metricsServerGatherer{}) }
+	if err := measurement.Register(metricsServerPrometheusMeasurementName, create); err != nil {
+		klog.Fatalf("Cannot register %s: %v", metricsServerPrometheusMeasurementName, err)
+	}
+}
+
+type metricsServerGatherer struct{}
+
+func (g *metricsServerGatherer) Gather(executor QueryExecutor, startTime time.Time, config *measurement.Config) ([]measurement.Summary, error) {
+	latencyMetrics, err := g.gatherLatencyMetrics(executor, startTime)
+	if err != nil {
+		return nil, err
+	}
+	content, err := util.PrettyPrintJSON(latencyMetrics)
+	if err != nil {
+		return nil, err
+	}
+	summaries := []measurement.Summary{measurement.CreateSummary(metricsServerPrometheusMeasurementName, "json", content)}
+	return summaries, nil
+}
+
+func (g *metricsServerGatherer) IsEnabled(config *measurement.Config) bool {
+	return config.CloudProvider.Features().SupportMetricsServerMetrics
+}
+
+func (g *metricsServerGatherer) String() string {
+	return metricsServerPrometheusMeasurementName
+}
+
+func (g *metricsServerGatherer) gatherLatencyMetrics(executor QueryExecutor, startTime time.Time) (*measurementutil.LatencyMetric, error) {
+	endTime := time.Now()
+	measurementDuration := endTime.Sub(startTime)
+	promDuration := measurementutil.ToPrometheusTime(measurementDuration)
+
+	errList := errors.NewErrorList()
+	result := &measurementutil.LatencyMetric{}
+
+	for _, percentile := range desiredMsPercentiles {
+
+		query := fmt.Sprintf(metricsServerLatencyQuery, percentile, promDuration)
+		samples, err := executor.Query(query, endTime)
+		if err != nil {
+			errList.Append(fmt.Errorf("failed to execute query %q, err - %v", query, err))
+			continue
+		}
+
+		if len(samples) != 1 {
+			errList.Append(fmt.Errorf("got unexpected number of samples: %d for query %q", len(samples), query))
+			continue
+		}
+
+		result.SetQuantile(percentile, time.Duration(float64(samples[0].Value)*float64(time.Second)))
+	}
+
+	if !errList.IsEmpty() {
+		return nil, fmt.Errorf("failed to compute latencies, errors - %s", errList.Error())
+	}
+
+	return result, nil
+}

clusterloader2/pkg/measurement/common/probes/manifests/metricsServer/metrics-server-probe-deployment.yaml

@@ -0,0 +1,44 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: probes
+  name: metrics-server-prober
+  labels:
+    probe: metrics-server-probe
+spec:
+  selector:
+    matchLabels:
+      probe: metrics-server-probe
+  replicas: {{.Replicas}}
+  template:
+    metadata:
+      labels:
+        probe: metrics-server-probe
+    spec:
+      containers:
+        - name: prober
+          image: gcr.io/k8s-testimages/probes:v0.0.4
+          args:
+            - --metric-bind-address=0.0.0.0:8080
+            - --mode=kubeclient
+            - --kubeclient-probe-resourceGroup=metrics.k8s.io
+            - --kubeclient-probe-resourceVersion=v1beta1
+            - --kubeclient-probe-resourceName=pods
+            - --kubeclient-probe-interval=30s
+            - --logtostderr=false
+            - --log_file=/var/log/cl2-kube-client.log
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+          ports:
+            - containerPort: 8080
+              name: metrics
+          volumeMounts:
+            - name: logs-volume
+              mountPath: /var/log
+      serviceAccountName: metrics-server-prober
+      volumes:
+        - name: logs-volume
+          hostPath:
+            path: /var/log

clusterloader2/pkg/measurement/common/probes/manifests/metricsServer/metrics-server-probe-rbac.yaml

@@ -0,0 +1,32 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-server-prober
+rules:
+  - apiGroups:
+      - "metrics.k8s.io"
+    resources:
+      - "pods"
+      - "nodes"
+    verbs:
+      - get
+      - list
+---
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-server-prober
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-server-prober
+subjects:
+  - kind: ServiceAccount
+    name: metrics-server-prober
+    namespace: probes
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: metrics-server-prober
+  namespace: probes

clusterloader2/pkg/measurement/common/probes/probes.go

@@ -58,6 +58,14 @@ var (
 		Manifests:        "dnsLookup/*yaml",
 		ProbeLabelValues: []string{"dns"},
 	}
+
+	metricsServerLatencyConfig = proberConfig{
+		Name:             "InClusterAPIServerRequestLatency",
+		MetricVersion:    "v1",
+		Query:            "quantile_over_time(0.99, probes:in_cluster_apiserver_request_latency:histogram_quantile[%v])",
+		Manifests:        "metricsServer/*.yaml",
+		ProbeLabelValues: []string{"kube-client"},
+	}
 )
 
 func init() {
@@ -69,6 +77,10 @@ func init() {
 	if err := measurement.Register(dnsLookupConfig.Name, create); err != nil {
 		klog.Errorf("cannot register %s: %v", dnsLookupConfig.Name, err)
 	}
+	create = func() measurement.Measurement { return createProber(metricsServerLatencyConfig) }
+	if err := measurement.Register(metricsServerLatencyConfig.Name, create); err != nil {
+		klog.Errorf("cannot register %s: %v", metricsServerLatencyConfig.Name, err)
+	}
 }
 
 type proberConfig struct {

clusterloader2/pkg/provider/gce.go

@@ -45,6 +45,7 @@ func NewGCEProvider(_ map[string]string) Provider {
 			SupportGrabMetricsFromKubelets:      true,
 			SupportAccessAPIServerPprofEndpoint: true,
 			SupportKubeStateMetrics:             true,
+			SupportMetricsServerMetrics:         true,
 		},
 	}
 }

clusterloader2/pkg/provider/provider.go

@@ -54,6 +54,8 @@ type Features struct {
 	SupportGrabMetricsFromKubelets bool
 	// SupportKubeStateMetrics determines if running kube-state-metrics is supported.
 	SupportKubeStateMetrics bool
+	// SupportMetricsServerMetrics determines if running metrics server is supported.
+	SupportMetricsServerMetrics bool
 
 	// ShouldPrometheusScrapeApiserverOnly determines if we should set PROMETHEUS_SCRAPE_APISERVER_ONLY by default.
 	ShouldPrometheusScrapeApiserverOnly bool

util-images/probes/Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.12.1 AS build-env
+FROM golang:1.13 AS build-env
 
 ARG gopkg=k8s.io/perf-tests/util-images/probes
 
@@ -8,7 +8,7 @@ ENV GO111MODULE on
 WORKDIR /go/src/$gopkg
 RUN CGO_ENABLED=0 go build -o /workspace/probes ./cmd
 
-FROM golang:1.12.1-alpine
+FROM golang:1.13-alpine
 WORKDIR /workspace
 COPY --from=build-env /workspace/probes .
 ENTRYPOINT ["/workspace/probes"]

util-images/probes/cmd/main.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"flag"
+	"k8s.io/perf-tests/util-images/probes/pkg/kubeclient"
 	"net/http"
 	_ "net/http/pprof"
 
@@ -30,7 +31,7 @@ import (
 
 var (
 	metricAddress = flag.String("metric-bind-address", "0.0.0.0:8080", "The address to serve the Prometheus metrics on.")
-	mode          = flag.String("mode", "", "Mode that should be run. Supported values: ping-server, ping-client, dns")
+	mode          = flag.String("mode", "", "Mode that should be run. Supported values: ping-server, ping-client, dns, kube-client")
 )
 
 func main() {
@@ -50,6 +51,8 @@ func main() {
 		pingserver.Run(pingserver.NewDefaultConfig())
 	case "dns":
 		dns.Run()
+	case "kubeclient":
+		kubeclient.Run(kubeclient.NewDefaultConfig())
 	default:
 		klog.Fatalf("Unrecognized mode: %q", *mode)
 	}

util-images/probes/go.mod

@@ -1,8 +1,9 @@
 module k8s.io/perf-tests/util-images/probes
 
-go 1.12
+go 1.13
 
 require (
 	github.com/prometheus/client_golang v0.9.2
-	k8s.io/klog v0.2.0
+	k8s.io/client-go v0.18.0
+	k8s.io/klog v1.0.0
 )