Run DNS probes in tests.

kubernetes · Aug 5, 2019 · f985e26 · f985e26
1 parent 5f1e17c
commit f985e26
Show file tree

Hide file tree

Showing 8 changed files with 119 additions and 10 deletions.
diff --git a/clusterloader2/pkg/measurement/common/probes/manifests/dns-prober-deployment.yaml b/clusterloader2/pkg/measurement/common/probes/manifests/dns-prober-deployment.yaml
@@ -0,0 +1,36 @@
+{{$ENABLE_DNS_PROBES := DefaultParam .ENABLE_DNS_PROBES false}}
+
+{{if $ENABLE_DNS_PROBES}}
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  namespace: probes
+  name: dns
+  labels:
+    probe: dns
+spec:
+  selector:
+    matchLabels:
+      probe: dns
+  replicas: {{.Replicas}}
+  template:
+    metadata:
+      labels:
+        probe: dns
+    spec:
+      containers:
+        - name: dns
+          image: gcr.io/k8s-testimages/probes:v0.0.2
+          args:
+            - --metric-bind-address=0.0.0.0:8080
+            - --mode=dns
+            # Instead of creating dedicated "null-service" use one that's already exists
+            - --dns-probe-url=ping-server.probes
+          resources:
+            limits:
+              cpu: 100m
+              memory: 100Mi
+          ports:
+            - containerPort: 8080
+              name: metrics
+{{end}}
diff --git a/clusterloader2/pkg/measurement/common/probes/manifests/dns-prober-service.yaml b/clusterloader2/pkg/measurement/common/probes/manifests/dns-prober-service.yaml
@@ -0,0 +1,17 @@
+{{$ENABLE_DNS_PROBES := DefaultParam .ENABLE_DNS_PROBES false}}
+
+{{if $ENABLE_DNS_PROBES}}
+apiVersion: v1
+kind: Service
+metadata:
+  namespace: probes
+  name: dns
+  labels:
+    probe: dns
+spec:
+  ports:
+    - name: metrics
+      port: 8080
+  selector:
+    probe: dns
+{{end}}
diff --git a/clusterloader2/pkg/measurement/common/probes/manifests/dns-prober-serviceMonitor.yaml b/clusterloader2/pkg/measurement/common/probes/manifests/dns-prober-serviceMonitor.yaml
@@ -0,0 +1,19 @@
+{{$ENABLE_DNS_PROBES := DefaultParam .ENABLE_DNS_PROBES false}}
+
+{{if $ENABLE_DNS_PROBES}}
+apiVersion: monitoring.coreos.com/v1
+kind: ServiceMonitor
+metadata:
+  namespace: probes
+  name: dns
+spec:
+  endpoints:
+    - interval: 30s
+      port: metrics
+  namespaceSelector:
+    matchNames:
+      - probes
+  selector:
+    matchLabels:
+      probe: dns
+{{end}}
diff --git a/clusterloader2/pkg/measurement/common/probes/probes.go b/clusterloader2/pkg/measurement/common/probes/probes.go
@@ -17,14 +17,15 @@ limitations under the License.
 package probes
 
 import (
+	"errors"
 	"fmt"
 	"time"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog"
-	"k8s.io/perf-tests/clusterloader2/pkg/errors"
+	cl2errors "k8s.io/perf-tests/clusterloader2/pkg/errors"
 	"k8s.io/perf-tests/clusterloader2/pkg/framework"
 	"k8s.io/perf-tests/clusterloader2/pkg/framework/client"
 	"k8s.io/perf-tests/clusterloader2/pkg/measurement"
@@ -43,6 +44,9 @@ const (
 	checkProbesReadyTimeout  = 5 * time.Minute
 
 	currentProbesMetricsVersion = "v1"
+
+	networkLatencyQuery = "quantile_over_time(0.99, probes:in_cluster_network_latency:histogram_quantile[%v])"
+	dnsLookupQuery      = "quantile_over_time(0.99, probes:dns_lookup_latency:histogram_quantile[%v])"
 )
 
 var (
@@ -57,19 +61,20 @@ func init() {
 
 func createProbesMeasurement() measurement.Measurement {
 	return &probesMeasurement{
-		probeNameToPrometheusQueryTmpl: map[string]string{
-			"in_cluster_network_latency": "quantile_over_time(0.99, probes:in_cluster_network_latency:histogram_quantile[%v])",
+		queryTmpl: map[string]string{
+			"in_cluster_network_latency": networkLatencyQuery,
+			"dns_lookup_latency":         dnsLookupQuery,
 		},
 	}
 }
 
 type probesMeasurement struct {
-	// probeNameToPrometheusQueryTmpl defines a config of this measurement. Updating the config in the
+	// queryTmpl defines a config of this measurement. Updating the config in the
 	// createProbesMeasurement method is the only place in go code that needs to be changed while
 	// adding a new probe.
 	// Each query template should accept a single %v placeholder corresponding to the query window
 	// length. See the 'in_cluster_network_latency' as an example.
-	probeNameToPrometheusQueryTmpl map[string]string
+	queryTmpl map[string]string
 
 	framework        *framework.Framework
 	replicasPerProbe int
@@ -94,7 +99,7 @@ func (p *probesMeasurement) Execute(config *measurement.MeasurementConfig) ([]me
 		return nil, p.start(config)
 	case "gather":
 		summaries, err := p.gather(config.Params)
-		if err != nil && !errors.IsMetricViolationError(err) {
+		if err != nil && !cl2errors.IsMetricViolationError(err) {
 			return nil, err
 		}
 		return summaries, err
@@ -131,7 +136,8 @@ func (p *probesMeasurement) initialize(config *measurement.MeasurementConfig) er
 	}
 	p.framework = config.ClusterFramework
 	p.replicasPerProbe = replicasPerProbe
-	p.templateMapping = map[string]interface{}{"Replicas": replicasPerProbe}
+	p.templateMapping = config.Overrides
+	p.templateMapping["Replicas"] = replicasPerProbe
 	return nil
 }
 
@@ -169,13 +175,16 @@ func (p *probesMeasurement) gather(params map[string]interface{}) ([]measurement
 	measurementEnd := time.Now()
 	var probeSummaries []measurement.Summary
 	var violationErrors []error
-	for probeName, queryTmpl := range p.probeNameToPrometheusQueryTmpl {
+	for probeName, queryTmpl := range p.queryTmpl {
 		query := prepareQuery(queryTmpl, p.startTime, measurementEnd)
 		executor := measurementutil.NewQueryExecutor(p.framework.GetClientSets().GetClient())
 		samples, err := executor.Query(query, measurementEnd)
 		if err != nil {
 			return nil, err
 		}
+		if len(samples) == 0 {
+			return nil, errors.New("query returned 0 samples")
+		}
 
 		latencyMetric, err := measurementutil.NewLatencyMetricPrometheus(samples)
 		if err != nil {
@@ -185,7 +194,7 @@ func (p *probesMeasurement) gather(params map[string]interface{}) ([]measurement
 		if threshold, ok := thresholds[probeName]; ok {
 			suffix = fmt.Sprintf(", expected perc99 <= %v", threshold.Perc99)
 			if err = latencyMetric.VerifyThreshold(threshold); err != nil {
-				violationErrors = append(violationErrors, errors.NewMetricViolationError(probeName, err.Error()))
+				violationErrors = append(violationErrors, cl2errors.NewMetricViolationError(probeName, err.Error()))
 				prefix = " WARNING"
 			}
 		}
@@ -198,7 +207,7 @@ func (p *probesMeasurement) gather(params map[string]interface{}) ([]measurement
 		probeSummaries = append(probeSummaries, probeSummary)
 	}
 	if len(violationErrors) > 0 {
-		err = errors.NewMetricViolationError("probers", fmt.Sprintf("there should not be errors from probers, got %v", violationErrors))
+		err = cl2errors.NewMetricViolationError("probers", fmt.Sprintf("there should not be errors from probers, got %v", violationErrors))
 	}
 	return probeSummaries, err
 }

diff --git a/clusterloader2/pkg/measurement/interface.go b/clusterloader2/pkg/measurement/interface.go
@@ -37,6 +37,8 @@ type MeasurementConfig struct {
 	// Identifier identifies this instance of measurement.
 	Identifier    string
 	CloudProvider string
+	// Overrides contains parameters passed as overrides
+	Overrides map[string]interface{}
 }
 
 // Measurement is an common interface for all measurements methods. It should be implemented by the user to

diff --git a/clusterloader2/pkg/measurement/manager.go b/clusterloader2/pkg/measurement/manager.go
@@ -55,13 +55,19 @@ func (mm *MeasurementManager) Execute(methodName string, identifier string, para
 	if err != nil {
 		return err
 	}
+	overrides, verr := config.GetMapping(mm.clusterLoaderConfig)
+	if verr != nil {
+		return verr
+	}
+
 	config := &MeasurementConfig{
 		ClusterFramework:    mm.clusterFramework,
 		PrometheusFramework: mm.prometheusFramework,
 		Params:              params,
 		TemplateProvider:    mm.templateProvider,
 		Identifier:          identifier,
 		CloudProvider:       mm.clusterLoaderConfig.ClusterConfig.Provider,
+		Overrides:           overrides,
 	}
 	summaries, err := measurementInstance.Execute(config)
 	mm.summaries = append(mm.summaries, summaries...)

diff --git a/clusterloader2/pkg/prometheus/manifests/prometheus-rules.yaml b/clusterloader2/pkg/prometheus/manifests/prometheus-rules.yaml
@@ -1,3 +1,5 @@
+{{$ENABLE_DNS_PROBES := DefaultParam .ENABLE_DNS_PROBES false}}
+
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:
@@ -42,6 +44,23 @@ spec:
       record: probes:in_cluster_network_latency:histogram_quantile
       labels:
         quantile: "0.50"
+    {{if $ENABLE_DNS_PROBES}}
+    - expr: |
+        histogram_quantile(0.99, sum(rate(probes_in_cluster_dns_latency_seconds_bucket[5m])) by (le))
+      record: probes:dns_lookup_latency:histogram_quantile
+      labels:
+        quantile: "0.99"
+    - expr: |
+        histogram_quantile(0.90, sum(rate(probes_in_cluster_dns_latency_seconds_bucket[5m])) by (le))
+      record: probes:dns_lookup_latency:histogram_quantile
+      labels:
+        quantile: "0.90"
+    - expr: |
+        histogram_quantile(0.50, sum(rate(probes_in_cluster_dns_latency_seconds_bucket[5m])) by (le))
+      record: probes:dns_lookup_latency:histogram_quantile
+      labels:
+        quantile: "0.50"
+    {{end}}
   - name: kube-proxy.rules
     rules:
     - expr: |

diff --git a/clusterloader2/testing/experiments/enable_dns_probes.yaml b/clusterloader2/testing/experiments/enable_dns_probes.yaml
@@ -1 +1,2 @@
 ENABLE_DNS_PROBES: true
+ENABLE_PROBES: true