Probes - job code & docker image.

Probes are simple jobs running inside kubernetes cluster measuring SLIs that could not be measured through other means.

Ref. kubernetes/kubernetes#76012

probes/Dockerfile

@@ -0,0 +1,17 @@
+FROM golang:1.12.1 AS build-env
+
+ARG gopkg=k8s.io/perf-tests/probes
+
+ADD ["cmd", "/go/src/$gopkg/cmd"]
+ADD ["pkg", "/go/src/$gopkg/pkg"]
+ADD ["go.mod", "/go/src/$gopkg"]
+ADD ["go.sum", "/go/src/$gopkg"]
+
+ENV GO111MODULE on
+WORKDIR /go/src/$gopkg
+RUN CGO_ENABLED=0 go build -o /workspace/probes ./cmd
+
+FROM golang:1.12.1-alpine
+WORKDIR /workspace
+COPY --from=build-env /workspace/probes .
+ENTRYPOINT ["/workspace/probes"]

probes/Makefile

@@ -0,0 +1,17 @@
+PROJECT = k8s-testimages
+IMG = gcr.io/$(PROJECT)/probes
+TAG = v0.0.1
+
+all: push
+
+.PHONY: build
+build: cmd
+	docker build --pull -t $(IMG):$(TAG) .
+	docker tag $(IMG):$(TAG) $(IMG):latest
+	@echo Built $(IMG):$(TAG) and tagged with latest
+
+.PHONY: push
+push: build
+	docker push $(IMG):$(TAG)
+	docker push $(IMG):latest
+	@echo Pushed $(IMG) with :latest and :$(TAG) tags

probes/OWNERS

@@ -0,0 +1,7 @@
+approvers:
+- mm4tt
+- wojtek-t
+reviewers:
+- mm4tt
+- wojtek-t
+

probes/README.md

@@ -0,0 +1,41 @@
+# Probes
+
+Probes are simple jobs running inside kubernetes cluster measuring SLIs that could not be measured through other means.
+## Probe Types
+
+### Ping Client
+
+This probe exports the `probes_in_cluster_network_latency_seconds` metric.
+It implements the [Network Latency Scalability SLI]
+
+#### Running locally
+
+```
+go run cmd/main.go --mode=ping-client --metric-bind-address=:8071 --ping-server-address=127.0.0.1:8081 --stderrthreshold=INFO
+```
+
+### Ping Server
+
+This probe doesn't export any metrics, it's needed for the **Ping Client** to work. 
+
+#### Running locally
+```
+go run cmd/main.go --mode=ping-server --metric-bind-address=:8070 --ping-server-bind-address=0.0.0.0:8081 --stderrthreshold=INFO
+```
+
+
+## Building and Releasing
+
+1. Increment the `TAG` in the Makefile.
+2. `make build`
+3. Test changes with `docker run gcr.io/k8s-testimages/probes:latest -- ...`
+4. Release with `make push`
+
+
+## Go Modules
+
+This project uses [Go Modules] to manage the external dependencies. 
+
+
+[Go Modules]: https://github.com/golang/go/wiki/Modules
+[Network Latency Scalability SLI]: https://github.com/kubernetes/community/blob/master/sig-scalability/slos/network_latency.md

probes/cmd/main.go

@@ -0,0 +1,65 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	"k8s.io/klog"
+	"k8s.io/perf-tests/probes/pkg/ping/client"
+	"k8s.io/perf-tests/probes/pkg/ping/server"
+)
+
+var (
+	metricAddress = flag.String("metric-bind-address", "0.0.0.0:8080", "The address to serve the Prometheus metrics on.")
+	mode          = flag.String("mode", "", "Mode that should be run. Supported values: ping-server, ping-client")
+)
+
+func main() {
+	klog.InitFlags(flag.CommandLine)
+	flag.Parse()
+	verifyFlags()
+
+	klog.Infof("I'm probes.")
+	klog.Infof("Mode is: %s\n", *mode)
+
+	go func() {
+		http.Handle("/metrics", promhttp.Handler())
+		klog.Infof("Serving metrics on %s\n", *metricAddress)
+		klog.Fatal(http.ListenAndServe(*metricAddress, nil))
+	}()
+
+	switch *mode {
+	case "ping-client":
+		pingclient.Run(pingclient.NewDefaultPingClientConfig())
+	case "ping-server":
+		pingserver.Run(pingserver.NewDefaultPingServerConfig())
+	default:
+		klog.Fatalf("Unrecognized mode: '%s'", *mode)
+	}
+}
+
+func verifyFlags() {
+	if *metricAddress == "" {
+		klog.Fatal("--metric-bind-address not set!")
+	}
+	if *mode == "" {
+		klog.Fatal("--mode not set!")
+	}
+}

probes/go.mod

@@ -0,0 +1,8 @@
+module k8s.io/perf-tests/probes
+
+go 1.12
+
+require (
+	github.com/prometheus/client_golang v0.9.2
+	k8s.io/klog v0.2.0
+)

probes/go.sum

@@ -0,0 +1,19 @@
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
+github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
+github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
+github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+k8s.io/klog v0.2.0 h1:0ElL0OHzF3N+OhoJTL0uca20SxtYt4X4+bzHeqrB83c=
+k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=

probes/pkg/metrics/metrics.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "probes"
+)
+
+var (
+	// InClusterNetworkLatency implements the In-Cluster Network Programming SLI, see
+	// https://github.com/kubernetes/community/blob/master/sig-scalability/slos/network_latency.md
+	InClusterNetworkLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
+		Namespace: namespace,
+		Name:      "in_cluster_network_latency_seconds",
+		Buckets:   prometheus.ExponentialBuckets(0.000001, 2, 26), // from 1us up to ~1min
+		Help:      "Histogram of the time (in seconds) it took to ping a ping-server instance.",
+	})
+)
+
+func init() {
+	prometheus.MustRegister(InClusterNetworkLatency)
+}

probes/pkg/ping/client/client.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pingclient
+
+import (
+	"flag"
+	"net/http"
+	"time"
+
+	"k8s.io/klog"
+	"k8s.io/perf-tests/probes/pkg/metrics"
+)
+
+var (
+	pingServerAddress = flag.String("ping-server-address", "", "The address of the ping server")
+	pingSleepDuration = flag.Duration("ping-sleep-duration", 1*time.Second, "Duration of the sleep between pings")
+)
+
+// PingClientConfig configures the "ping-client" probe.
+type PingClientConfig struct {
+	pingServerAddress string
+	pingSleepDuration time.Duration
+}
+
+// NewDefaultPingClientConfig creates a default "ping-client" config.
+func NewDefaultPingClientConfig() *PingClientConfig {
+	if *pingServerAddress == "" {
+		klog.Fatal("--ping-server-address not set!")
+	}
+	return &PingClientConfig{
+		pingServerAddress: *pingServerAddress,
+		pingSleepDuration: *pingSleepDuration,
+	}
+}
+
+// Run runs the ping client probe that periodically pings the ping server and exports latency metric.
+func Run(config *PingClientConfig) {
+	for {
+		time.Sleep(config.pingSleepDuration)
+		klog.Infof("ping -> %s...\n", config.pingServerAddress)
+		start := time.Now()
+		if err := ping(config.pingServerAddress); err != nil {
+			klog.Infof("Got error: %v", err)
+			// TODO(mm4tt): Increment server not available gauge metric.
+			continue
+		}
+		end := time.Now()
+		latency := end.Sub(start)
+		klog.Infof("Request took: %v\n", latency)
+		metrics.InClusterNetworkLatency.Observe(latency.Seconds())
+	}
+}
+
+func ping(serverAddress string) error {
+	_, err := http.Get("http://" + serverAddress)
+	return err
+}

probes/pkg/ping/server/server.go

@@ -0,0 +1,55 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pingserver
+
+import (
+	"flag"
+	"net/http"
+
+	"k8s.io/klog"
+)
+
+var (
+	pingServerBindAddress = flag.String("ping-server-bind-address", "", "The address to bind for the ping server")
+)
+
+// PingServerConfig configures the "ping-server" probe.
+type PingServerConfig struct {
+	pingServerBindAddress string
+}
+
+// NewDefaultPingServerConfig creates a default "ping-server" config.
+func NewDefaultPingServerConfig() *PingServerConfig {
+	if *pingServerBindAddress == "" {
+		klog.Fatal("--ping-server-bind-address not set!")
+	}
+	return &PingServerConfig{
+		pingServerBindAddress: *pingServerBindAddress,
+	}
+}
+
+// Run runs the ping server.
+func Run(config *PingServerConfig) {
+	klog.Infof("Listening on %s \n", config.pingServerBindAddress)
+	http.HandleFunc("/", pong)
+	klog.Fatal(http.ListenAndServe(config.pingServerBindAddress, nil))
+}
+
+func pong(w http.ResponseWriter, r *http.Request) {
+	klog.Infof("pong -> %s\n", r.RemoteAddr)
+	w.Write([]byte("pong"))
+}

verify/verify-flags/known-flags.txt

@@ -18,3 +18,7 @@ report-dir
 enable-prometheus-server
 tear-down-prometheus-server
 kubemark-root-kubeconfig
+metric-bind-address
+ping-server-address
+ping-server-bind-address
+ping-sleep-duration