forked from kubernetes/perf-tests
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Probes are simple jobs running inside kubernetes cluster measuring SLIs that could not be measured through other means. Ref. kubernetes/kubernetes#76012
- Loading branch information
Showing
11 changed files
with
344 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
FROM golang:1.12.1 AS build-env | ||
|
||
ARG gopkg=k8s.io/perf-tests/probes | ||
|
||
ADD ["cmd", "/go/src/$gopkg/cmd"] | ||
ADD ["pkg", "/go/src/$gopkg/pkg"] | ||
ADD ["go.mod", "/go/src/$gopkg"] | ||
ADD ["go.sum", "/go/src/$gopkg"] | ||
|
||
ENV GO111MODULE on | ||
WORKDIR /go/src/$gopkg | ||
RUN CGO_ENABLED=0 go build -o /workspace/probes ./cmd | ||
|
||
FROM golang:1.12.1-alpine | ||
WORKDIR /workspace | ||
COPY --from=build-env /workspace/probes . | ||
ENTRYPOINT ["/workspace/probes"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,17 @@ | ||
PROJECT = k8s-testimages | ||
IMG = gcr.io/$(PROJECT)/probes | ||
TAG = v0.0.1 | ||
|
||
all: push | ||
|
||
.PHONY: build | ||
build: cmd | ||
docker build --pull -t $(IMG):$(TAG) . | ||
docker tag $(IMG):$(TAG) $(IMG):latest | ||
@echo Built $(IMG):$(TAG) and tagged with latest | ||
|
||
.PHONY: push | ||
push: build | ||
docker push $(IMG):$(TAG) | ||
docker push $(IMG):latest | ||
@echo Pushed $(IMG) with :latest and :$(TAG) tags |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,7 @@ | ||
approvers: | ||
- mm4tt | ||
- wojtek-t | ||
reviewers: | ||
- mm4tt | ||
- wojtek-t | ||
|
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
# Probes | ||
|
||
Probes are simple jobs running inside kubernetes cluster measuring SLIs that could not be measured through other means. | ||
## Probe Types | ||
|
||
### Ping Client | ||
|
||
This probe exports the `probes_in_cluster_network_latency_seconds` metric. | ||
It implements the [Network Latency Scalability SLI] | ||
|
||
#### Running locally | ||
|
||
``` | ||
go run cmd/main.go --mode=ping-client --metric-bind-address=:8071 --ping-server-address=127.0.0.1:8081 --stderrthreshold=INFO | ||
``` | ||
|
||
### Ping Server | ||
|
||
This probe doesn't export any metrics, it's needed for the **Ping Client** to work. | ||
|
||
#### Running locally | ||
``` | ||
go run cmd/main.go --mode=ping-server --metric-bind-address=:8070 --ping-server-bind-address=0.0.0.0:8081 --stderrthreshold=INFO | ||
``` | ||
|
||
|
||
## Building and Releasing | ||
|
||
1. Increment the `TAG` in the Makefile. | ||
2. `make build` | ||
3. Test changes with `docker run gcr.io/k8s-testimages/probes:latest -- ...` | ||
4. Release with `make push` | ||
|
||
|
||
## Go Modules | ||
|
||
This project uses [Go Modules] to manage the external dependencies. | ||
|
||
|
||
[Go Modules]: https://github.com/golang/go/wiki/Modules | ||
[Network Latency Scalability SLI]: https://github.com/kubernetes/community/blob/master/sig-scalability/slos/network_latency.md |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,65 @@ | ||
/* | ||
Copyright 2019 The Kubernetes Authors. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package main | ||
|
||
import ( | ||
"flag" | ||
"net/http" | ||
|
||
"github.com/prometheus/client_golang/prometheus/promhttp" | ||
"k8s.io/klog" | ||
"k8s.io/perf-tests/probes/pkg/ping/client" | ||
"k8s.io/perf-tests/probes/pkg/ping/server" | ||
) | ||
|
||
var ( | ||
metricAddress = flag.String("metric-bind-address", "0.0.0.0:8080", "The address to serve the Prometheus metrics on.") | ||
mode = flag.String("mode", "", "Mode that should be run. Supported values: ping-server, ping-client") | ||
) | ||
|
||
func main() { | ||
klog.InitFlags(flag.CommandLine) | ||
flag.Parse() | ||
verifyFlags() | ||
|
||
klog.Infof("I'm probes.") | ||
klog.Infof("Mode is: %s\n", *mode) | ||
|
||
go func() { | ||
http.Handle("/metrics", promhttp.Handler()) | ||
klog.Infof("Serving metrics on %s\n", *metricAddress) | ||
klog.Fatal(http.ListenAndServe(*metricAddress, nil)) | ||
}() | ||
|
||
switch *mode { | ||
case "ping-client": | ||
pingclient.Run(pingclient.NewDefaultPingClientConfig()) | ||
case "ping-server": | ||
pingserver.Run(pingserver.NewDefaultPingServerConfig()) | ||
default: | ||
klog.Fatalf("Unrecognized mode: '%s'", *mode) | ||
} | ||
} | ||
|
||
func verifyFlags() { | ||
if *metricAddress == "" { | ||
klog.Fatal("--metric-bind-address not set!") | ||
} | ||
if *mode == "" { | ||
klog.Fatal("--mode not set!") | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,8 @@ | ||
module k8s.io/perf-tests/probes | ||
|
||
go 1.12 | ||
|
||
require ( | ||
github.com/prometheus/client_golang v0.9.2 | ||
k8s.io/klog v0.2.0 | ||
) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,19 @@ | ||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= | ||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= | ||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= | ||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= | ||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= | ||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= | ||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= | ||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= | ||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= | ||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= | ||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= | ||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= | ||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= | ||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= | ||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | ||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= | ||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | ||
k8s.io/klog v0.2.0 h1:0ElL0OHzF3N+OhoJTL0uca20SxtYt4X4+bzHeqrB83c= | ||
k8s.io/klog v0.2.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
/* | ||
Copyright 2019 The Kubernetes Authors. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package metrics | ||
|
||
import ( | ||
"github.com/prometheus/client_golang/prometheus" | ||
) | ||
|
||
const ( | ||
namespace = "probes" | ||
) | ||
|
||
var ( | ||
// InClusterNetworkLatency implements the In-Cluster Network Programming SLI, see | ||
// https://github.com/kubernetes/community/blob/master/sig-scalability/slos/network_latency.md | ||
InClusterNetworkLatency = prometheus.NewHistogram(prometheus.HistogramOpts{ | ||
Namespace: namespace, | ||
Name: "in_cluster_network_latency_seconds", | ||
Buckets: prometheus.ExponentialBuckets(0.000001, 2, 26), // from 1us up to ~1min | ||
Help: "Histogram of the time (in seconds) it took to ping a ping-server instance.", | ||
}) | ||
) | ||
|
||
func init() { | ||
prometheus.MustRegister(InClusterNetworkLatency) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,70 @@ | ||
/* | ||
Copyright 2019 The Kubernetes Authors. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package pingclient | ||
|
||
import ( | ||
"flag" | ||
"net/http" | ||
"time" | ||
|
||
"k8s.io/klog" | ||
"k8s.io/perf-tests/probes/pkg/metrics" | ||
) | ||
|
||
var ( | ||
pingServerAddress = flag.String("ping-server-address", "", "The address of the ping server") | ||
pingSleepDuration = flag.Duration("ping-sleep-duration", 1*time.Second, "Duration of the sleep between pings") | ||
) | ||
|
||
// PingClientConfig configures the "ping-client" probe. | ||
type PingClientConfig struct { | ||
pingServerAddress string | ||
pingSleepDuration time.Duration | ||
} | ||
|
||
// NewDefaultPingClientConfig creates a default "ping-client" config. | ||
func NewDefaultPingClientConfig() *PingClientConfig { | ||
if *pingServerAddress == "" { | ||
klog.Fatal("--ping-server-address not set!") | ||
} | ||
return &PingClientConfig{ | ||
pingServerAddress: *pingServerAddress, | ||
pingSleepDuration: *pingSleepDuration, | ||
} | ||
} | ||
|
||
// Run runs the ping client probe that periodically pings the ping server and exports latency metric. | ||
func Run(config *PingClientConfig) { | ||
for { | ||
time.Sleep(config.pingSleepDuration) | ||
klog.V(4).Infof("ping -> %s...\n", config.pingServerAddress) | ||
startTime := time.Now() | ||
if err := ping(config.pingServerAddress); err != nil { | ||
klog.Warningf("Got error: %v", err) | ||
// TODO(mm4tt): Increment server not available gauge metric. | ||
continue | ||
} | ||
latency := time.Since(startTime) | ||
klog.V(4).Infof("Request took: %v\n", latency) | ||
metrics.InClusterNetworkLatency.Observe(latency.Seconds()) | ||
} | ||
} | ||
|
||
func ping(serverAddress string) error { | ||
_, err := http.Get("http://" + serverAddress) | ||
return err | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,56 @@ | ||
/* | ||
Copyright 2019 The Kubernetes Authors. | ||
Licensed under the Apache License, Version 2.0 (the "License"); | ||
you may not use this file except in compliance with the License. | ||
You may obtain a copy of the License at | ||
http://www.apache.org/licenses/LICENSE-2.0 | ||
Unless required by applicable law or agreed to in writing, software | ||
distributed under the License is distributed on an "AS IS" BASIS, | ||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | ||
See the License for the specific language governing permissions and | ||
limitations under the License. | ||
*/ | ||
|
||
package pingserver | ||
|
||
import ( | ||
"flag" | ||
"net/http" | ||
|
||
"k8s.io/klog" | ||
) | ||
|
||
var ( | ||
pingServerBindAddress = flag.String("ping-server-bind-address", "", "The address to bind for the ping server") | ||
) | ||
|
||
// PingServerConfig configures the "ping-server" probe. | ||
type PingServerConfig struct { | ||
pingServerBindAddress string | ||
} | ||
|
||
// NewDefaultPingServerConfig creates a default "ping-server" config. | ||
func NewDefaultPingServerConfig() *PingServerConfig { | ||
if *pingServerBindAddress == "" { | ||
klog.Fatal("--ping-server-bind-address not set!") | ||
} | ||
return &PingServerConfig{ | ||
pingServerBindAddress: *pingServerBindAddress, | ||
} | ||
} | ||
|
||
// Run runs the ping server. | ||
func Run(config *PingServerConfig) { | ||
klog.Infof("Listening on %s \n", config.pingServerBindAddress) | ||
http.HandleFunc("/", pong) | ||
klog.Fatal(http.ListenAndServe(config.pingServerBindAddress, nil)) | ||
} | ||
|
||
func pong(w http.ResponseWriter, r *http.Request) { | ||
klog.V(4).Infof("pong -> %s\n", r.RemoteAddr) | ||
w.WriteHeader(http.StatusOK) | ||
w.Write([]byte("pong")) | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters