Skip to content
Permalink
Browse files

collector: shipper_objects_release_durations histogram

Back in 24787c8, we decided to collect
a summary instead of a histogram because we did not know how the
distribution of values looked like. It turns out that with a busy enough
cluster, a lot of releases may remain incomplete for... well, forever.
And although we know that, we still don't know any more about the
distribution of releases that actually eventually complete.

In any case, this distribution might be different for every shipper
installation, so there's not a whole lot of sense in hardcoding these
values anyway. So shipper-state-metrics learned a new argument,
`-release-duration-buckets`, where you can pass a comma-separated list
of buckets, in seconds, for the histogram.
  • Loading branch information...
juliogreff authored and icanhazbroccoli committed Sep 3, 2019
1 parent 85a4b46 commit c004702b15e7507012e2ee3fc00e414ba130b117
Showing with 41 additions and 85 deletions.
  1. +0 −25 NOTICE
  2. +5 −8 cmd/shipper-state-metrics/collector.go
  3. +21 −0 cmd/shipper-state-metrics/main.go
  4. +15 −52 cmd/shipper-state-metrics/math.go
25 NOTICE
@@ -3,28 +3,3 @@ Copyright 2018 Booking.com.

This software contains code derived from the sample-controller by The
Kubernetes Authors.

This software contains code derived from the stats project by Montana Flynn,
under the following license:

The MIT License (MIT)

Copyright (c) 2014-2015 Montana Flynn (https://anonfunction.com)

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
@@ -89,6 +89,8 @@ type ShipperStateMetrics struct {
secretsLister kubelisters.SecretLister

shipperNs string

releaseDurationBuckets []float64
}

func (ssm ShipperStateMetrics) Collect(ch chan<- prometheus.Metric) {
@@ -212,18 +214,13 @@ func (ssm ShipperStateMetrics) collectReleases(ch chan<- prometheus.Metric) {
ch <- prometheus.MustNewConstMetric(relsDesc, prometheus.GaugeValue, v, unkey(k)...)
}

quantiles := []float64{0.5, 0.9, 0.99}

for condition, ages := range relAgesByCondition {
count := uint64(len(ages))
sum := Sum(ages)
summary, err := MakeSummary(ages, quantiles)
if err != nil {
klog.Warningf("collect Releases: %s", err)
return
}
histogram := MakeHistogram(ages, ssm.releaseDurationBuckets)

ch <- prometheus.MustNewConstSummary(relDurationDesc, count, sum, summary, condition)
ch <- prometheus.MustNewConstHistogram(relDurationDesc, count,
sum, histogram, condition)
}
}

@@ -5,6 +5,8 @@ import (
"net/http"
"os"
"os/signal"
"strconv"
"strings"
"syscall"
"time"

@@ -25,6 +27,8 @@ var (
kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
addr = flag.String("addr", ":8890", "Addr to expose /metrics on.")
ns = flag.String("namespace", shipper.ShipperNamespace, "Namespace for Shipper resources.")

relDurationBuckets = flag.String("release-duration-buckets", "15,30,45,60,120", "Comma-separated list of buckets for the shipper_objects_release_durations histogram, in seconds")
)

func main() {
@@ -68,6 +72,8 @@ func main() {
secretsLister: kubeInformerFactory.Core().V1().Secrets().Lister(),

shipperNs: *ns,

releaseDurationBuckets: parseFloat64Slice(*relDurationBuckets),
}
prometheus.MustRegister(ssm)

@@ -113,6 +119,21 @@ func setupSignalHandler() <-chan struct{} {
return stopCh
}

func parseFloat64Slice(str string) []float64 {
strSlice := strings.Split(str, ",")
float64Slice := make([]float64, len(strSlice))

for i, b := range strSlice {
n, err := strconv.ParseFloat(b, 64)
if err != nil {
klog.Fatal(err)
}
float64Slice[i] = n
}

return float64Slice
}

type klogStdLogger struct{}

func (klogStdLogger) Println(v ...interface{}) {
@@ -1,60 +1,23 @@
package main

import (
"errors"
"math"
"sort"
)

// Copyright (c) 2014-2015 Montana Flynn (https://anonfunction.com)
// Licensed under the MIT License (check NOTICE for full license)
// Obtained from https://github.com/montanaflynn/stats, minor modifications
// were applied
func Percentile(input []float64, percent float64) (percentile float64, err error) {
// Find the length of items in the slice
il := len(input)

// Return an error for empty slices
if il == 0 {
return 0, errors.New("empty input")
}

// Return error for less than 0 or greater than 100 percentages
if percent < 0 || percent > 100 {
return 0, errors.New("percent out of bounds")
}

// Sort the data
sort.Float64s(input)

// Return the last item
if percent == 100.0 {
return input[il-1], nil
}

// Find ordinal ranking
or := int(math.Ceil(float64(il) * percent / 100))

// Return the item that is in the place of the ordinal rank
if or == 0 {
return input[0], nil
}
return input[or-1], nil

}

func MakeSummary(input []float64, quantiles []float64) (map[float64]float64, error) {
summary := make(map[float64]float64)

for _, p := range quantiles {
percentile, err := Percentile(input, p*100)
summary[p] = percentile
if err != nil {
return nil, err
func MakeHistogram(input []float64, buckets []float64) map[float64]uint64 {
histogram := make(map[float64]uint64)

for _, value := range input {
for _, bucket := range buckets {
// NOTE(jgreff): yes, this leaves out values in the
// +Inf bucket. Prometheus doesn't put them in any
// buckets, but infers it from the total count of
// observations, minus the observations in all the
// other buckets.
if value < bucket {
histogram[bucket]++
break
}
}
}

return summary, nil
return histogram
}

func Sum(input []float64) float64 {

0 comments on commit c004702

Please sign in to comment.
You can’t perform that action at this time.