collector: shipper_objects_release_durations histogram

Back in 24787c8, we decided to collect
a summary instead of a histogram because we did not know how the
distribution of values looked like. It turns out that with a busy enough
cluster, a lot of releases may remain incomplete for... well, forever.
And although we know that, we still don't know any more about the
distribution of releases that actually eventually complete.

In any case, this distribution might be different for every shipper
installation, so there's not a whole lot of sense in hardcoding these
values anyway. So shipper-state-metrics learned a new argument,
`-release-duration-buckets`, where you can pass a comma-separated list
of buckets, in seconds, for the histogram.

NOTICE

@@ -3,28 +3,3 @@ Copyright 2018 Booking.com.
 
 This software contains code derived from the sample-controller by The
 Kubernetes Authors.
-
-This software contains code derived from the stats project by Montana Flynn,
-under the following license:
-
-The MIT License (MIT)
-
-Copyright (c) 2014-2015 Montana Flynn (https://anonfunction.com)
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.

cmd/shipper-state-metrics/collector.go

@@ -89,6 +89,8 @@ type ShipperStateMetrics struct {
 	secretsLister kubelisters.SecretLister
 
 	shipperNs string
+
+	releaseDurationBuckets []float64
 }
 
 func (ssm ShipperStateMetrics) Collect(ch chan<- prometheus.Metric) {
@@ -212,18 +214,13 @@ func (ssm ShipperStateMetrics) collectReleases(ch chan<- prometheus.Metric) {
 		ch <- prometheus.MustNewConstMetric(relsDesc, prometheus.GaugeValue, v, unkey(k)...)
 	}
 
-	quantiles := []float64{0.5, 0.9, 0.99}
-
 	for condition, ages := range relAgesByCondition {
 		count := uint64(len(ages))
 		sum := Sum(ages)
-		summary, err := MakeSummary(ages, quantiles)
-		if err != nil {
-			klog.Warningf("collect Releases: %s", err)
-			return
-		}
+		histogram := MakeHistogram(ages, ssm.releaseDurationBuckets)
 
-		ch <- prometheus.MustNewConstSummary(relDurationDesc, count, sum, summary, condition)
+		ch <- prometheus.MustNewConstHistogram(relDurationDesc, count,
+			sum, histogram, condition)
 	}
 }
 

cmd/shipper-state-metrics/main.go

@@ -5,6 +5,8 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
+	"strings"
 	"syscall"
 	"time"
 
@@ -25,6 +27,8 @@ var (
 	kubeconfig = flag.String("kubeconfig", "", "Path to a kubeconfig. Only required if out-of-cluster.")
 	addr       = flag.String("addr", ":8890", "Addr to expose /metrics on.")
 	ns         = flag.String("namespace", shipper.ShipperNamespace, "Namespace for Shipper resources.")
+
+	relDurationBuckets = flag.String("release-duration-buckets", "15,30,45,60,120", "Comma-separated list of buckets for the shipper_objects_release_durations histogram, in seconds")
 )
 
 func main() {
@@ -68,6 +72,8 @@ func main() {
 		secretsLister: kubeInformerFactory.Core().V1().Secrets().Lister(),
 
 		shipperNs: *ns,
+
+		releaseDurationBuckets: parseFloat64Slice(*relDurationBuckets),
 	}
 	prometheus.MustRegister(ssm)
 
@@ -113,6 +119,21 @@ func setupSignalHandler() <-chan struct{} {
 	return stopCh
 }
 
+func parseFloat64Slice(str string) []float64 {
+	strSlice := strings.Split(str, ",")
+	float64Slice := make([]float64, len(strSlice))
+
+	for i, b := range strSlice {
+		n, err := strconv.ParseFloat(b, 64)
+		if err != nil {
+			klog.Fatal(err)
+		}
+		float64Slice[i] = n
+	}
+
+	return float64Slice
+}
+
 type klogStdLogger struct{}
 
 func (klogStdLogger) Println(v ...interface{}) {

cmd/shipper-state-metrics/math.go

@@ -1,60 +1,23 @@
 package main
 
-import (
-	"errors"
-	"math"
-	"sort"
-)
-
-// Copyright (c) 2014-2015 Montana Flynn (https://anonfunction.com)
-// Licensed under the MIT License (check NOTICE for full license)
-// Obtained from https://github.com/montanaflynn/stats, minor modifications
-// were applied
-func Percentile(input []float64, percent float64) (percentile float64, err error) {
-	// Find the length of items in the slice
-	il := len(input)
-
-	// Return an error for empty slices
-	if il == 0 {
-		return 0, errors.New("empty input")
-	}
-
-	// Return error for less than 0 or greater than 100 percentages
-	if percent < 0 || percent > 100 {
-		return 0, errors.New("percent out of bounds")
-	}
-
-	// Sort the data
-	sort.Float64s(input)
-
-	// Return the last item
-	if percent == 100.0 {
-		return input[il-1], nil
-	}
-
-	// Find ordinal ranking
-	or := int(math.Ceil(float64(il) * percent / 100))
-
-	// Return the item that is in the place of the ordinal rank
-	if or == 0 {
-		return input[0], nil
-	}
-	return input[or-1], nil
-
-}
-
-func MakeSummary(input []float64, quantiles []float64) (map[float64]float64, error) {
-	summary := make(map[float64]float64)
-
-	for _, p := range quantiles {
-		percentile, err := Percentile(input, p*100)
-		summary[p] = percentile
-		if err != nil {
-			return nil, err
+func MakeHistogram(input []float64, buckets []float64) map[float64]uint64 {
+	histogram := make(map[float64]uint64)
+
+	for _, value := range input {
+		for _, bucket := range buckets {
+			// NOTE(jgreff): yes, this leaves out values in the
+			// +Inf bucket. Prometheus doesn't put them in any
+			// buckets, but infers it from the total count of
+			// observations, minus the observations in all the
+			// other buckets.
+			if value < bucket {
+				histogram[bucket]++
+				break
+			}
 		}
 	}
 
-	return summary, nil
+	return histogram
 }
 
 func Sum(input []float64) float64 {