From 12f623882badd0008354060211e439c16a858e47 Mon Sep 17 00:00:00 2001 From: mario rodriguez Date: Wed, 9 Oct 2019 16:30:11 +0200 Subject: [PATCH 1/4] Upgrade golang client to v1.0.0 version --- go.mod | 10 +--------- go.sum | 49 ++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index f177ef7..7d4d1fe 100644 --- a/go.mod +++ b/go.mod @@ -3,15 +3,7 @@ module github.com/cabify/gotoprom go 1.12 require ( - github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 // indirect - github.com/gogo/protobuf v1.2.1 // indirect - github.com/golang/protobuf v1.2.0 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect - github.com/prometheus/client_golang v0.9.1 - github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 // indirect - github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 // indirect - github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d // indirect + github.com/prometheus/client_golang v1.0.0 github.com/stretchr/objx v0.2.0 // indirect github.com/stretchr/testify v1.3.0 - golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 // indirect ) diff --git a/go.sum b/go.sum index 1266d47..4a886dd 100644 --- a/go.sum +++ b/go.sum @@ -1,31 +1,62 @@ +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0 h1:HWo1m869IqiPhD389kmkxeTalrjNbbJTC8LXupb+sl0= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE= -github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= -github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v0.9.1 h1:K47Rk0v/fkEfwfQet2KWhscE0cJzjgCCDBG2KHZoVno= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0 h1:vrDKnkGzuGvhNAL56c7DBz29ZL+KxnoR0x7enabFceM= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39 h1:Cto4X6SVMWRPBkJ/3YHn1iDGDGc/Z+sW+AEMKHMVvN4= -github.com/prometheus/common v0.0.0-20181020173914-7e9e6cabbd39/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90 h1:S/YWwWx/RA8rT8tKFRuGUZhuA90OyIBpPCXkcbwU8DE= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/common v0.4.1 h1:K0MGApIoQvMw27RTdJkPbr3JZ7DNbtxQNyi5STVM6Kw= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d h1:GoAlyOgbOEIFdaDqxJVlbOQ1DtGmZWs/Qau0hIlk+WQ= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2 h1:6LJUbpNm42llc4HRCuvApCSWB/WfhuNo9K98Q9sNGfs= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5 h1:mzjBh+S5frKOsOBobWIMAbXavqjmgO17k/2puhcFR94= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From f4c3f454afa5d771a9c3416c2286653746136626 Mon Sep 17 00:00:00 2001 From: mario rodriguez Date: Thu, 10 Oct 2019 11:12:00 +0200 Subject: [PATCH 2/4] Add objectives to summaries from struct tag Prometheus has deprecated the default values for objectives and they now need to be explicitly specified. Default values are those that client_golang library had before deprecation in v1. If no objectives are specified, default objectives are used. --- prometheusvanilla/builders.go | 41 ++++++++++++++++++++++++++---- prometheusvanilla/builders_test.go | 31 ++++++++++++++++------ 2 files changed, 59 insertions(+), 13 deletions(-) diff --git a/prometheusvanilla/builders.go b/prometheusvanilla/builders.go index cf1643a..2d29f96 100644 --- a/prometheusvanilla/builders.go +++ b/prometheusvanilla/builders.go @@ -2,6 +2,7 @@ package prometheusvanilla import ( "fmt" + "math" "reflect" "strconv" "strings" @@ -85,13 +86,14 @@ func BuildSummary(name, help, namespace string, labelNames []string, tag reflect if err != nil { return nil, nil, fmt.Errorf("build summary %q: %s", name, err) } - + objectives, err := objectivesFromTag(tag) sum := prometheus.NewSummaryVec( prometheus.SummaryOpts{ - Name: name, - Help: help, - Namespace: namespace, - MaxAge: maxAge, + Name: name, + Help: help, + Namespace: namespace, + MaxAge: maxAge, + Objectives: objectives, }, labelNames, ) @@ -135,3 +137,32 @@ func maxAgeFromTag(tag reflect.StructTag) (time.Duration, error) { } return maxAgeDuration, nil } + +func objectivesFromTag(tag reflect.StructTag) (map[float64]float64, error) { + quantileString, ok := tag.Lookup("objectives") + if !ok { + return DefaultObjectives(), nil + } + quantileSlice := strings.Split(quantileString, ",") + objectives := make(map[float64]float64, len(quantileSlice)) + for _, quantile := range quantileSlice { + q, err := strconv.ParseFloat(quantile, 64) + if err != nil { + return nil, fmt.Errorf("invalid objective specified: %s", err) + } + objectives[q] = absError(q) + } + return objectives, nil +} + +// DefaultObjectives provides a list of objectives you can use when you don't know what to use yet. +func DefaultObjectives() map[float64]float64 { + return map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001} +} + +// absError will calculate the absolute error for a given objective up to 3 decimal places. +// The variance is calculated based on the given quantile. +// Values in lower quantiles have a higher probability of being similar, so we can apply greater variances. +func absError(obj float64) float64 { + return math.Round((0.1*(1-obj))*1000) / 1000 +} diff --git a/prometheusvanilla/builders_test.go b/prometheusvanilla/builders_test.go index 374ad84..45abd3c 100644 --- a/prometheusvanilla/builders_test.go +++ b/prometheusvanilla/builders_test.go @@ -16,13 +16,15 @@ const ( ) var ( - labels = make(prometheus.Labels, 2) - keys = make([]string, 0, len(labels)) - defaultTag reflect.StructTag = `name:"some_name" help:"some help for the metric"` - bucketsTag reflect.StructTag = `name:"some_name" help:"some help for the metric" buckets:"0.001,0.005,0.01,0.05,0.1,0.5,1,5,10"` - maxAgeTag reflect.StructTag = `name:"some_name" help:"some help for the metric" max_age:"1h"` - expectedBuckets = []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10} - expectedMaxAge = time.Hour + labels = make(prometheus.Labels, 2) + keys = make([]string, 0, len(labels)) + defaultTag reflect.StructTag = `name:"some_name" help:"some help for the metric"` + bucketsTag reflect.StructTag = `name:"some_name" help:"some help for the metric" buckets:"0.001,0.005,0.01,0.05,0.1,0.5,1,5,10"` + maxAgeTag reflect.StructTag = `name:"some_name" help:"some help for the metric" max_age:"1h"` + objectivesTag reflect.StructTag = `name:"some_name" help:"some help for the metric" max_age:"1h" objectives:"0.55,0.95,0.98"` + expectedBuckets = []float64{0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10} + expectedMaxAge = time.Hour + expectedObjectives = map[float64]float64{0.55: 0.045, 0.95: 0.005, 0.98: 0.002} ) func TestBuilders(t *testing.T) { @@ -62,7 +64,7 @@ func TestBuilders(t *testing.T) { }) t.Run("Test building a summary with malformed max_age", func(t *testing.T) { - _, _, err := BuildSummary(name, help, nameSpace, keys, `max_age:"one year"`) + _, _, err := BuildSummary(name, help, nameSpace, keys, `max_age:"one year" objectives:"0.1,0.25"`) assert.Error(t, err) }) } @@ -94,6 +96,19 @@ func TestMaxAge(t *testing.T) { }) } +func TestObjectives(t *testing.T) { + t.Run("Test parsing objectives from tag", func(t *testing.T) { + obj, err := objectivesFromTag(objectivesTag) + assert.NoError(t, err) + assert.Equal(t, obj, expectedObjectives) + }) + t.Run("Test returning default objective values when none are specified", func(t *testing.T) { + obj, err := objectivesFromTag(defaultTag) + assert.NoError(t, err) + assert.Equal(t, obj, DefaultObjectives()) + }) +} + func initLabels() { labels["labels1"] = "labels" labels["labels2"] = "labels" From c5137c878ee13d21ef4dfa03f4a9ed4eb446a919 Mon Sep 17 00:00:00 2001 From: mario rodriguez Date: Thu, 10 Oct 2019 11:27:56 +0200 Subject: [PATCH 3/4] Update CHANGELEOG with v0.3.0 changes on summaries - Add objectives to summaries through struct tag and set default values when none specified - Upgraded client_golang to v1 --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 98c8b3c..2fd0a89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [0.3.0] - 2019-10-10 +### Added +- Add objectives to summaries through struct tag and set default values when none specified +### Changed +- Upgraded client_golang to v1 + ## [0.2.1] - 2019-06-05 ### Changed - Reduced the number of default buckets from 12 to 7 between 0.05s and 10s From 9e1f2b1717cb8515a86eb2a8063c8a053667a943 Mon Sep 17 00:00:00 2001 From: mario rodriguez Date: Thu, 10 Oct 2019 11:49:39 +0200 Subject: [PATCH 4/4] Add tests for absError --- prometheusvanilla/builders_test.go | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/prometheusvanilla/builders_test.go b/prometheusvanilla/builders_test.go index 45abd3c..0af61eb 100644 --- a/prometheusvanilla/builders_test.go +++ b/prometheusvanilla/builders_test.go @@ -1,6 +1,7 @@ package prometheusvanilla import ( + "fmt" "reflect" "testing" "time" @@ -109,6 +110,26 @@ func TestObjectives(t *testing.T) { }) } +func TestAbsError(t *testing.T) { + tests := []struct { + objective float64 + absErr float64 + }{ + {0.5, 0.05}, + {0.99, 0.001}, + {0.999, 0}, + {0, 0.1}, + {-10, 1.1}, + } + + for _, test := range tests { + t.Run(fmt.Sprintf("Test calculate absolute error for obejctive %2f", test.objective), func(t *testing.T) { + e := absError(test.objective) + assert.Equal(t, test.absErr, e) + }) + } +} + func initLabels() { labels["labels1"] = "labels" labels["labels2"] = "labels"