From eabb2028a2733f7123f8eb798c6c14d670dc8fad Mon Sep 17 00:00:00 2001 From: Henrique Rodrigues Date: Fri, 2 Nov 2018 15:10:55 -0300 Subject: [PATCH 1/2] getting custom metrics on prometheus --- app.go | 20 ++++----- config/config.go | 6 +++ metrics/models.go | 49 ++++++++++++++++++++++ metrics/prometheus_reporter.go | 75 +++++++++++++++++++++++++++++++--- 4 files changed, 133 insertions(+), 17 deletions(-) create mode 100644 metrics/models.go diff --git a/app.go b/app.go index 68bf6655..073e29bd 100644 --- a/app.go +++ b/app.go @@ -142,21 +142,19 @@ func Configure( func configureMetrics(serverType string) { app.metricsReporters = make([]metrics.Reporter, 0) - constTags := app.config.GetStringMapString("pitaya.metrics.constTags") + if app.config.GetBool("pitaya.metrics.prometheus.enabled") { port := app.config.GetInt("pitaya.metrics.prometheus.port") - game := app.config.GetString("pitaya.game") - logger.Log.Infof( - "prometheus is enabled, configuring the metrics reporter on port %d", port, - ) - - additionalTags := app.config.GetStringMapString("pitaya.metrics.additionalTags") - prometheus := metrics.GetPrometheusReporter(serverType, game, port, - constTags, additionalTags) - AddMetricsReporter(prometheus) + logger.Log.Infof("prometheus is enabled, configuring reporter on port %d", port) + prometheus, err := metrics.GetPrometheusReporter(serverType, app.config, constTags) + if err != nil { + logger.Log.Errorf("failed to start prometheus metrics reporter, skipping %v", err) + } else { + AddMetricsReporter(prometheus) + } } else { - logger.Log.Info("prometheus is disabled, the metrics reporter will not be enabled") + logger.Log.Info("prometheus is disabled, reporter will not be enabled") } if app.config.GetBool("pitaya.metrics.statsd.enabled") { diff --git a/config/config.go b/config/config.go index 0ee21cd5..4aaff5b6 100644 --- a/config/config.go +++ b/config/config.go @@ -98,6 +98,7 @@ func (c *Config) fillDefaultValues() { "pitaya.metrics.constTags": map[string]string{}, "pitaya.metrics.additionalTags": map[string]string{}, "pitaya.metrics.periodicMetrics.period": "15s", + "pitaya.metrics.custom": map[string]interface{}{}, "pitaya.defaultpipelines.structvalidation.enabled": false, "pitaya.worker.redis.url": "localhost:6379", "pitaya.worker.redis.pool": "10", @@ -151,3 +152,8 @@ func (c *Config) Get(s string) interface{} { func (c *Config) GetStringMapString(s string) map[string]string { return c.config.GetStringMapString(s) } + +// UnmarshalKey unmarshals key into v +func (c *Config) UnmarshalKey(s string, v interface{}) error { + return c.config.UnmarshalKey(s, v) +} diff --git a/metrics/models.go b/metrics/models.go new file mode 100644 index 00000000..31cc533b --- /dev/null +++ b/metrics/models.go @@ -0,0 +1,49 @@ +package metrics + +import ( + "github.com/topfreegames/pitaya/config" +) + +// Summary defines a summary metric +type Summary struct { + Subsystem string + Name string + Help string + Objectives map[float64]float64 + Labels []string +} + +// Gauge defines a gauge metric +type Gauge struct { + Subsystem string + Name string + Help string + Labels []string +} + +// Counter defines a counter metric +type Counter struct { + Subsystem string + Name string + Help string + Labels []string +} + +// CustomMetricsSpec has all metrics specs +type CustomMetricsSpec struct { + Summaries []*Summary + Gauges []*Gauge + Counters []*Counter +} + +// NewCustomMetricsSpec returns a *CustomMetricsSpec by reading config key +func NewCustomMetricsSpec(config *config.Config) (*CustomMetricsSpec, error) { + var spec CustomMetricsSpec + + err := config.UnmarshalKey("pitaya.metrics.custom", &spec) + if err != nil { + return nil, err + } + + return &spec, nil +} diff --git a/metrics/prometheus_reporter.go b/metrics/prometheus_reporter.go index df1a1e5b..ff162c16 100644 --- a/metrics/prometheus_reporter.go +++ b/metrics/prometheus_reporter.go @@ -27,6 +27,7 @@ import ( "sync" "github.com/prometheus/client_golang/prometheus" + "github.com/topfreegames/pitaya/config" "github.com/topfreegames/pitaya/constants" ) @@ -45,9 +46,57 @@ type PrometheusReporter struct { additionalLabels map[string]string } +func (p *PrometheusReporter) registerCustomMetrics( + constLabels map[string]string, + additionalLabelsKeys []string, + spec *CustomMetricsSpec, +) { + for _, summary := range spec.Summaries { + p.summaryReportersMap[summary.Name] = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: "pitaya", + Subsystem: summary.Subsystem, + Name: summary.Name, + Help: summary.Help, + Objectives: summary.Objectives, + ConstLabels: constLabels, + }, + append(additionalLabelsKeys, summary.Labels...), + ) + } + + for _, gauge := range spec.Gauges { + p.gaugeReportersMap[gauge.Name] = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "pitaya", + Subsystem: gauge.Subsystem, + Name: gauge.Name, + Help: gauge.Help, + ConstLabels: constLabels, + }, + append(additionalLabelsKeys, gauge.Labels...), + ) + } + + for _, counter := range spec.Counters { + p.countReportersMap[counter.Name] = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "pitaya", + Subsystem: counter.Subsystem, + Name: counter.Name, + Help: counter.Help, + ConstLabels: constLabels, + }, + append(additionalLabelsKeys, counter.Labels...), + ) + } +} + func (p *PrometheusReporter) registerMetrics( constLabels, additionalLabels map[string]string, + spec *CustomMetricsSpec, ) { + constLabels["game"] = p.game constLabels["serverType"] = p.serverType @@ -57,6 +106,8 @@ func (p *PrometheusReporter) registerMetrics( additionalLabelsKeys = append(additionalLabelsKeys, key) } + p.registerCustomMetrics(constLabels, additionalLabelsKeys, spec) + // HandlerResponseTimeMs summary p.summaryReportersMap[ResponseTime] = prometheus.NewSummaryVec( prometheus.SummaryOpts{ @@ -212,10 +263,21 @@ func (p *PrometheusReporter) registerMetrics( // GetPrometheusReporter gets the prometheus reporter singleton func GetPrometheusReporter( - serverType, game string, - port int, - constLabels, additionalLabels map[string]string, -) *PrometheusReporter { + serverType string, + config *config.Config, + constLabels map[string]string, +) (*PrometheusReporter, error) { + var ( + port = config.GetInt("pitaya.metrics.prometheus.port") + game = config.GetString("pitaya.game") + additionalLabels = config.GetStringMapString("pitaya.metrics.additionalTags") + ) + + spec, err := NewCustomMetricsSpec(config) + if err != nil { + return nil, err + } + once.Do(func() { prometheusReporter = &PrometheusReporter{ serverType: serverType, @@ -224,13 +286,14 @@ func GetPrometheusReporter( summaryReportersMap: make(map[string]*prometheus.SummaryVec), gaugeReportersMap: make(map[string]*prometheus.GaugeVec), } - prometheusReporter.registerMetrics(constLabels, additionalLabels) + prometheusReporter.registerMetrics(constLabels, additionalLabels, spec) http.Handle("/metrics", prometheus.Handler()) go (func() { log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil)) })() }) - return prometheusReporter + + return prometheusReporter, nil } // ReportSummary reports a summary metric From 61add6356a1749625bef02ee1fff6538604a88a5 Mon Sep 17 00:00:00 2001 From: Henrique Rodrigues Date: Sun, 4 Nov 2018 19:13:34 -0200 Subject: [PATCH 2/2] update on doc --- docs/configuration.rst | 64 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/docs/configuration.rst b/docs/configuration.rst index 42f2aa77..0c85c30e 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -224,6 +224,70 @@ Metrics Reporting - 15s - string - Period that system metrics will be reported + * - pitaya.metrics.custom.counters + - []map[string]interface{} + - []map[string]interface + - Custom metrics counter + * - pitaya.metrics.custom.counters[].Subsystem + - "" + - string + - Custom counter subsystem name + * - pitaya.metrics.custom.counters[].Name + - "" + - string + - Custom counter name, must not be empty + * - pitaya.metrics.custom.counters[].Help + - "" + - string + - Custom counter help which explain what is the metric, must not be empty + * - pitaya.metrics.custom.counters[].Labels + - []string{} + - []string + - Custom counter labels the metric will carry + * - pitaya.metrics.custom.gauges + - []map[string]interface{} + - []map[string]interface + - Custom metrics gauge + * - pitaya.metrics.custom.gauges[].Subsystem + - "" + - string + - Custom gauge subsystem name + * - pitaya.metrics.custom.gauges[].Name + - "" + - string + - Custom gauge name, must not be empty + * - pitaya.metrics.custom.gauges[].Help + - "" + - string + - Custom gauge help which explain what is the metric, must not be empty + * - pitaya.metrics.custom.gauges[].Labels + - []string{} + - []string + - Custom gauge labels the metric will carry + * - pitaya.metrics.custom.summaries + - []map[string]interface{} + - []map[string]interface + - Custom metrics summary + * - pitaya.metrics.custom.summaries[].Subsystem + - "" + - string + - Custom summary subsystem name + * - pitaya.metrics.custom.summaries[].Name + - "" + - string + - Custom summary name, must not be empty + * - pitaya.metrics.custom.summaries[].Help + - "" + - string + - Custom summary help which explain what is the metric, must not be empty + * - pitaya.metrics.custom.summaries[].Labels + - []string{} + - []string + - Custom summary labels the metric will carry + * - pitaya.metrics.custom.summaries[].Objectives + - map[float64]float64 + - map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001} + - Custom summary objectives with quantiles Concurrency ===========