diff --git a/.chloggen/redisreceiver_cmd_latency.yaml b/.chloggen/redisreceiver_cmd_latency.yaml new file mode 100755 index 0000000000000..6e2d632bdc9ba --- /dev/null +++ b/.chloggen/redisreceiver_cmd_latency.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: redisreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Added `redis.cmd.latency` metric. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [6942] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [user] diff --git a/receiver/redisreceiver/documentation.md b/receiver/redisreceiver/documentation.md index 4483a9b8d8afd..759d107267f6e 100644 --- a/receiver/redisreceiver/documentation.md +++ b/receiver/redisreceiver/documentation.md @@ -292,6 +292,21 @@ Total number of calls for a command | ---- | ----------- | ------ | | cmd | Redis command name | Any Str | +### redis.cmd.latency + +Command execution latency + +| Unit | Metric Type | Value Type | +| ---- | ----------- | ---------- | +| us | Gauge | Double | + +#### Attributes + +| Name | Description | Values | +| ---- | ----------- | ------ | +| cmd | Redis command name | Any Str | +| percentile | Percentile | Str: ``p50``, ``p99``, ``p99.9`` | + ### redis.cmd.usec Total time for all executions of this command diff --git a/receiver/redisreceiver/internal/metadata/generated_config.go b/receiver/redisreceiver/internal/metadata/generated_config.go index 8fa7b783bf7ea..cea09b3550e9d 100644 --- a/receiver/redisreceiver/internal/metadata/generated_config.go +++ b/receiver/redisreceiver/internal/metadata/generated_config.go @@ -30,6 +30,7 @@ type MetricsConfig struct { RedisClientsMaxInputBuffer MetricConfig `mapstructure:"redis.clients.max_input_buffer"` RedisClientsMaxOutputBuffer MetricConfig `mapstructure:"redis.clients.max_output_buffer"` RedisCmdCalls MetricConfig `mapstructure:"redis.cmd.calls"` + RedisCmdLatency MetricConfig `mapstructure:"redis.cmd.latency"` RedisCmdUsec MetricConfig `mapstructure:"redis.cmd.usec"` RedisCommands MetricConfig `mapstructure:"redis.commands"` RedisCommandsProcessed MetricConfig `mapstructure:"redis.commands.processed"` @@ -77,6 +78,9 @@ func DefaultMetricsConfig() MetricsConfig { RedisCmdCalls: MetricConfig{ Enabled: false, }, + RedisCmdLatency: MetricConfig{ + Enabled: false, + }, RedisCmdUsec: MetricConfig{ Enabled: false, }, diff --git a/receiver/redisreceiver/internal/metadata/generated_config_test.go b/receiver/redisreceiver/internal/metadata/generated_config_test.go index c3a9e64ff4f45..f0043cd738318 100644 --- a/receiver/redisreceiver/internal/metadata/generated_config_test.go +++ b/receiver/redisreceiver/internal/metadata/generated_config_test.go @@ -31,6 +31,7 @@ func TestMetricsBuilderConfig(t *testing.T) { RedisClientsMaxInputBuffer: MetricConfig{Enabled: true}, RedisClientsMaxOutputBuffer: MetricConfig{Enabled: true}, RedisCmdCalls: MetricConfig{Enabled: true}, + RedisCmdLatency: MetricConfig{Enabled: true}, RedisCmdUsec: MetricConfig{Enabled: true}, RedisCommands: MetricConfig{Enabled: true}, RedisCommandsProcessed: MetricConfig{Enabled: true}, @@ -74,6 +75,7 @@ func TestMetricsBuilderConfig(t *testing.T) { RedisClientsMaxInputBuffer: MetricConfig{Enabled: false}, RedisClientsMaxOutputBuffer: MetricConfig{Enabled: false}, RedisCmdCalls: MetricConfig{Enabled: false}, + RedisCmdLatency: MetricConfig{Enabled: false}, RedisCmdUsec: MetricConfig{Enabled: false}, RedisCommands: MetricConfig{Enabled: false}, RedisCommandsProcessed: MetricConfig{Enabled: false}, diff --git a/receiver/redisreceiver/internal/metadata/generated_metrics.go b/receiver/redisreceiver/internal/metadata/generated_metrics.go index a225a72d0a3a0..5ec5d63834328 100644 --- a/receiver/redisreceiver/internal/metadata/generated_metrics.go +++ b/receiver/redisreceiver/internal/metadata/generated_metrics.go @@ -11,6 +11,36 @@ import ( "go.opentelemetry.io/collector/receiver" ) +// AttributePercentile specifies the a value percentile attribute. +type AttributePercentile int + +const ( + _ AttributePercentile = iota + AttributePercentileP50 + AttributePercentileP99 + AttributePercentileP999 +) + +// String returns the string representation of the AttributePercentile. +func (av AttributePercentile) String() string { + switch av { + case AttributePercentileP50: + return "p50" + case AttributePercentileP99: + return "p99" + case AttributePercentileP999: + return "p99.9" + } + return "" +} + +// MapAttributePercentile is a helper map of string to AttributePercentile attribute value. +var MapAttributePercentile = map[string]AttributePercentile{ + "p50": AttributePercentileP50, + "p99": AttributePercentileP99, + "p99.9": AttributePercentileP999, +} + // AttributeRole specifies the a value role attribute. type AttributeRole int @@ -332,6 +362,58 @@ func newMetricRedisCmdCalls(cfg MetricConfig) metricRedisCmdCalls { return m } +type metricRedisCmdLatency struct { + data pmetric.Metric // data buffer for generated metric. + config MetricConfig // metric config provided by user. + capacity int // max observed number of data points added to the metric. +} + +// init fills redis.cmd.latency metric with initial data. +func (m *metricRedisCmdLatency) init() { + m.data.SetName("redis.cmd.latency") + m.data.SetDescription("Command execution latency") + m.data.SetUnit("us") + m.data.SetEmptyGauge() + m.data.Gauge().DataPoints().EnsureCapacity(m.capacity) +} + +func (m *metricRedisCmdLatency) recordDataPoint(start pcommon.Timestamp, ts pcommon.Timestamp, val float64, cmdAttributeValue string, percentileAttributeValue string) { + if !m.config.Enabled { + return + } + dp := m.data.Gauge().DataPoints().AppendEmpty() + dp.SetStartTimestamp(start) + dp.SetTimestamp(ts) + dp.SetDoubleValue(val) + dp.Attributes().PutStr("cmd", cmdAttributeValue) + dp.Attributes().PutStr("percentile", percentileAttributeValue) +} + +// updateCapacity saves max length of data point slices that will be used for the slice capacity. +func (m *metricRedisCmdLatency) updateCapacity() { + if m.data.Gauge().DataPoints().Len() > m.capacity { + m.capacity = m.data.Gauge().DataPoints().Len() + } +} + +// emit appends recorded metric data to a metrics slice and prepares it for recording another set of data points. +func (m *metricRedisCmdLatency) emit(metrics pmetric.MetricSlice) { + if m.config.Enabled && m.data.Gauge().DataPoints().Len() > 0 { + m.updateCapacity() + m.data.MoveTo(metrics.AppendEmpty()) + m.init() + } +} + +func newMetricRedisCmdLatency(cfg MetricConfig) metricRedisCmdLatency { + m := metricRedisCmdLatency{config: cfg} + if cfg.Enabled { + m.data = pmetric.NewMetric() + m.init() + } + return m +} + type metricRedisCmdUsec struct { data pmetric.Metric // data buffer for generated metric. config MetricConfig // metric config provided by user. @@ -1759,6 +1841,7 @@ type MetricsBuilder struct { metricRedisClientsMaxInputBuffer metricRedisClientsMaxInputBuffer metricRedisClientsMaxOutputBuffer metricRedisClientsMaxOutputBuffer metricRedisCmdCalls metricRedisCmdCalls + metricRedisCmdLatency metricRedisCmdLatency metricRedisCmdUsec metricRedisCmdUsec metricRedisCommands metricRedisCommands metricRedisCommandsProcessed metricRedisCommandsProcessed @@ -1810,6 +1893,7 @@ func NewMetricsBuilder(mbc MetricsBuilderConfig, settings receiver.CreateSetting metricRedisClientsMaxInputBuffer: newMetricRedisClientsMaxInputBuffer(mbc.Metrics.RedisClientsMaxInputBuffer), metricRedisClientsMaxOutputBuffer: newMetricRedisClientsMaxOutputBuffer(mbc.Metrics.RedisClientsMaxOutputBuffer), metricRedisCmdCalls: newMetricRedisCmdCalls(mbc.Metrics.RedisCmdCalls), + metricRedisCmdLatency: newMetricRedisCmdLatency(mbc.Metrics.RedisCmdLatency), metricRedisCmdUsec: newMetricRedisCmdUsec(mbc.Metrics.RedisCmdUsec), metricRedisCommands: newMetricRedisCommands(mbc.Metrics.RedisCommands), metricRedisCommandsProcessed: newMetricRedisCommandsProcessed(mbc.Metrics.RedisCommandsProcessed), @@ -1904,6 +1988,7 @@ func (mb *MetricsBuilder) EmitForResource(rmo ...ResourceMetricsOption) { mb.metricRedisClientsMaxInputBuffer.emit(ils.Metrics()) mb.metricRedisClientsMaxOutputBuffer.emit(ils.Metrics()) mb.metricRedisCmdCalls.emit(ils.Metrics()) + mb.metricRedisCmdLatency.emit(ils.Metrics()) mb.metricRedisCmdUsec.emit(ils.Metrics()) mb.metricRedisCommands.emit(ils.Metrics()) mb.metricRedisCommandsProcessed.emit(ils.Metrics()) @@ -1977,6 +2062,11 @@ func (mb *MetricsBuilder) RecordRedisCmdCallsDataPoint(ts pcommon.Timestamp, val mb.metricRedisCmdCalls.recordDataPoint(mb.startTime, ts, val, cmdAttributeValue) } +// RecordRedisCmdLatencyDataPoint adds a data point to redis.cmd.latency metric. +func (mb *MetricsBuilder) RecordRedisCmdLatencyDataPoint(ts pcommon.Timestamp, val float64, cmdAttributeValue string, percentileAttributeValue AttributePercentile) { + mb.metricRedisCmdLatency.recordDataPoint(mb.startTime, ts, val, cmdAttributeValue, percentileAttributeValue.String()) +} + // RecordRedisCmdUsecDataPoint adds a data point to redis.cmd.usec metric. func (mb *MetricsBuilder) RecordRedisCmdUsecDataPoint(ts pcommon.Timestamp, val int64, cmdAttributeValue string) { mb.metricRedisCmdUsec.recordDataPoint(mb.startTime, ts, val, cmdAttributeValue) diff --git a/receiver/redisreceiver/internal/metadata/generated_metrics_test.go b/receiver/redisreceiver/internal/metadata/generated_metrics_test.go index 889d9c395250b..683a075347e3a 100644 --- a/receiver/redisreceiver/internal/metadata/generated_metrics_test.go +++ b/receiver/redisreceiver/internal/metadata/generated_metrics_test.go @@ -73,6 +73,9 @@ func TestMetricsBuilder(t *testing.T) { allMetricsCount++ mb.RecordRedisCmdCallsDataPoint(ts, 1, "cmd-val") + allMetricsCount++ + mb.RecordRedisCmdLatencyDataPoint(ts, 1, "cmd-val", AttributePercentileP50) + allMetricsCount++ mb.RecordRedisCmdUsecDataPoint(ts, 1, "cmd-val") @@ -275,6 +278,24 @@ func TestMetricsBuilder(t *testing.T) { attrVal, ok := dp.Attributes().Get("cmd") assert.True(t, ok) assert.EqualValues(t, "cmd-val", attrVal.Str()) + case "redis.cmd.latency": + assert.False(t, validatedMetrics["redis.cmd.latency"], "Found a duplicate in the metrics slice: redis.cmd.latency") + validatedMetrics["redis.cmd.latency"] = true + assert.Equal(t, pmetric.MetricTypeGauge, ms.At(i).Type()) + assert.Equal(t, 1, ms.At(i).Gauge().DataPoints().Len()) + assert.Equal(t, "Command execution latency", ms.At(i).Description()) + assert.Equal(t, "us", ms.At(i).Unit()) + dp := ms.At(i).Gauge().DataPoints().At(0) + assert.Equal(t, start, dp.StartTimestamp()) + assert.Equal(t, ts, dp.Timestamp()) + assert.Equal(t, pmetric.NumberDataPointValueTypeDouble, dp.ValueType()) + assert.Equal(t, float64(1), dp.DoubleValue()) + attrVal, ok := dp.Attributes().Get("cmd") + assert.True(t, ok) + assert.EqualValues(t, "cmd-val", attrVal.Str()) + attrVal, ok = dp.Attributes().Get("percentile") + assert.True(t, ok) + assert.EqualValues(t, "p50", attrVal.Str()) case "redis.cmd.usec": assert.False(t, validatedMetrics["redis.cmd.usec"], "Found a duplicate in the metrics slice: redis.cmd.usec") validatedMetrics["redis.cmd.usec"] = true diff --git a/receiver/redisreceiver/internal/metadata/testdata/config.yaml b/receiver/redisreceiver/internal/metadata/testdata/config.yaml index d546d3e12fe33..33972c5b607c9 100644 --- a/receiver/redisreceiver/internal/metadata/testdata/config.yaml +++ b/receiver/redisreceiver/internal/metadata/testdata/config.yaml @@ -11,6 +11,8 @@ all_set: enabled: true redis.cmd.calls: enabled: true + redis.cmd.latency: + enabled: true redis.cmd.usec: enabled: true redis.commands: @@ -82,6 +84,8 @@ none_set: enabled: false redis.cmd.calls: enabled: false + redis.cmd.latency: + enabled: false redis.cmd.usec: enabled: false redis.commands: diff --git a/receiver/redisreceiver/latencystats.go b/receiver/redisreceiver/latencystats.go new file mode 100644 index 0000000000000..fdfecc08a1b6a --- /dev/null +++ b/receiver/redisreceiver/latencystats.go @@ -0,0 +1,40 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package redisreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/redisreceiver" + +import ( + "fmt" + "strconv" + "strings" +) + +// Holds percentile latencies, e.g. "p99" -> 1.5. +type latencies map[string]float64 + +// parseLatencyStats parses the values part of one entry in Redis latencystats section, +// e.g. "p50=181.247,p99=309.247,p99.9=1023.999". +func parseLatencyStats(str string) (latencies, error) { + res := make(latencies) + + pairs := strings.Split(strings.TrimSpace(str), ",") + + for _, pairStr := range pairs { + pair := strings.Split(pairStr, "=") + if len(pair) != 2 { + return nil, fmt.Errorf("unexpected latency percentiles pair '%s'", pairStr) + } + + key := pair[0] + valueStr := pair[1] + + value, err := strconv.ParseFloat(valueStr, 64) + if err != nil { + return nil, err + } + + res[key] = value + } + + return res, nil +} diff --git a/receiver/redisreceiver/latencystats_test.go b/receiver/redisreceiver/latencystats_test.go new file mode 100644 index 0000000000000..31b7930a7e8fb --- /dev/null +++ b/receiver/redisreceiver/latencystats_test.go @@ -0,0 +1,34 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package redisreceiver + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseLatencyStats(t *testing.T) { + ls, err := parseLatencyStats("p50=181.247,p55=182.271,p99=309.247,p99.9=1023.999") + require.Nil(t, err) + require.Equal(t, ls["p50"], 181.247) + require.Equal(t, ls["p55"], 182.271) + require.Equal(t, ls["p99"], 309.247) + require.Equal(t, ls["p99.9"], 1023.999) +} + +func TestParseMalformedLatencyStats(t *testing.T) { + tests := []struct{ name, stats string }{ + {"missing value", "p50=42.0,p90=50.0,p99.9="}, + {"missing equals", "p50=42.0,p90=50.0,p99.9"}, + {"extra comma", "p50=42.0,,p90=50.0"}, + {"wrong value type", "p50=asdf"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + _, err := parseLatencyStats(test.stats) + require.NotNil(t, err) + }) + } +} diff --git a/receiver/redisreceiver/metadata.yaml b/receiver/redisreceiver/metadata.yaml index 64628fb3ca759..75e696aa7ac03 100644 --- a/receiver/redisreceiver/metadata.yaml +++ b/receiver/redisreceiver/metadata.yaml @@ -45,6 +45,13 @@ attributes: cmd: description: Redis command name type: string + percentile: + description: Percentile + type: string + enum: + - p50 + - p99 + - p99.9 metrics: redis.maxmemory: @@ -84,6 +91,14 @@ metrics: aggregation_temporality: cumulative attributes: [cmd] + redis.cmd.latency: + enabled: false + description: Command execution latency + unit: us + gauge: + value_type: double + attributes: [cmd, percentile] + redis.uptime: enabled: true description: Number of seconds since Redis server start diff --git a/receiver/redisreceiver/redis_scraper.go b/receiver/redisreceiver/redis_scraper.go index efe0a0c8e433a..8d242fca729db 100644 --- a/receiver/redisreceiver/redis_scraper.go +++ b/receiver/redisreceiver/redis_scraper.go @@ -93,7 +93,7 @@ func (rs *redisScraper) Scrape(context.Context) (pmetric.Metrics, error) { rs.recordCommonMetrics(now, inf) rs.recordKeyspaceMetrics(now, inf) rs.recordRoleMetrics(now, inf) - rs.recordCmdStatsMetrics(now, inf) + rs.recordCmdMetrics(now, inf) rb := rs.mb.NewResourceBuilder() rb.SetRedisVersion(rs.getRedisVersion(inf)) return rs.mb.Emit(metadata.WithResource(rb.Emit())), nil @@ -169,32 +169,58 @@ func (rs *redisScraper) recordRoleMetrics(ts pcommon.Timestamp, inf info) { } } -// recordCmdStatsMetrics records metrics from 'command_stats' Redis info key-value pairs -// e.g. "cmdstat_mget:calls=1685,usec=6032,usec_per_call=3.58,rejected_calls=0,failed_calls=0" -// but only calls and usec at the moment. -func (rs *redisScraper) recordCmdStatsMetrics(ts pcommon.Timestamp, inf info) { - cmdPrefix := "cmdstat_" +// recordCmdMetrics records per-command metrics from Redis info. +// These include command stats and command latency percentiles. +// Examples: +// +// "cmdstat_mget:calls=1685,usec=6032,usec_per_call=3.58,rejected_calls=0,failed_calls=0" +// "latency_percentiles_usec_lastsave:p50=1.003,p99=1.003,p99.9=1.003" +func (rs *redisScraper) recordCmdMetrics(ts pcommon.Timestamp, inf info) { + const cmdstatPrefix = "cmdstat_" + const latencyPrefix = "latency_percentiles_usec_" + for key, val := range inf { - if !strings.HasPrefix(key, cmdPrefix) { + if strings.HasPrefix(key, cmdstatPrefix) { + rs.recordCmdStatsMetrics(ts, key[len(cmdstatPrefix):], val) + } else if strings.HasPrefix(key, latencyPrefix) { + rs.recordCmdLatencyMetrics(ts, key[len(latencyPrefix):], val) + } + } +} + +// recordCmdStatsMetrics records metrics for a particlar Redis command. +// Only 'calls' and 'usec' are recorded at the moment. +// 'cmd' is the Redis command, 'val' is the values string (e.g. "calls=1685,usec=6032,usec_per_call=3.58,rejected_calls=0,failed_calls=0"). +func (rs *redisScraper) recordCmdStatsMetrics(ts pcommon.Timestamp, cmd, val string) { + parts := strings.Split(strings.TrimSpace(val), ",") + for _, element := range parts { + subParts := strings.Split(element, "=") + if len(subParts) == 1 { + continue + } + parsed, err := strconv.ParseInt(subParts[1], 10, 64) + if err != nil { // skip bad items continue } + if subParts[0] == "calls" { + rs.mb.RecordRedisCmdCallsDataPoint(ts, parsed, cmd) + } else if subParts[0] == "usec" { + rs.mb.RecordRedisCmdUsecDataPoint(ts, parsed, cmd) + } + } +} - cmd := key[len(cmdPrefix):] - parts := strings.Split(strings.TrimSpace(val), ",") - for _, element := range parts { - subParts := strings.Split(element, "=") - if len(subParts) == 1 { - continue - } - parsed, err := strconv.ParseInt(subParts[1], 10, 64) - if err != nil { // skip bad items - continue - } - if subParts[0] == "calls" { - rs.mb.RecordRedisCmdCallsDataPoint(ts, parsed, cmd) - } else if subParts[0] == "usec" { - rs.mb.RecordRedisCmdUsecDataPoint(ts, parsed, cmd) - } +// recordCmdLatencyMetrics record latency metrics of a particular Redis command. +// 'cmd' is the Redis command, 'val' is the values string (e.g. "p50=1.003,p99=1.003,p99.9=1.003). +func (rs *redisScraper) recordCmdLatencyMetrics(ts pcommon.Timestamp, cmd, val string) { + latencies, err := parseLatencyStats(val) + if err != nil { + return + } + + for percentile, latency := range latencies { + if percentileAttr, ok := metadata.MapAttributePercentile[percentile]; ok { + rs.mb.RecordRedisCmdLatencyDataPoint(ts, latency, cmd, percentileAttr) } } }