From 6310e5928579e6408b5b2d05f47e4e3061fee92f Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Tue, 27 Sep 2022 09:23:05 -0400 Subject: [PATCH 1/2] cmd/tempo-vulture: share rand.Rand instance across multiple searches grafana/tempo#1760 noted that creating a new rand.Rand every time a random number is generated causes a poor distrubution of the search space, where numbers appear 7x more frequently than they do when using a shared rand.Rand instance. This is demonstrated using the following Go playground link: https://go.dev/play/p/EkhINRfOO5p This commit changes Vulture to use a shared rand.Rand instance instead of creating a new one each time a search or read is performed. --- cmd/tempo-vulture/main.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/tempo-vulture/main.go b/cmd/tempo-vulture/main.go index a985523abac..863fcccf43e 100644 --- a/cmd/tempo-vulture/main.go +++ b/cmd/tempo-vulture/main.go @@ -82,6 +82,8 @@ func main() { startTime := actualStartTime tickerWrite := time.NewTicker(tempoWriteBackoffDuration) + r := rand.New(rand.NewSource(actualStartTime.Unix())) + var tickerRead *time.Ticker if tempoReadBackoffDuration > 0 { tickerRead = time.NewTicker(tempoReadBackoffDuration) @@ -140,7 +142,7 @@ func main() { go func() { for now := range tickerRead.C { var seed time.Time - startTime, seed = selectPastTimestamp(startTime, now, interval, tempoRetentionDuration) + startTime, seed = selectPastTimestamp(startTime, now, interval, tempoRetentionDuration, r) log := logger.With( zap.String("org_id", tempoOrgID), @@ -173,7 +175,7 @@ func main() { if tickerSearch != nil { go func() { for now := range tickerSearch.C { - _, seed := selectPastTimestamp(startTime, now, interval, tempoRetentionDuration) + _, seed := selectPastTimestamp(startTime, now, interval, tempoRetentionDuration, r) log := logger.With( zap.String("org_id", tempoOrgID), zap.Int64("seed", seed.Unix()), @@ -251,7 +253,7 @@ func pushMetrics(metrics traceMetrics) { metricTracesErrors.WithLabelValues("notfound_search_attribute").Add(float64(metrics.notFoundSearchAttribute)) } -func selectPastTimestamp(start, stop time.Time, interval time.Duration, retention time.Duration) (newStart, ts time.Time) { +func selectPastTimestamp(start, stop time.Time, interval, retention time.Duration, r *rand.Rand) (newStart, ts time.Time) { oldest := stop.Add(-retention) if oldest.After(start) { @@ -260,7 +262,7 @@ func selectPastTimestamp(start, stop time.Time, interval time.Duration, retentio newStart = start } - ts = time.Unix(generateRandomInt(newStart.Unix(), stop.Unix(), rand.New(rand.NewSource(start.Unix()))), 0) + ts = time.Unix(generateRandomInt(newStart.Unix(), stop.Unix(), r), 0) return newStart.Round(interval), ts.Round(interval) } From 03402bf0409bc3e44725962c694e58d24ced11d6 Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Tue, 27 Sep 2022 09:27:26 -0400 Subject: [PATCH 2/2] update CHANGELOG --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cc0302ace1d..90bb6225387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,7 +6,7 @@ * [CHANGE] Make DNS address fully qualified to reduce DNS lookups in Kubernetes [#1687](https://github.com/grafana/tempo/pull/1687) (@electron0zero) * [CHANGE] Improve parquet compaction memory profile when dropping spans [#1692](https://github.com/grafana/tempo/pull/1692) (@joe-elliott) * [CHANGE] Increase default values for `server.grpc_server_max_recv_msg_size` and `server.grpc_server_max_send_msg_size` from 4MB to 16MB [#1688](https://github.com/grafana/tempo/pull/1688) (@mapno) -* [CHANGE] Use Parquet for local block search, tag search and tag value search instead of flatbuffers. A configuration value +* [CHANGE] Use Parquet for local block search, tag search and tag value search instead of flatbuffers. A configuration value (`ingester.use_flatbuffer_search`) is provided to continue using flatbuffers. - **BREAKING CHANGE** Makes Parquet the default encoding. * [CHANGE] **BREAKING CHANGE** Use storage.trace.block.row_group_size_bytes to cut rows during compaction instead of @@ -33,6 +33,7 @@ query_frontend: hedge_requests_at: 5s hedge_requests_up_to: 3 ``` +* [ENHANCEMENT] Vulture now has improved distribution of the random traces it searches. [#1763](https://github.com/grafana/tempo/pull/1763) (@rfratto) * [BUGFIX] Honor caching and buffering settings when finding traces by id [#1697](https://github.com/grafana/tempo/pull/1697) (@joe-elliott) * [BUGFIX] Correctly propagate errors from the iterator layer up through the queriers [#1723](https://github.com/grafana/tempo/pull/1723) (@joe-elliott)