From cf5e9c7ef223ed7b2e268909dcb9cf1af22439c3 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Tue, 25 Jan 2022 15:06:19 +0200 Subject: [PATCH 01/12] Add memExemplar in stripeSeries as first iteration Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/series.go | 49 ++++++++++++++++++++++++++++++++------- pkg/metrics/wal/wal.go | 22 ++++++++++++++++++ 2 files changed, 63 insertions(+), 8 deletions(-) diff --git a/pkg/metrics/wal/series.go b/pkg/metrics/wal/series.go index fd1de2effd9b..d38d24d49340 100644 --- a/pkg/metrics/wal/series.go +++ b/pkg/metrics/wal/series.go @@ -84,6 +84,12 @@ func (m seriesHashmap) del(hash uint64, ref uint64) { } } +type memExemplar struct { + ts int64 + value float64 + labels labels.Labels +} + const ( // defaultStripeSize is the default number of entries to allocate in the // stripeSeries hash map. @@ -97,10 +103,11 @@ const ( // // This code is copied from the Prometheus TSDB. type stripeSeries struct { - size int - series []map[uint64]*memSeries - hashes []seriesHashmap - locks []stripeLock + size int + series []map[uint64]*memSeries + hashes []seriesHashmap + exemplars []map[uint64]*memExemplar + locks []stripeLock } type stripeLock struct { @@ -112,10 +119,11 @@ type stripeLock struct { func newStripeSeries() *stripeSeries { stripeSize := defaultStripeSize s := &stripeSeries{ - size: stripeSize, - series: make([]map[uint64]*memSeries, stripeSize), - hashes: make([]seriesHashmap, stripeSize), - locks: make([]stripeLock, stripeSize), + size: stripeSize, + series: make([]map[uint64]*memSeries, stripeSize), + hashes: make([]seriesHashmap, stripeSize), + exemplars: make([]map[uint64]*memExemplar, stripeSize), + locks: make([]stripeLock, stripeSize), } for i := range s.series { @@ -124,6 +132,9 @@ func newStripeSeries() *stripeSeries { for i := range s.hashes { s.hashes[i] = seriesHashmap{} } + for i := range s.exemplars { + s.exemplars[i] = map[uint64]*memExemplar{} + } return s } @@ -171,6 +182,10 @@ func (s *stripeSeries) gc(mint int64) map[uint64]struct{} { delete(s.series[i], series.ref) s.hashes[j].del(seriesHash, series.ref) + // Since the series is gone, we'll also delete + // the latest stored exemplar. + delete(s.exemplars[i], series.ref) + if i != j { s.locks[j].Unlock() } @@ -216,6 +231,24 @@ func (s *stripeSeries) set(hash uint64, series *memSeries) { s.locks[i].Unlock() } +func (s *stripeSeries) getLatestExemplar(id uint64) *memExemplar { + i := id & uint64(s.size-1) + + s.locks[i].RLock() + exemplar := s.exemplars[i][id] + s.locks[i].RUnlock() + + return exemplar +} + +func (s *stripeSeries) setLatestExemplar(id uint64, exemplar *memExemplar) { + i := id & uint64(s.size-1) + + s.locks[i].Lock() + s.exemplars[i][id] = exemplar + s.locks[i].Unlock() +} + func (s *stripeSeries) iterator() *stripeSeriesIterator { return &stripeSeriesIterator{s} } diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go index b5e8c3b7372a..6e0caa456795 100644 --- a/pkg/metrics/wal/wal.go +++ b/pkg/metrics/wal/wal.go @@ -646,6 +646,16 @@ func (a *appender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Exempl } } + // Check for duplicate vs last stored exemplar for this series, and discard those. + // Otherwise, record the current exemplar as the latest + prevExemplar := a.w.series.getLatestExemplar(ref) + if prevExemplar != nil { + if exemplarsEqual(*prevExemplar, e) { + return 0, nil + } + a.w.series.setLatestExemplar(ref, &memExemplar{ts: e.Ts, value: e.Value, labels: e.Labels}) + } + a.exemplars = append(a.exemplars, record.RefExemplar{ Ref: ref, T: e.Ts, @@ -714,3 +724,15 @@ func (a *appender) Rollback() error { a.w.appenderPool.Put(a) return nil } + +func exemplarsEqual(me memExemplar, pe exemplar.Exemplar) bool { + if !labels.Equal(me.labels, pe.Labels) { + return false + } + + if me.ts != pe.Ts { + return false + } + + return me.value == pe.Value +} From 172a2936f6eaabdfad731c22b74d846c6942f403 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Tue, 25 Jan 2022 17:23:19 +0200 Subject: [PATCH 02/12] Add test for skipped duplicate exemplars; Simplify conditional Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/wal.go | 8 +++----- pkg/metrics/wal/wal_test.go | 41 +++++++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go index 6e0caa456795..0c7fcdd69dec 100644 --- a/pkg/metrics/wal/wal.go +++ b/pkg/metrics/wal/wal.go @@ -649,12 +649,10 @@ func (a *appender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Exempl // Check for duplicate vs last stored exemplar for this series, and discard those. // Otherwise, record the current exemplar as the latest prevExemplar := a.w.series.getLatestExemplar(ref) - if prevExemplar != nil { - if exemplarsEqual(*prevExemplar, e) { - return 0, nil - } - a.w.series.setLatestExemplar(ref, &memExemplar{ts: e.Ts, value: e.Value, labels: e.Labels}) + if prevExemplar != nil && exemplarsEqual(*prevExemplar, e) { + return 0, nil } + a.w.series.setLatestExemplar(ref, &memExemplar{ts: e.Ts, value: e.Value, labels: e.Labels}) a.exemplars = append(a.exemplars, record.RefExemplar{ Ref: ref, diff --git a/pkg/metrics/wal/wal_test.go b/pkg/metrics/wal/wal_test.go index fccc0a023aa5..61bd894dfa4f 100644 --- a/pkg/metrics/wal/wal_test.go +++ b/pkg/metrics/wal/wal_test.go @@ -104,6 +104,47 @@ func TestStorage(t *testing.T) { require.Equal(t, expectedExemplars, actualExemplars) } +func TestStorage_DuplicateExemplarsIgnored(t *testing.T) { + walDir, err := ioutil.TempDir(os.TempDir(), "wal") + require.NoError(t, err) + defer os.RemoveAll(walDir) + + s, err := NewStorage(log.NewNopLogger(), nil, walDir) + require.NoError(t, err) + + app := s.Appender(context.Background()) + + sRef, err := app.Append(0, labels.Labels{{Name: "a", Value: "1"}}, 0, 0) + require.NoError(t, err, "should not reject valid series") + + // If the Labels, Value or Timestamp are different than the last exemplar, + // then a new one should be appended; Otherwise, it should be skipped. + e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true} + app.AppendExemplar(sRef, nil, e) + app.AppendExemplar(sRef, nil, e) + + e.Labels = labels.Labels{{Name: "b", Value: "2"}} + app.AppendExemplar(sRef, nil, e) + app.AppendExemplar(sRef, nil, e) + app.AppendExemplar(sRef, nil, e) + + e.Value = 42 + app.AppendExemplar(sRef, nil, e) + app.AppendExemplar(sRef, nil, e) + + e.Ts = 25 + app.AppendExemplar(sRef, nil, e) + app.AppendExemplar(sRef, nil, e) + + require.NoError(t, app.Commit()) + collector := walDataCollector{} + replayer := walReplayer{w: &collector} + require.NoError(t, replayer.Replay(s.wal.Dir())) + + // We had 9 calls to AppendExemplar but only 4 of them should have been appended + require.Equal(t, 4, len(collector.exemplars)) +} + func TestStorage_ExistingWAL(t *testing.T) { walDir, err := ioutil.TempDir(os.TempDir(), "wal") require.NoError(t, err) From a3aac1d4115088146599c3318d2a61f3df983d22 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Wed, 26 Jan 2022 11:33:55 +0200 Subject: [PATCH 03/12] Add changelog entry; discard test errors --- CHANGELOG.md | 2 ++ pkg/metrics/wal/wal_test.go | 20 ++++++++++---------- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index b198119135ca..73fd778b9e01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ - [BUGFIX] Operator: Add missing proxy_url field from generated remote_write configs. (@rfratto) +- [ENHANCEMENT] The agent no longer appends duplicate exemplars. (@tpaschalis) + # v0.22.0 (2022-01-13) This release has deprecations. Please read [DEPRECATION] entries and consult diff --git a/pkg/metrics/wal/wal_test.go b/pkg/metrics/wal/wal_test.go index 61bd894dfa4f..c66bbb9a5874 100644 --- a/pkg/metrics/wal/wal_test.go +++ b/pkg/metrics/wal/wal_test.go @@ -120,28 +120,28 @@ func TestStorage_DuplicateExemplarsIgnored(t *testing.T) { // If the Labels, Value or Timestamp are different than the last exemplar, // then a new one should be appended; Otherwise, it should be skipped. e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true} - app.AppendExemplar(sRef, nil, e) - app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) e.Labels = labels.Labels{{Name: "b", Value: "2"}} - app.AppendExemplar(sRef, nil, e) - app.AppendExemplar(sRef, nil, e) - app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) e.Value = 42 - app.AppendExemplar(sRef, nil, e) - app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) e.Ts = 25 - app.AppendExemplar(sRef, nil, e) - app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) + _, _ = app.AppendExemplar(sRef, nil, e) require.NoError(t, app.Commit()) collector := walDataCollector{} replayer := walReplayer{w: &collector} require.NoError(t, replayer.Replay(s.wal.Dir())) - // We had 9 calls to AppendExemplar but only 4 of them should have been appended + // We had 9 calls to AppendExemplar but only 4 of those should have gotten through require.Equal(t, 4, len(collector.exemplars)) } From 38f14d665b4416c7eee0051880ec0cf0537cc35b Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Wed, 26 Jan 2022 16:38:10 +0200 Subject: [PATCH 04/12] Move changelog entry Signed-off-by: Paschalis Tsilias --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 134017479623..ecf3f2270349 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ - [ENHANCEMENT] integrations-next: Add `extra_labels` to add a custom set of labels to integration targets. (@rfratto) +- [ENHANCEMENT] The agent no longer appends duplicate exemplars. (@tpaschalis) + - [BUGFIX] Fixed issue where Grafana Agent may panic if there is a very large WAL loading while old WALs are being deleted or the `/agent/api/v1/targets` endpoint is called. (@tpaschalis) @@ -16,8 +18,6 @@ - [BUGFIX] Honor the specified log format in the traces subsystem (@mapno) -- [ENHANCEMENT] The agent no longer appends duplicate exemplars. (@tpaschalis) - # v0.22.0 (2022-01-13) This release has deprecations. Please read [DEPRECATION] entries and consult From 08faa07d4ff7c2ae25f3e3152f5b38a814cfb3b8 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Wed, 26 Jan 2022 18:10:22 +0200 Subject: [PATCH 05/12] Add Benchmark for AppendExemplar Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/wal_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pkg/metrics/wal/wal_test.go b/pkg/metrics/wal/wal_test.go index c66bbb9a5874..0373878cd9ad 100644 --- a/pkg/metrics/wal/wal_test.go +++ b/pkg/metrics/wal/wal_test.go @@ -372,6 +372,24 @@ func TestStorage_TruncateAfterClose(t *testing.T) { require.Error(t, ErrWALClosed, s.Truncate(0)) } +func BenchmarkAppendExemplar(b *testing.B) { + walDir, _ := ioutil.TempDir(os.TempDir(), "wal") + defer os.RemoveAll(walDir) + + s, _ := NewStorage(log.NewNopLogger(), nil, walDir) + app := s.Appender(context.Background()) + sRef, _ := app.Append(0, labels.Labels{{Name: "a", Value: "1"}}, 0, 0) + + e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true} + for i := 0; i < b.N; i++ { + e.Ts = int64(i) + _, _ = app.AppendExemplar(sRef, nil, e) + } + + // Actually use appended exemplars in case they get eliminated + app.Commit() +} + type sample struct { ts int64 val float64 From a1e4044216023a3a2640b55d13e6a9c79a160d13 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Wed, 26 Jan 2022 18:16:29 +0200 Subject: [PATCH 06/12] Discard error on added benchmark Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/wal_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/wal/wal_test.go b/pkg/metrics/wal/wal_test.go index 0373878cd9ad..1692608b3bf8 100644 --- a/pkg/metrics/wal/wal_test.go +++ b/pkg/metrics/wal/wal_test.go @@ -387,7 +387,7 @@ func BenchmarkAppendExemplar(b *testing.B) { } // Actually use appended exemplars in case they get eliminated - app.Commit() + _ = app.Commit() } type sample struct { From 249600d2d892e5b3ba788de4284e37e4f4a59c64 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Thu, 27 Jan 2022 10:05:50 +0200 Subject: [PATCH 07/12] Use original exemplar struct instead of custom memExemplar Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/series.go | 17 ++++++----------- pkg/metrics/wal/wal.go | 18 ++++-------------- 2 files changed, 10 insertions(+), 25 deletions(-) diff --git a/pkg/metrics/wal/series.go b/pkg/metrics/wal/series.go index d38d24d49340..4ec3c8170111 100644 --- a/pkg/metrics/wal/series.go +++ b/pkg/metrics/wal/series.go @@ -3,6 +3,7 @@ package wal import ( "sync" + "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/intern" "github.com/prometheus/prometheus/pkg/labels" ) @@ -84,12 +85,6 @@ func (m seriesHashmap) del(hash uint64, ref uint64) { } } -type memExemplar struct { - ts int64 - value float64 - labels labels.Labels -} - const ( // defaultStripeSize is the default number of entries to allocate in the // stripeSeries hash map. @@ -106,7 +101,7 @@ type stripeSeries struct { size int series []map[uint64]*memSeries hashes []seriesHashmap - exemplars []map[uint64]*memExemplar + exemplars []map[uint64]*exemplar.Exemplar locks []stripeLock } @@ -122,7 +117,7 @@ func newStripeSeries() *stripeSeries { size: stripeSize, series: make([]map[uint64]*memSeries, stripeSize), hashes: make([]seriesHashmap, stripeSize), - exemplars: make([]map[uint64]*memExemplar, stripeSize), + exemplars: make([]map[uint64]*exemplar.Exemplar, stripeSize), locks: make([]stripeLock, stripeSize), } @@ -133,7 +128,7 @@ func newStripeSeries() *stripeSeries { s.hashes[i] = seriesHashmap{} } for i := range s.exemplars { - s.exemplars[i] = map[uint64]*memExemplar{} + s.exemplars[i] = map[uint64]*exemplar.Exemplar{} } return s } @@ -231,7 +226,7 @@ func (s *stripeSeries) set(hash uint64, series *memSeries) { s.locks[i].Unlock() } -func (s *stripeSeries) getLatestExemplar(id uint64) *memExemplar { +func (s *stripeSeries) getLatestExemplar(id uint64) *exemplar.Exemplar { i := id & uint64(s.size-1) s.locks[i].RLock() @@ -241,7 +236,7 @@ func (s *stripeSeries) getLatestExemplar(id uint64) *memExemplar { return exemplar } -func (s *stripeSeries) setLatestExemplar(id uint64, exemplar *memExemplar) { +func (s *stripeSeries) setLatestExemplar(id uint64, exemplar *exemplar.Exemplar) { i := id & uint64(s.size-1) s.locks[i].Lock() diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go index 0c7fcdd69dec..ae3936776470 100644 --- a/pkg/metrics/wal/wal.go +++ b/pkg/metrics/wal/wal.go @@ -648,11 +648,13 @@ func (a *appender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Exempl // Check for duplicate vs last stored exemplar for this series, and discard those. // Otherwise, record the current exemplar as the latest + // Prometheus returns 0 when encountering duplicates, so we do the same here. prevExemplar := a.w.series.getLatestExemplar(ref) - if prevExemplar != nil && exemplarsEqual(*prevExemplar, e) { + if prevExemplar != nil && prevExemplar.Equals(e) { + // Duplicate, don't return an error but don't accept the exemplar. return 0, nil } - a.w.series.setLatestExemplar(ref, &memExemplar{ts: e.Ts, value: e.Value, labels: e.Labels}) + a.w.series.setLatestExemplar(ref, &e) a.exemplars = append(a.exemplars, record.RefExemplar{ Ref: ref, @@ -722,15 +724,3 @@ func (a *appender) Rollback() error { a.w.appenderPool.Put(a) return nil } - -func exemplarsEqual(me memExemplar, pe exemplar.Exemplar) bool { - if !labels.Equal(me.labels, pe.Labels) { - return false - } - - if me.ts != pe.Ts { - return false - } - - return me.value == pe.Value -} From a2aff45c39793c48a790db4a822b5afcc3d31c43 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Thu, 27 Jan 2022 10:27:30 +0200 Subject: [PATCH 08/12] Surround benchmark loop with start/stop timers and close test storage Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/wal.go | 2 +- pkg/metrics/wal/wal_test.go | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go index ae3936776470..9eb5f57c4f26 100644 --- a/pkg/metrics/wal/wal.go +++ b/pkg/metrics/wal/wal.go @@ -647,7 +647,7 @@ func (a *appender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Exempl } // Check for duplicate vs last stored exemplar for this series, and discard those. - // Otherwise, record the current exemplar as the latest + // Otherwise, record the current exemplar as the latest. // Prometheus returns 0 when encountering duplicates, so we do the same here. prevExemplar := a.w.series.getLatestExemplar(ref) if prevExemplar != nil && prevExemplar.Equals(e) { diff --git a/pkg/metrics/wal/wal_test.go b/pkg/metrics/wal/wal_test.go index 1692608b3bf8..dbbd0f9cab7b 100644 --- a/pkg/metrics/wal/wal_test.go +++ b/pkg/metrics/wal/wal_test.go @@ -377,14 +377,17 @@ func BenchmarkAppendExemplar(b *testing.B) { defer os.RemoveAll(walDir) s, _ := NewStorage(log.NewNopLogger(), nil, walDir) + defer s.Close() app := s.Appender(context.Background()) sRef, _ := app.Append(0, labels.Labels{{Name: "a", Value: "1"}}, 0, 0) + b.StartTimer() e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true} for i := 0; i < b.N; i++ { e.Ts = int64(i) _, _ = app.AppendExemplar(sRef, nil, e) } + b.StopTimer() // Actually use appended exemplars in case they get eliminated _ = app.Commit() From 47eaa4a1d9812cdd4db28ba784373ab263ad92e5 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Thu, 27 Jan 2022 10:36:45 +0200 Subject: [PATCH 09/12] Add comment about prepopulating exemplars on WAL startup Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/wal.go | 2 ++ pkg/metrics/wal/wal_test.go | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go index 9eb5f57c4f26..d203f451fcbe 100644 --- a/pkg/metrics/wal/wal.go +++ b/pkg/metrics/wal/wal.go @@ -282,6 +282,8 @@ func (w *Storage) loadWAL(r *wal.Reader) (err error) { decoded <- samples case record.Tombstones, record.Exemplars: // We don't care about decoding tombstones or exemplars + // If decide to decode exemplars, we should make sure to prepopulate + // stripeSeries.exemplars in the next block by using setLatestExemplar. continue default: errCh <- &wal.CorruptionErr{ diff --git a/pkg/metrics/wal/wal_test.go b/pkg/metrics/wal/wal_test.go index dbbd0f9cab7b..a1cad5c4f86d 100644 --- a/pkg/metrics/wal/wal_test.go +++ b/pkg/metrics/wal/wal_test.go @@ -380,9 +380,9 @@ func BenchmarkAppendExemplar(b *testing.B) { defer s.Close() app := s.Appender(context.Background()) sRef, _ := app.Append(0, labels.Labels{{Name: "a", Value: "1"}}, 0, 0) + e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true} b.StartTimer() - e := exemplar.Exemplar{Labels: labels.Labels{{Name: "a", Value: "1"}}, Value: 20, Ts: 10, HasTs: true} for i := 0; i < b.N; i++ { e.Ts = int64(i) _, _ = app.AppendExemplar(sRef, nil, e) From 82a53587bf6732e6020426f87562ded9c352cb47 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Thu, 27 Jan 2022 10:58:32 +0200 Subject: [PATCH 10/12] Wire in the totalAppendedExemplars metric Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/wal.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go index d203f451fcbe..bab9b8ace3c8 100644 --- a/pkg/metrics/wal/wal.go +++ b/pkg/metrics/wal/wal.go @@ -665,6 +665,7 @@ func (a *appender) AppendExemplar(ref uint64, _ labels.Labels, e exemplar.Exempl Labels: e.Labels, }) + a.w.metrics.totalAppendedExemplars.Inc() return s.ref, nil } From 1d602ccfc2383aa5556c31f084f2245497921000 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Thu, 27 Jan 2022 18:33:53 +0200 Subject: [PATCH 11/12] Make comment more discoverable Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/wal.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/metrics/wal/wal.go b/pkg/metrics/wal/wal.go index bab9b8ace3c8..15814d070739 100644 --- a/pkg/metrics/wal/wal.go +++ b/pkg/metrics/wal/wal.go @@ -282,7 +282,7 @@ func (w *Storage) loadWAL(r *wal.Reader) (err error) { decoded <- samples case record.Tombstones, record.Exemplars: // We don't care about decoding tombstones or exemplars - // If decide to decode exemplars, we should make sure to prepopulate + // TODO: If decide to decode exemplars, we should make sure to prepopulate // stripeSeries.exemplars in the next block by using setLatestExemplar. continue default: From 09d62a22c0740bdb2d87416d06173fddccd00a76 Mon Sep 17 00:00:00 2001 From: Paschalis Tsilias Date: Thu, 27 Jan 2022 18:38:48 +0200 Subject: [PATCH 12/12] Make sure we're recording exemplars for non-nil series ref only Signed-off-by: Paschalis Tsilias --- pkg/metrics/wal/series.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pkg/metrics/wal/series.go b/pkg/metrics/wal/series.go index 4ec3c8170111..35bda48e2974 100644 --- a/pkg/metrics/wal/series.go +++ b/pkg/metrics/wal/series.go @@ -239,8 +239,11 @@ func (s *stripeSeries) getLatestExemplar(id uint64) *exemplar.Exemplar { func (s *stripeSeries) setLatestExemplar(id uint64, exemplar *exemplar.Exemplar) { i := id & uint64(s.size-1) + // Make sure that's a valid series id and record its latest exemplar s.locks[i].Lock() - s.exemplars[i][id] = exemplar + if s.series[i][id] != nil { + s.exemplars[i][id] = exemplar + } s.locks[i].Unlock() }