From 91392006897514cbe210c8b617bb1099036ce021 Mon Sep 17 00:00:00 2001 From: Janos Date: Fri, 11 Feb 2022 15:32:47 +0100 Subject: [PATCH 01/91] Initial implementation --- pkg/ingester/active_series.go | 19 +++- pkg/ingester/ingester.go | 148 +++++++++++++++++++++++-------- pkg/ingester/metrics.go | 10 +-- pkg/ingester/runtime_matchers.go | 21 +++++ pkg/mimir/modules.go | 1 + pkg/mimir/runtime_config.go | 16 ++++ 6 files changed, 170 insertions(+), 45 deletions(-) create mode 100644 pkg/ingester/runtime_matchers.go diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index f187466f5a..8f3cc35855 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -22,8 +22,9 @@ const ( // ActiveSeries is keeping track of recently active series for a single tenant. type ActiveSeries struct { - asm *ActiveSeriesMatchers - stripes [numActiveSeriesStripes]activeSeriesStripe + asm *ActiveSeriesMatchers + stripes [numActiveSeriesStripes]activeSeriesStripe + lastUpdate time.Time } // activeSeriesStripe holds a subset of the series timestamps for a single tenant. @@ -63,6 +64,20 @@ func NewActiveSeries(asm *ActiveSeriesMatchers) *ActiveSeries { return c } +func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers) { + c.asm = asm + for i := 0; i < numActiveSeriesStripes; i++ { + c.stripes[i].mu.Lock() + defer c.stripes[i].mu.Unlock() + } + + for i := 0; i < numActiveSeriesStripes; i++ { + c.stripes[i].asm = asm + c.stripes[i].activeMatching = makeIntSliceIfNotEmpty(len(asm.MatcherNames())) + } + c.lastUpdate = time.Now() +} + // Updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. func (c *ActiveSeries) UpdateSeries(series labels.Labels, now time.Time, labelsCopy func(labels.Labels) labels.Labels) { fp := fingerprint(series) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index e210949ee1..0845a83142 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -16,6 +16,7 @@ import ( "net/http" "os" "path/filepath" + "reflect" "strings" "sync" "time" @@ -121,10 +122,10 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackers ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)."` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + RuntimeMatchersConfigFn func() *RuntimeMatchersConfig `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -153,7 +154,6 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.ActiveSeriesMetricsEnabled, "ingester.active-series-metrics-enabled", true, "Enable tracking of active series and export them as metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsUpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsIdleTimeout, "ingester.active-series-metrics-idle-timeout", 10*time.Minute, "After what time a series is considered to be inactive.") - f.Var(&cfg.ActiveSeriesCustomTrackers, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag.") f.BoolVar(&cfg.StreamChunksWhenUsingBlocks, "ingester.stream-chunks-when-using-blocks", true, "Stream chunks from ingesters to queriers.") f.DurationVar(&cfg.ExemplarsUpdatePeriod, "ingester.exemplars-update-period", 15*time.Second, "Period with which to update per-user max exemplars.") @@ -198,8 +198,6 @@ type Ingester struct { metrics *ingesterMetrics logger log.Logger - activeSeriesMatcher *ActiveSeriesMatchers - lifecycler *ring.Lifecycler limits *validation.Overrides limiter *Limiter @@ -237,6 +235,9 @@ type Ingester struct { // Rate of pushed samples. Used to limit global samples push rate. ingestionRate *util_math.EwmaRate inflightPushRequests atomic.Int64 + + // Last seen activeSeriesConfiguration. Need for diff based updating + runtimeMatchersConfig *RuntimeMatchersConfig } func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { @@ -250,18 +251,18 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus } return &Ingester{ - cfg: cfg, - limits: limits, - logger: logger, - activeSeriesMatcher: &ActiveSeriesMatchers{}, - - tsdbs: make(map[string]*userTSDB), - usersMetadata: make(map[string]*userMetricsMetadata), - bucket: bucketClient, - tsdbMetrics: newTSDBMetrics(registerer), - forceCompactTrigger: make(chan requestWithUsersAndCallback), - shipTrigger: make(chan requestWithUsersAndCallback), - seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), + cfg: cfg, + limits: limits, + logger: logger, + + tsdbs: make(map[string]*userTSDB), + usersMetadata: make(map[string]*userMetricsMetadata), + bucket: bucketClient, + tsdbMetrics: newTSDBMetrics(registerer), + forceCompactTrigger: make(chan requestWithUsersAndCallback), + shipTrigger: make(chan requestWithUsersAndCallback), + seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), + runtimeMatchersConfig: cfg.RuntimeMatchersConfigFn(), }, nil } @@ -279,13 +280,7 @@ func New(cfg Config, clientConfig client.Config, limits *validation.Overrides, r } i.clientConfig = clientConfig i.ingestionRate = util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval) - i.metrics = newIngesterMetrics(registerer, cfg.ActiveSeriesMetricsEnabled, i.activeSeriesMatcher.MatcherNames(), i.getInstanceLimits, i.ingestionRate, &i.inflightPushRequests) - - asm, err := NewActiveSeriesMatchers(cfg.ActiveSeriesCustomTrackers) - if err != nil { - return nil, err - } - i.activeSeriesMatcher = asm + i.metrics = newIngesterMetrics(registerer, cfg.ActiveSeriesMetricsEnabled, i.getInstanceLimits, i.ingestionRate, &i.inflightPushRequests) // Replace specific metrics which we can't directly track but we need to read // them from the underlying system (ie. TSDB). @@ -333,7 +328,7 @@ func NewForFlusher(cfg Config, limits *validation.Overrides, registerer promethe if err != nil { return nil, err } - i.metrics = newIngesterMetrics(registerer, false, nil, i.getInstanceLimits, nil, &i.inflightPushRequests) + i.metrics = newIngesterMetrics(registerer, false, i.getInstanceLimits, nil, &i.inflightPushRequests) i.shipperIngesterID = "flusher" @@ -465,6 +460,7 @@ func (i *Ingester) updateLoop(ctx context.Context) error { i.applyExemplarsSettings() case <-activeSeriesTickerChan: + i.reloadConfig(time.Now()) i.updateActiveSeries(time.Now()) case <-ctx.Done(): @@ -475,6 +471,70 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } +func (i *Ingester) reloadConfig(now time.Time) { + newConfig := i.cfg.RuntimeMatchersConfigFn() + // it is crucial to only reload matchers which have been changed, as this function runs even if there is no change in config + // first check the generic matchers + if i.runtimeMatchersConfig.GenericMatchers.String() != newConfig.GenericMatchers.String() { + // need to replace everything + for _, userID := range i.getTSDBUsers() { + userDB := i.getTSDB(userID) + if userDB == nil { + continue + } + err := i.ReplaceMatchers(newConfig, userID, userDB) + if err != nil { + level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) + } + } + } else { + // for each users check if the config has changed + for _, userID := range i.getTSDBUsers() { + userDB := i.getTSDB(userID) + if userDB == nil { + continue + } + oldValue, oldOk := i.runtimeMatchersConfig.TenantSpecificMatchers[userID] + newValue, newOk := newConfig.TenantSpecificMatchers[userID] + if oldOk != newOk || !reflect.DeepEqual(oldValue, newValue) { + err := i.ReplaceMatchers(newConfig, userID, userDB) + if err != nil { + level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) + } + } + } + } + + i.runtimeMatchersConfig = newConfig +} + +func (i *Ingester) ReplaceMatchers(newConfig *RuntimeMatchersConfig, userID string, userDB *userTSDB) error { + newTrackerConfig, err := createCustomTrackerConfig(newConfig, userID) + if err != nil { + return err + } + newMatcher, err := NewActiveSeriesMatchers(newTrackerConfig) + if err != nil { + return err + } + i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.asm.names) + userDB.activeSeries.ReloadSeriesMatchers(newMatcher) + return nil +} + +func createCustomTrackerConfig(runtimeConfig *RuntimeMatchersConfig, userID string) (ActiveSeriesCustomTrackersConfig, error) { + userPart := "" + if val, ok := runtimeConfig.TenantSpecificMatchers[userID]; ok { + userPart = val.String() + } else if defaultVal, ok := runtimeConfig.TenantSpecificMatchers["default"]; ok { + userPart = defaultVal.String() + } + desiredConfig := runtimeConfig.GenericMatchers.String() + ";" + userPart + asc := &ActiveSeriesCustomTrackersConfig{} + err := asc.Set(desiredConfig) + return *asc, err +} + func (i *Ingester) updateActiveSeries(now time.Time) { purgeTime := now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout) for _, userID := range i.getTSDBUsers() { @@ -490,14 +550,19 @@ func (i *Ingester) updateActiveSeries(now time.Time) { } else { i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) } - for idx, name := range i.activeSeriesMatcher.MatcherNames() { - // We only set the metrics for matchers that actually exist, to avoid increasing cardinality with zero valued metrics. - if activeMatching[idx] > 0 { - i.metrics.activeSeriesCustomTrackersPerUser.WithLabelValues(userID, name).Set(float64(activeMatching[idx])) - } else { - i.metrics.activeSeriesCustomTrackersPerUser.DeleteLabelValues(userID, name) + if userDB.activeSeries.lastUpdate.IsZero() || userDB.activeSeries.lastUpdate.After(now.Add(i.cfg.ActiveSeriesMetricsIdleTimeout)) { + // Do not publish metrics until the new matcher setup had time to catch up + // LastUpdate is Zero when it never get updated + for idx, name := range userDB.activeSeries.asm.names { + // We only set the metrics for matchers that actually exist, to avoid increasing cardinality with zero valued metrics. + if activeMatching[idx] > 0 { + i.metrics.activeSeriesCustomTrackersPerUser.WithLabelValues(userID, name).Set(float64(activeMatching[idx])) + } else { + i.metrics.activeSeriesCustomTrackersPerUser.DeleteLabelValues(userID, name) + } } } + } } @@ -1444,10 +1509,20 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() + var activeSeriesMathers *ActiveSeriesMatchers + customTrackerConfig, err := createCustomTrackerConfig(i.cfg.RuntimeMatchersConfigFn(), userID) + if err != nil { + level.Error(i.logger).Log("msg", "failed to create custom tracker config, using empty one", "user", userID, "err", err) + } + activeSeriesMathers, err = NewActiveSeriesMatchers(customTrackerConfig) + if err != nil { + level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) + activeSeriesMathers = &ActiveSeriesMatchers{} + } userDB := &userTSDB{ userID: userID, - activeSeries: NewActiveSeries(i.activeSeriesMatcher), + activeSeries: NewActiveSeries(activeSeriesMathers), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), @@ -1570,9 +1645,7 @@ func (i *Ingester) closeAllTSDB() { i.metrics.memUsers.Dec() i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) - for _, name := range i.metrics.activeSeriesCustomTrackerNames { - i.metrics.activeSeriesCustomTrackersPerUser.DeleteLabelValues(userID, name) - } + i.metrics.deletePerUserCustomTrackerMetrics(userID, db.activeSeries.asm.names) }(userDB) } @@ -1969,6 +2042,7 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes i.deleteUserMetadata(userID) i.metrics.deletePerUserMetrics(userID) + i.metrics.deletePerUserCustomTrackerMetrics(userID, userDB.activeSeries.asm.names) validation.DeletePerUserValidationMetrics(userID, i.logger) diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 65108852d4..b75378e1a0 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -32,7 +32,6 @@ type ingesterMetrics struct { activeSeriesPerUser *prometheus.GaugeVec activeSeriesCustomTrackersPerUser *prometheus.GaugeVec - activeSeriesCustomTrackerNames []string // Global limit metrics maxUsersGauge prometheus.GaugeFunc @@ -54,7 +53,6 @@ type ingesterMetrics struct { func newIngesterMetrics( r prometheus.Registerer, activeSeriesEnabled bool, - activeSeriesCustomTrackerNames []string, instanceLimitsFn func() *InstanceLimits, ingestionRate *util_math.EwmaRate, inflightRequests *atomic.Int64, @@ -220,9 +218,6 @@ func newIngesterMetrics( Name: "cortex_ingester_active_series_custom_tracker", Help: "Number of currently active series matching a pre-configured label matchers per user.", }, []string{"user", "name"}), - // activeSeriesCustomTrackerNames contains all the values for the `name` label of activeSeriesCustomTrackersPerUser, - // so we can delete all the labels for each user when needed. - activeSeriesCustomTrackerNames: activeSeriesCustomTrackerNames, compactionsTriggered: promauto.With(r).NewCounter(prometheus.CounterOpts{ Name: "cortex_ingester_tsdb_compactions_triggered_total", @@ -264,7 +259,10 @@ func (m *ingesterMetrics) deletePerUserMetrics(userID string) { m.memMetadataCreatedTotal.DeleteLabelValues(userID) m.memMetadataRemovedTotal.DeleteLabelValues(userID) m.activeSeriesPerUser.DeleteLabelValues(userID) - for _, name := range m.activeSeriesCustomTrackerNames { +} + +func (m *ingesterMetrics) deletePerUserCustomTrackerMetrics(userID string, customTrackerMetrics []string) { + for _, name := range customTrackerMetrics { m.activeSeriesCustomTrackersPerUser.DeleteLabelValues(userID, name) } } diff --git a/pkg/ingester/runtime_matchers.go b/pkg/ingester/runtime_matchers.go new file mode 100644 index 0000000000..e517615512 --- /dev/null +++ b/pkg/ingester/runtime_matchers.go @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package ingester + +// RuntimeMatchers holds the definition of custom tracking rules +type RuntimeMatchersConfig struct { + GenericMatchers *ActiveSeriesCustomTrackersConfig `yaml:"generic_matchers"` + TenantSpecificMatchers map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_machers"` +} + +// Sets default runtime matchers for unmarshalling. +var defaultRuntimeMatchers *RuntimeMatchersConfig = nil + +// UnmarshalYAML implements the yaml.Unmarshaler interface. If give +func (l *RuntimeMatchersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + if defaultInstanceLimits != nil { + *l = *defaultRuntimeMatchers + } + type plain RuntimeMatchersConfig // type indirection to make sure we don't go into recursive loop + return unmarshal((*plain)(l)) +} diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index cacc138eb3..98f0e86990 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -420,6 +420,7 @@ func (t *Mimir) initIngesterService() (serv services.Service, err error) { t.Cfg.Ingester.LifecyclerConfig.ListenPort = t.Cfg.Server.GRPCListenPort t.Cfg.Ingester.StreamTypeFn = ingesterChunkStreaming(t.RuntimeConfig) t.Cfg.Ingester.InstanceLimitsFn = ingesterInstanceLimits(t.RuntimeConfig) + t.Cfg.Ingester.RuntimeMatchersConfigFn = runtimeMatchersConfig(t.RuntimeConfig) t.tsdbIngesterConfig() t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger) diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index 769de95bce..8fd558df9c 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -34,6 +34,8 @@ type runtimeConfigValues struct { IngesterChunkStreaming *bool `yaml:"ingester_stream_chunks_when_using_blocks"` IngesterLimits *ingester.InstanceLimits `yaml:"ingester_limits"` + + RuntimeMatchersConfig *ingester.RuntimeMatchersConfig `yaml:"runtime_matchers"` } // runtimeConfigTenantLimits provides per-tenant limit overrides based on a runtimeconfig.Manager @@ -145,6 +147,20 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } +func runtimeMatchersConfig(manager *runtimeconfig.Manager) func() *ingester.RuntimeMatchersConfig { + if manager == nil { + return nil + } + + return func() *ingester.RuntimeMatchersConfig { + val := manager.GetConfig() + if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { + return cfg.RuntimeMatchersConfig + } + return nil + } +} + func runtimeConfigHandler(runtimeCfgManager *runtimeconfig.Manager, defaultLimits validation.Limits) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { cfg, ok := runtimeCfgManager.GetConfig().(*runtimeConfigValues) From ceaecba7a8703a4133b1ce6ca4bc62136cf9255b Mon Sep 17 00:00:00 2001 From: Janos Date: Fri, 11 Feb 2022 17:31:34 +0100 Subject: [PATCH 02/91] fixing test --- pkg/ingester/ingester.go | 21 +++++++++++++++++---- pkg/ingester/ingester_test.go | 11 ++++++++--- pkg/ingester/runtime_matchers.go | 6 +++--- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 0845a83142..ce2b5acd06 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -262,7 +262,7 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus forceCompactTrigger: make(chan requestWithUsersAndCallback), shipTrigger: make(chan requestWithUsersAndCallback), seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), - runtimeMatchersConfig: cfg.RuntimeMatchersConfigFn(), + runtimeMatchersConfig: defaultRuntimeMatchers, }, nil } @@ -472,10 +472,10 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } func (i *Ingester) reloadConfig(now time.Time) { - newConfig := i.cfg.RuntimeMatchersConfigFn() + newConfig := i.getRuntimeMatchersConfig() // it is crucial to only reload matchers which have been changed, as this function runs even if there is no change in config // first check the generic matchers - if i.runtimeMatchersConfig.GenericMatchers.String() != newConfig.GenericMatchers.String() { + if i.runtimeMatchersConfig == nil || i.runtimeMatchersConfig.GenericMatchers.String() != newConfig.GenericMatchers.String() { // need to replace everything for _, userID := range i.getTSDBUsers() { userDB := i.getTSDB(userID) @@ -535,6 +535,19 @@ func createCustomTrackerConfig(runtimeConfig *RuntimeMatchersConfig, userID stri return *asc, err } +func (i *Ingester) getRuntimeMatchersConfig() *RuntimeMatchersConfig { + if i.cfg.RuntimeMatchersConfigFn == nil { + return &RuntimeMatchersConfig{} + } + + r := i.cfg.RuntimeMatchersConfigFn() + if r == nil { + return &RuntimeMatchersConfig{} + } + + return r +} + func (i *Ingester) updateActiveSeries(now time.Time) { purgeTime := now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout) for _, userID := range i.getTSDBUsers() { @@ -1510,7 +1523,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() var activeSeriesMathers *ActiveSeriesMatchers - customTrackerConfig, err := createCustomTrackerConfig(i.cfg.RuntimeMatchersConfigFn(), userID) + customTrackerConfig, err := createCustomTrackerConfig(i.getRuntimeMatchersConfig(), userID) if err != nil { level.Error(i.logger).Log("msg", "failed to create custom tracker config, using empty one", "user", userID, "err", err) } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index b46685a583..6898396808 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5271,9 +5271,14 @@ func TestIngesterActiveSeries(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.LifecyclerConfig.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - cfg.ActiveSeriesCustomTrackers = map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, + cfg.RuntimeMatchersConfigFn = func() *RuntimeMatchersConfig { + genericMatchers := map[string]string{ + "bool_is_true": `{bool="true"}`, + "bool_is_false": `{bool="false"}`, + } + return &RuntimeMatchersConfig{ + GenericMatchers: (ActiveSeriesCustomTrackersConfig)(genericMatchers), + } } ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) diff --git a/pkg/ingester/runtime_matchers.go b/pkg/ingester/runtime_matchers.go index e517615512..8dd812ed8f 100644 --- a/pkg/ingester/runtime_matchers.go +++ b/pkg/ingester/runtime_matchers.go @@ -4,8 +4,8 @@ package ingester // RuntimeMatchers holds the definition of custom tracking rules type RuntimeMatchersConfig struct { - GenericMatchers *ActiveSeriesCustomTrackersConfig `yaml:"generic_matchers"` - TenantSpecificMatchers map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_machers"` + GenericMatchers ActiveSeriesCustomTrackersConfig `yaml:"generic_matchers"` + TenantSpecificMatchers map[string]ActiveSeriesCustomTrackersConfig `yaml:"tenant_machers"` } // Sets default runtime matchers for unmarshalling. @@ -13,7 +13,7 @@ var defaultRuntimeMatchers *RuntimeMatchersConfig = nil // UnmarshalYAML implements the yaml.Unmarshaler interface. If give func (l *RuntimeMatchersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - if defaultInstanceLimits != nil { + if defaultRuntimeMatchers != nil { *l = *defaultRuntimeMatchers } type plain RuntimeMatchersConfig // type indirection to make sure we don't go into recursive loop From 31f697b435ad19a5c382d6d4b7ff89f6effbde17 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 15 Feb 2022 09:25:32 +0100 Subject: [PATCH 03/91] More straightworward logic --- pkg/ingester/ingester.go | 92 +++++++++++++++----------------- pkg/ingester/ingester_test.go | 2 +- pkg/ingester/runtime_matchers.go | 2 +- 3 files changed, 44 insertions(+), 52 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index ce2b5acd06..107bef742b 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -16,7 +16,6 @@ import ( "net/http" "os" "path/filepath" - "reflect" "strings" "sync" "time" @@ -473,34 +472,40 @@ func (i *Ingester) updateLoop(ctx context.Context) error { func (i *Ingester) reloadConfig(now time.Time) { newConfig := i.getRuntimeMatchersConfig() + // if the config is empty, we are done + if newConfig == nil { + i.runtimeMatchersConfig = newConfig + return + } + + defaultMatchersEquals := i.runtimeMatchersConfig.DefaultMatchers.String() == newConfig.DefaultMatchers.String() + // it is crucial to only reload matchers which have been changed, as this function runs even if there is no change in config - // first check the generic matchers - if i.runtimeMatchersConfig == nil || i.runtimeMatchersConfig.GenericMatchers.String() != newConfig.GenericMatchers.String() { - // need to replace everything - for _, userID := range i.getTSDBUsers() { - userDB := i.getTSDB(userID) - if userDB == nil { - continue - } - err := i.ReplaceMatchers(newConfig, userID, userDB) - if err != nil { - level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) - } + for _, userID := range i.getTSDBUsers() { + userDB := i.getTSDB(userID) + if userDB == nil { + continue } - } else { - // for each users check if the config has changed - for _, userID := range i.getTSDBUsers() { - userDB := i.getTSDB(userID) - if userDB == nil { - continue - } - oldValue, oldOk := i.runtimeMatchersConfig.TenantSpecificMatchers[userID] - newValue, newOk := newConfig.TenantSpecificMatchers[userID] - if oldOk != newOk || !reflect.DeepEqual(oldValue, newValue) { - err := i.ReplaceMatchers(newConfig, userID, userDB) - if err != nil { - level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) - } + oldValue, oldOk := i.runtimeMatchersConfig.TenantSpecificMatchers[userID] + newValue, newOk := newConfig.TenantSpecificMatchers[userID] + var replaceValue ActiveSeriesCustomTrackersConfig = nil + if !newOk && !oldOk && !defaultMatchersEquals { + // default changed and no overwrite + replaceValue = newConfig.DefaultMatchers + } else if newOk && !oldOk { + // tenant specific added + replaceValue = newValue + } else if !newOk && oldOk { + // tenant specific removed + replaceValue = newConfig.DefaultMatchers + } else if oldValue.String() != newValue.String() { + // tenant specific changed + replaceValue = newValue + } + if replaceValue != nil { + err := i.ReplaceMatchers(replaceValue, userDB) + if err != nil { + level.Error(i.logger).Log("msg", "failed to update config", "user", userID, "err", err) } } } @@ -508,12 +513,8 @@ func (i *Ingester) reloadConfig(now time.Time) { i.runtimeMatchersConfig = newConfig } -func (i *Ingester) ReplaceMatchers(newConfig *RuntimeMatchersConfig, userID string, userDB *userTSDB) error { - newTrackerConfig, err := createCustomTrackerConfig(newConfig, userID) - if err != nil { - return err - } - newMatcher, err := NewActiveSeriesMatchers(newTrackerConfig) +func (i *Ingester) ReplaceMatchers(config ActiveSeriesCustomTrackersConfig, userDB *userTSDB) error { + newMatcher, err := NewActiveSeriesMatchers(config) if err != nil { return err } @@ -522,19 +523,6 @@ func (i *Ingester) ReplaceMatchers(newConfig *RuntimeMatchersConfig, userID stri return nil } -func createCustomTrackerConfig(runtimeConfig *RuntimeMatchersConfig, userID string) (ActiveSeriesCustomTrackersConfig, error) { - userPart := "" - if val, ok := runtimeConfig.TenantSpecificMatchers[userID]; ok { - userPart = val.String() - } else if defaultVal, ok := runtimeConfig.TenantSpecificMatchers["default"]; ok { - userPart = defaultVal.String() - } - desiredConfig := runtimeConfig.GenericMatchers.String() + ";" + userPart - asc := &ActiveSeriesCustomTrackersConfig{} - err := asc.Set(desiredConfig) - return *asc, err -} - func (i *Ingester) getRuntimeMatchersConfig() *RuntimeMatchersConfig { if i.cfg.RuntimeMatchersConfigFn == nil { return &RuntimeMatchersConfig{} @@ -1523,11 +1511,15 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() var activeSeriesMathers *ActiveSeriesMatchers - customTrackerConfig, err := createCustomTrackerConfig(i.getRuntimeMatchersConfig(), userID) - if err != nil { - level.Error(i.logger).Log("msg", "failed to create custom tracker config, using empty one", "user", userID, "err", err) + cfg := i.cfg.RuntimeMatchersConfigFn() + val, ok := cfg.TenantSpecificMatchers[userID] + var config ActiveSeriesCustomTrackersConfig = nil + if ok { + config = val + } else { + config = cfg.DefaultMatchers } - activeSeriesMathers, err = NewActiveSeriesMatchers(customTrackerConfig) + activeSeriesMathers, err := NewActiveSeriesMatchers(config) if err != nil { level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) activeSeriesMathers = &ActiveSeriesMatchers{} diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 6898396808..50af4d3cfd 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5277,7 +5277,7 @@ func TestIngesterActiveSeries(t *testing.T) { "bool_is_false": `{bool="false"}`, } return &RuntimeMatchersConfig{ - GenericMatchers: (ActiveSeriesCustomTrackersConfig)(genericMatchers), + DefaultMatchers: (ActiveSeriesCustomTrackersConfig)(genericMatchers), } } diff --git a/pkg/ingester/runtime_matchers.go b/pkg/ingester/runtime_matchers.go index 8dd812ed8f..45bbfe8a8d 100644 --- a/pkg/ingester/runtime_matchers.go +++ b/pkg/ingester/runtime_matchers.go @@ -4,7 +4,7 @@ package ingester // RuntimeMatchers holds the definition of custom tracking rules type RuntimeMatchersConfig struct { - GenericMatchers ActiveSeriesCustomTrackersConfig `yaml:"generic_matchers"` + DefaultMatchers ActiveSeriesCustomTrackersConfig `yaml:"default_matchers"` TenantSpecificMatchers map[string]ActiveSeriesCustomTrackersConfig `yaml:"tenant_machers"` } From 6318a97c772799c577904d512e5694900a90c63c Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 15 Feb 2022 10:14:54 +0100 Subject: [PATCH 04/91] Some tests --- pkg/ingester/ingester.go | 12 +++++------ pkg/ingester/ingester_test.go | 4 ++-- pkg/ingester/runtime_matchers.go | 2 +- pkg/ingester/runtime_matchers_test.go | 29 +++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 9 deletions(-) create mode 100644 pkg/ingester/runtime_matchers_test.go diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 107bef742b..9208a14519 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -1511,15 +1511,15 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() var activeSeriesMathers *ActiveSeriesMatchers - cfg := i.cfg.RuntimeMatchersConfigFn() - val, ok := cfg.TenantSpecificMatchers[userID] - var config ActiveSeriesCustomTrackersConfig = nil + matchersCfg := i.getRuntimeMatchersConfig() + val, ok := matchersCfg.TenantSpecificMatchers[userID] + var matchers ActiveSeriesCustomTrackersConfig = nil if ok { - config = val + matchers = val } else { - config = cfg.DefaultMatchers + matchers = matchersCfg.DefaultMatchers } - activeSeriesMathers, err := NewActiveSeriesMatchers(config) + activeSeriesMathers, err := NewActiveSeriesMatchers(matchers) if err != nil { level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) activeSeriesMathers = &ActiveSeriesMatchers{} diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 50af4d3cfd..f30aa360e6 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5272,12 +5272,12 @@ func TestIngesterActiveSeries(t *testing.T) { cfg.LifecyclerConfig.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries cfg.RuntimeMatchersConfigFn = func() *RuntimeMatchersConfig { - genericMatchers := map[string]string{ + defaultMatchers := map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, } return &RuntimeMatchersConfig{ - DefaultMatchers: (ActiveSeriesCustomTrackersConfig)(genericMatchers), + DefaultMatchers: (ActiveSeriesCustomTrackersConfig)(defaultMatchers), } } diff --git a/pkg/ingester/runtime_matchers.go b/pkg/ingester/runtime_matchers.go index 45bbfe8a8d..37302e7d49 100644 --- a/pkg/ingester/runtime_matchers.go +++ b/pkg/ingester/runtime_matchers.go @@ -5,7 +5,7 @@ package ingester // RuntimeMatchers holds the definition of custom tracking rules type RuntimeMatchersConfig struct { DefaultMatchers ActiveSeriesCustomTrackersConfig `yaml:"default_matchers"` - TenantSpecificMatchers map[string]ActiveSeriesCustomTrackersConfig `yaml:"tenant_machers"` + TenantSpecificMatchers map[string]ActiveSeriesCustomTrackersConfig `yaml:"tenant_matchers"` } // Sets default runtime matchers for unmarshalling. diff --git a/pkg/ingester/runtime_matchers_test.go b/pkg/ingester/runtime_matchers_test.go new file mode 100644 index 0000000000..7636ce35af --- /dev/null +++ b/pkg/ingester/runtime_matchers_test.go @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package ingester + +import ( + "testing" + + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +func TestRuntimeMatchersUnmarshal(t *testing.T) { + r := RuntimeMatchersConfig{} + input := ` +default_matchers: + integrations/apolloserver: "{job='integrations/apollo-server'}" + integrations/caddy: "{job='integrations/caddy'}" +tenant_matchers: + 1: + team_A: "{grafanacloud_team='team_a'}" + team_B: "{grafanacloud_team='team_b'}" +` + + require.NoError(t, yaml.UnmarshalStrict([]byte(input), &r)) + require.Equal(t, "{job='integrations/apollo-server'}", r.DefaultMatchers["integrations/apolloserver"]) + require.Equal(t, "{job='integrations/caddy'}", r.DefaultMatchers["integrations/caddy"]) + require.Equal(t, "{grafanacloud_team='team_a'}", r.TenantSpecificMatchers["1"]["team_A"]) + require.Equal(t, "{grafanacloud_team='team_b'}", r.TenantSpecificMatchers["1"]["team_B"]) +} From 2be8c3b91e2014e974c7545c93fa5a702aa91180 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 15 Feb 2022 10:59:31 +0100 Subject: [PATCH 05/91] Clearing out tests --- cmd/mimir/help-all.txt.tmpl | 2 - cmd/mimir/help.txt.tmpl | 2 - pkg/ingester/ingester_test.go | 94 ++++++++++++++++++++++++----------- 3 files changed, 64 insertions(+), 34 deletions(-) diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 63eb1d521a..caccf708e6 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -909,8 +909,6 @@ Usage of ./cmd/mimir/mimir: Expected number of samples per timeseries, used for preallocations. (default 10) -ingester-client.expected-timeseries int Expected number of timeseries per request, used for preallocations. (default 100) - -ingester.active-series-custom-trackers value - Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag. -ingester.active-series-metrics-enabled Enable tracking of active series and export them as metrics. (default true) -ingester.active-series-metrics-idle-timeout duration diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index 8285865465..ce51e1ef2d 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -371,8 +371,6 @@ Usage of ./cmd/mimir/mimir: Expected number of samples per timeseries, used for preallocations. (default 10) -ingester-client.expected-timeseries int Expected number of timeseries per request, used for preallocations. (default 100) - -ingester.active-series-custom-trackers value - Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag. -ingester.active-series-metrics-enabled Enable tracking of active series and export them as metrics. (default true) -ingester.availability-zone string diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index f30aa360e6..ab6419337d 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5120,8 +5120,12 @@ func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []mimirpb } func TestIngesterActiveSeries(t *testing.T) { - metricLabelsBoolTrue := labels.FromStrings(labels.MetricName, "test", "bool", "true") - metricLabelsBoolFalse := labels.FromStrings(labels.MetricName, "test", "bool", "false") + labelsToPush := []labels.Labels{ + labels.FromStrings(labels.MetricName, "test_metric", "bool", "false", "team", "a"), + labels.FromStrings(labels.MetricName, "test_metric", "bool", "false", "team", "b"), + labels.FromStrings(labels.MetricName, "test_metric", "bool", "true", "team", "a"), + labels.FromStrings(labels.MetricName, "test_metric", "bool", "true", "team", "b"), + } req := func(lbls labels.Labels, t time.Time) *mimirpb.WriteRequest { return mimirpb.ToWriteRequest( @@ -5137,7 +5141,8 @@ func TestIngesterActiveSeries(t *testing.T) { "cortex_ingester_active_series", "cortex_ingester_active_series_custom_tracker", } - userID := "test" + userID := "test_user" + userID2 := "other_test_user" tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) @@ -5149,12 +5154,16 @@ func TestIngesterActiveSeries(t *testing.T) { test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for _, req := range []*mimirpb.WriteRequest{ - req(metricLabelsBoolTrue, now.Add(-2*time.Minute)), - req(metricLabelsBoolTrue, now.Add(-1*time.Minute)), - } { + for i, label := range labelsToPush { ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) require.NoError(t, err) } @@ -5164,10 +5173,14 @@ func TestIngesterActiveSeries(t *testing.T) { expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="test"} 1 + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 ` // Check tracked Prometheus metrics @@ -5178,13 +5191,16 @@ func TestIngesterActiveSeries(t *testing.T) { test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() - // We're pushing samples at firstPushTime - 1m, but they're accounted as pushed at firstPushTime - for _, req := range []*mimirpb.WriteRequest{ - req(metricLabelsBoolTrue, firstPushTime.Add(-time.Minute)), - req(metricLabelsBoolFalse, firstPushTime.Add(-time.Minute)), - } { + for i, label := range labelsToPush { ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) require.NoError(t, err) } @@ -5194,11 +5210,14 @@ func TestIngesterActiveSeries(t *testing.T) { expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 2 + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="test"} 1 - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="test"} 1 + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 ` // Check tracked Prometheus metrics @@ -5212,9 +5231,12 @@ func TestIngesterActiveSeries(t *testing.T) { // Sleep another millisecond to make sure that secondPushTime is strictly less than the append time of the second push. time.Sleep(time.Millisecond) - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req(metricLabelsBoolTrue, secondPushTime)) - require.NoError(t, err) + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } // Update active series for metrics check in the future. // We update them in the exact moment in time where append time of the first push is already considered idle, @@ -5224,10 +5246,11 @@ func TestIngesterActiveSeries(t *testing.T) { expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test"} 1 + cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="test"} 1 + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 ` // Check tracked Prometheus metrics @@ -5243,12 +5266,16 @@ func TestIngesterActiveSeries(t *testing.T) { test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for _, req := range []*mimirpb.WriteRequest{ - req(metricLabelsBoolTrue, now.Add(-2*time.Minute)), - req(metricLabelsBoolTrue, now.Add(-1*time.Minute)), - } { + for i, label := range labelsToPush { ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) require.NoError(t, err) } @@ -5276,8 +5303,15 @@ func TestIngesterActiveSeries(t *testing.T) { "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, } + teamMatchers := map[string]ActiveSeriesCustomTrackersConfig{ + "test_user": map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }, + } return &RuntimeMatchersConfig{ - DefaultMatchers: (ActiveSeriesCustomTrackersConfig)(defaultMatchers), + DefaultMatchers: (ActiveSeriesCustomTrackersConfig)(defaultMatchers), + TenantSpecificMatchers: teamMatchers, } } From 638699796d66ccf2dcbdde3aa21f702d727dfd8c Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 15 Feb 2022 11:03:46 +0100 Subject: [PATCH 06/91] Removing unused code --- pkg/ingester/ingester.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 9208a14519..2b16545bc4 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -472,12 +472,6 @@ func (i *Ingester) updateLoop(ctx context.Context) error { func (i *Ingester) reloadConfig(now time.Time) { newConfig := i.getRuntimeMatchersConfig() - // if the config is empty, we are done - if newConfig == nil { - i.runtimeMatchersConfig = newConfig - return - } - defaultMatchersEquals := i.runtimeMatchersConfig.DefaultMatchers.String() == newConfig.DefaultMatchers.String() // it is crucial to only reload matchers which have been changed, as this function runs even if there is no change in config From f4dd1d51086cb7f7ccb0bb0634ea0b7f3b6072cf Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 15 Feb 2022 11:20:18 +0100 Subject: [PATCH 07/91] Smaller fixes --- pkg/ingester/ingester.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 2b16545bc4..5f1160018c 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -235,7 +235,7 @@ type Ingester struct { ingestionRate *util_math.EwmaRate inflightPushRequests atomic.Int64 - // Last seen activeSeriesConfiguration. Need for diff based updating + // Last seen activeSeriesConfiguration. Needed for diff based updating. runtimeMatchersConfig *RuntimeMatchersConfig } @@ -261,7 +261,7 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus forceCompactTrigger: make(chan requestWithUsersAndCallback), shipTrigger: make(chan requestWithUsersAndCallback), seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), - runtimeMatchersConfig: defaultRuntimeMatchers, + runtimeMatchersConfig: &RuntimeMatchersConfig{}, }, nil } From 34d495368e157a28555f0c881b636a36fd025bf4 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 15 Feb 2022 14:01:55 +0100 Subject: [PATCH 08/91] Fixes based on oleg's review --- pkg/ingester/ingester.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 5f1160018c..9ed9d35afb 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -545,7 +545,7 @@ func (i *Ingester) updateActiveSeries(now time.Time) { } else { i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) } - if userDB.activeSeries.lastUpdate.IsZero() || userDB.activeSeries.lastUpdate.After(now.Add(i.cfg.ActiveSeriesMetricsIdleTimeout)) { + if userDB.activeSeries.lastUpdate.Before(purgeTime) { // Do not publish metrics until the new matcher setup had time to catch up // LastUpdate is Zero when it never get updated for idx, name := range userDB.activeSeries.asm.names { @@ -1504,7 +1504,6 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - var activeSeriesMathers *ActiveSeriesMatchers matchersCfg := i.getRuntimeMatchersConfig() val, ok := matchersCfg.TenantSpecificMatchers[userID] var matchers ActiveSeriesCustomTrackersConfig = nil @@ -1513,15 +1512,15 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { } else { matchers = matchersCfg.DefaultMatchers } - activeSeriesMathers, err := NewActiveSeriesMatchers(matchers) + activeSeriesMatchers, err := NewActiveSeriesMatchers(matchers) if err != nil { level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) - activeSeriesMathers = &ActiveSeriesMatchers{} + activeSeriesMatchers = &ActiveSeriesMatchers{} } userDB := &userTSDB{ userID: userID, - activeSeries: NewActiveSeries(activeSeriesMathers), + activeSeries: NewActiveSeries(activeSeriesMatchers), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), From 46d96bf3acfb3ad5ea900489d9e67d4c29e32c83 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 15 Feb 2022 14:50:06 +0100 Subject: [PATCH 09/91] Simplifying diff logic by comparing current and present logic every time at config reload --- pkg/ingester/active_series_custom_tracker.go | 2 + pkg/ingester/ingester.go | 63 +++++++------------- 2 files changed, 22 insertions(+), 43 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 80fe4977ad..58d3772bdd 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -63,6 +63,7 @@ func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml in func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*ActiveSeriesMatchers, error) { asm := &ActiveSeriesMatchers{} + asm.config = &matchers for name, matcher := range matchers { sm, err := amlabels.ParseMatchers(matcher) if err != nil { @@ -83,6 +84,7 @@ func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*Active } type ActiveSeriesMatchers struct { + config *ActiveSeriesCustomTrackersConfig names []string matchers []labelsMatchers } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 9ed9d35afb..b4bd12a107 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -234,9 +234,6 @@ type Ingester struct { // Rate of pushed samples. Used to limit global samples push rate. ingestionRate *util_math.EwmaRate inflightPushRequests atomic.Int64 - - // Last seen activeSeriesConfiguration. Needed for diff based updating. - runtimeMatchersConfig *RuntimeMatchersConfig } func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { @@ -254,14 +251,13 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus limits: limits, logger: logger, - tsdbs: make(map[string]*userTSDB), - usersMetadata: make(map[string]*userMetricsMetadata), - bucket: bucketClient, - tsdbMetrics: newTSDBMetrics(registerer), - forceCompactTrigger: make(chan requestWithUsersAndCallback), - shipTrigger: make(chan requestWithUsersAndCallback), - seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), - runtimeMatchersConfig: &RuntimeMatchersConfig{}, + tsdbs: make(map[string]*userTSDB), + usersMetadata: make(map[string]*userMetricsMetadata), + bucket: bucketClient, + tsdbMetrics: newTSDBMetrics(registerer), + forceCompactTrigger: make(chan requestWithUsersAndCallback), + shipTrigger: make(chan requestWithUsersAndCallback), + seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), }, nil } @@ -471,40 +467,28 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } func (i *Ingester) reloadConfig(now time.Time) { - newConfig := i.getRuntimeMatchersConfig() - defaultMatchersEquals := i.runtimeMatchersConfig.DefaultMatchers.String() == newConfig.DefaultMatchers.String() - - // it is crucial to only reload matchers which have been changed, as this function runs even if there is no change in config + currentConfig := i.getRuntimeMatchersConfig() for _, userID := range i.getTSDBUsers() { userDB := i.getTSDB(userID) if userDB == nil { continue } - oldValue, oldOk := i.runtimeMatchersConfig.TenantSpecificMatchers[userID] - newValue, newOk := newConfig.TenantSpecificMatchers[userID] - var replaceValue ActiveSeriesCustomTrackersConfig = nil - if !newOk && !oldOk && !defaultMatchersEquals { - // default changed and no overwrite - replaceValue = newConfig.DefaultMatchers - } else if newOk && !oldOk { - // tenant specific added - replaceValue = newValue - } else if !newOk && oldOk { - // tenant specific removed - replaceValue = newConfig.DefaultMatchers - } else if oldValue.String() != newValue.String() { - // tenant specific changed - replaceValue = newValue - } - if replaceValue != nil { - err := i.ReplaceMatchers(replaceValue, userDB) + newMatchers := getActiveSeriesConfig(userID, *currentConfig) + if newMatchers.String() != userDB.activeSeries.asm.config.String() { + err := i.ReplaceMatchers(newMatchers, userDB) if err != nil { level.Error(i.logger).Log("msg", "failed to update config", "user", userID, "err", err) } } } +} - i.runtimeMatchersConfig = newConfig +func getActiveSeriesConfig(userID string, config RuntimeMatchersConfig) ActiveSeriesCustomTrackersConfig { + val, ok := config.TenantSpecificMatchers[userID] + if !ok { + return config.DefaultMatchers + } + return val } func (i *Ingester) ReplaceMatchers(config ActiveSeriesCustomTrackersConfig, userDB *userTSDB) error { @@ -1504,15 +1488,8 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - matchersCfg := i.getRuntimeMatchersConfig() - val, ok := matchersCfg.TenantSpecificMatchers[userID] - var matchers ActiveSeriesCustomTrackersConfig = nil - if ok { - matchers = val - } else { - matchers = matchersCfg.DefaultMatchers - } - activeSeriesMatchers, err := NewActiveSeriesMatchers(matchers) + matchersConfig := getActiveSeriesConfig(userID, *i.getRuntimeMatchersConfig()) + activeSeriesMatchers, err := NewActiveSeriesMatchers(matchersConfig) if err != nil { level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) activeSeriesMatchers = &ActiveSeriesMatchers{} From c7297767575123d0147d5a77f7efa6091db7fcdd Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 09:24:17 +0100 Subject: [PATCH 10/91] Making serialization fail if matchers are not parseable --- pkg/ingester/active_series_custom_tracker.go | 18 ++- pkg/ingester/ingester.go | 27 ++--- pkg/ingester/ingester_test.go | 114 +++++++++++++++---- pkg/ingester/runtime_matchers.go | 10 +- pkg/ingester/runtime_matchers_test.go | 4 - 5 files changed, 120 insertions(+), 53 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 58d3772bdd..62b43b6c8e 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -63,7 +63,6 @@ func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml in func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*ActiveSeriesMatchers, error) { asm := &ActiveSeriesMatchers{} - asm.config = &matchers for name, matcher := range matchers { sm, err := amlabels.ParseMatchers(matcher) if err != nil { @@ -84,11 +83,26 @@ func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*Active } type ActiveSeriesMatchers struct { - config *ActiveSeriesCustomTrackersConfig names []string matchers []labelsMatchers } +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (asm *ActiveSeriesMatchers) UnmarshalYAML(unmarshal func(interface{}) error) error { + m := ActiveSeriesCustomTrackersConfig{} + err := unmarshal(&m) + if err != nil { + return err + } + var newMatchers *ActiveSeriesMatchers + newMatchers, err = NewActiveSeriesMatchers(m) + if err != nil { + return err + } + *asm = *newMatchers + return nil +} + func (asm *ActiveSeriesMatchers) MatcherNames() []string { return asm.names } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index b4bd12a107..08e66b4a85 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -473,8 +473,8 @@ func (i *Ingester) reloadConfig(now time.Time) { if userDB == nil { continue } - newMatchers := getActiveSeriesConfig(userID, *currentConfig) - if newMatchers.String() != userDB.activeSeries.asm.config.String() { + newMatchers := getActiveSeriesMatchers(userID, *currentConfig) + if newMatchers != userDB.activeSeries.asm { err := i.ReplaceMatchers(newMatchers, userDB) if err != nil { level.Error(i.logger).Log("msg", "failed to update config", "user", userID, "err", err) @@ -483,21 +483,17 @@ func (i *Ingester) reloadConfig(now time.Time) { } } -func getActiveSeriesConfig(userID string, config RuntimeMatchersConfig) ActiveSeriesCustomTrackersConfig { +func getActiveSeriesMatchers(userID string, config RuntimeMatchersConfig) *ActiveSeriesMatchers { val, ok := config.TenantSpecificMatchers[userID] if !ok { - return config.DefaultMatchers + return &config.DefaultMatchers } - return val + return &val } -func (i *Ingester) ReplaceMatchers(config ActiveSeriesCustomTrackersConfig, userDB *userTSDB) error { - newMatcher, err := NewActiveSeriesMatchers(config) - if err != nil { - return err - } +func (i *Ingester) ReplaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) error { i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.asm.names) - userDB.activeSeries.ReloadSeriesMatchers(newMatcher) + userDB.activeSeries.ReloadSeriesMatchers(asm) return nil } @@ -1488,16 +1484,11 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - matchersConfig := getActiveSeriesConfig(userID, *i.getRuntimeMatchersConfig()) - activeSeriesMatchers, err := NewActiveSeriesMatchers(matchersConfig) - if err != nil { - level.Error(i.logger).Log("msg", "failed to apply runtime matchers", "user", userID, "err", err) - activeSeriesMatchers = &ActiveSeriesMatchers{} - } + newMatchers := getActiveSeriesMatchers(userID, *i.getRuntimeMatchersConfig()) userDB := &userTSDB{ userID: userID, - activeSeries: NewActiveSeries(activeSeriesMatchers), + activeSeries: NewActiveSeries(newMatchers), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index ab6419337d..d3236ee15b 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5144,13 +5144,32 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" + defaultRuntimeMatcherConfigFn := func() *RuntimeMatchersConfig { + defaultMatchers, _ := NewActiveSeriesMatchers(map[string]string{ + "bool_is_true": `{bool="true"}`, + "bool_is_false": `{bool="false"}`, + }) + teamMatchers, _ := NewActiveSeriesMatchers(map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }) + return &RuntimeMatchersConfig{ + DefaultMatchers: *defaultMatchers, + TenantSpecificMatchers: map[string]ActiveSeriesMatchers{ + "test_user": *teamMatchers, + }, + } + } + tests := map[string]struct { - test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) - reqs []*mimirpb.WriteRequest - expectedMetrics string - disableActiveSeries bool + test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) + reqs []*mimirpb.WriteRequest + expectedMetrics string + disableActiveSeries bool + RuntimeMatchersConfigFn func() *RuntimeMatchersConfig }{ "successful push, should count active series": { + RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5188,6 +5207,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should track custom matchers, removing when zero": { + RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5262,7 +5282,8 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "successful push, active series disabled": { - disableActiveSeries: true, + RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, + disableActiveSeries: true, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5284,6 +5305,72 @@ func TestIngesterActiveSeries(t *testing.T) { expectedMetrics := `` + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + }, + }, + "should not fail with empty runtime config": { + RuntimeMatchersConfigFn: func() *RuntimeMatchersConfig { + return nil + }, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + now := time.Now() + + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(now) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 + ` + + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + }, + }, + "should not fail with nil matchers config function": { + RuntimeMatchersConfigFn: nil, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + now := time.Now() + + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(now) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 + ` + // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, @@ -5298,22 +5385,7 @@ func TestIngesterActiveSeries(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.LifecyclerConfig.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - cfg.RuntimeMatchersConfigFn = func() *RuntimeMatchersConfig { - defaultMatchers := map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, - } - teamMatchers := map[string]ActiveSeriesCustomTrackersConfig{ - "test_user": map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - }, - } - return &RuntimeMatchersConfig{ - DefaultMatchers: (ActiveSeriesCustomTrackersConfig)(defaultMatchers), - TenantSpecificMatchers: teamMatchers, - } - } + cfg.RuntimeMatchersConfigFn = testData.RuntimeMatchersConfigFn ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) require.NoError(t, err) diff --git a/pkg/ingester/runtime_matchers.go b/pkg/ingester/runtime_matchers.go index 37302e7d49..bdad142054 100644 --- a/pkg/ingester/runtime_matchers.go +++ b/pkg/ingester/runtime_matchers.go @@ -4,18 +4,12 @@ package ingester // RuntimeMatchers holds the definition of custom tracking rules type RuntimeMatchersConfig struct { - DefaultMatchers ActiveSeriesCustomTrackersConfig `yaml:"default_matchers"` - TenantSpecificMatchers map[string]ActiveSeriesCustomTrackersConfig `yaml:"tenant_matchers"` + DefaultMatchers ActiveSeriesMatchers `yaml:"default_matchers"` + TenantSpecificMatchers map[string]ActiveSeriesMatchers `yaml:"tenant_matchers"` } -// Sets default runtime matchers for unmarshalling. -var defaultRuntimeMatchers *RuntimeMatchersConfig = nil - // UnmarshalYAML implements the yaml.Unmarshaler interface. If give func (l *RuntimeMatchersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - if defaultRuntimeMatchers != nil { - *l = *defaultRuntimeMatchers - } type plain RuntimeMatchersConfig // type indirection to make sure we don't go into recursive loop return unmarshal((*plain)(l)) } diff --git a/pkg/ingester/runtime_matchers_test.go b/pkg/ingester/runtime_matchers_test.go index 7636ce35af..7f37d47e36 100644 --- a/pkg/ingester/runtime_matchers_test.go +++ b/pkg/ingester/runtime_matchers_test.go @@ -22,8 +22,4 @@ tenant_matchers: ` require.NoError(t, yaml.UnmarshalStrict([]byte(input), &r)) - require.Equal(t, "{job='integrations/apollo-server'}", r.DefaultMatchers["integrations/apolloserver"]) - require.Equal(t, "{job='integrations/caddy'}", r.DefaultMatchers["integrations/caddy"]) - require.Equal(t, "{grafanacloud_team='team_a'}", r.TenantSpecificMatchers["1"]["team_A"]) - require.Equal(t, "{grafanacloud_team='team_b'}", r.TenantSpecificMatchers["1"]["team_B"]) } From c5d38a46c9b4e7181990aaf99e3d01f8e0da7f58 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 11:56:54 +0100 Subject: [PATCH 11/91] Proper checking of series matcher equality --- pkg/ingester/active_series_custom_tracker.go | 34 +++++++++++++++++++ .../active_series_custom_tracker_test.go | 19 +++++++++++ pkg/ingester/ingester.go | 2 +- 3 files changed, 54 insertions(+), 1 deletion(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 62b43b6c8e..abf0ea14d3 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -87,6 +87,40 @@ type ActiveSeriesMatchers struct { matchers []labelsMatchers } +func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { + if asm == nil && other != nil { + return false + } + if asm != nil && other == nil { + return false + } + if len(asm.names) != len(other.names) { + return false + } + if len(asm.matchers) != len(other.matchers) { + return false + } + + for i, _ := range asm.names { + if asm.names[i] != other.names[i] { + return false + } + } + + for i, _ := range asm.matchers { + if len(asm.matchers[i]) != len(other.matchers[i]) { + return false + } + for j, _ := range asm.matchers[i] { + if asm.matchers[i][j].String() != other.matchers[i][j].String() { + return false + } + } + } + + return true +} + // UnmarshalYAML implements the yaml.Unmarshaler interface. func (asm *ActiveSeriesMatchers) UnmarshalYAML(unmarshal func(interface{}) error) error { m := ActiveSeriesCustomTrackersConfig{} diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 998b7a7dc5..44da8067f3 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -191,6 +191,25 @@ func TestActiveSeriesMatcher_MalformedMatcher(t *testing.T) { } } +func TestActiveSeriesMatcher_Equality(t *testing.T) { + matcher1 := `foo:{foo="bar"};baz:{baz="bar"}` + matcher2 := `baz:{baz="bar"};foo:{foo="bar"}` + t.Run("Equality", func(t *testing.T) { + config1 := ActiveSeriesCustomTrackersConfig{} + err := config1.Set(matcher1) + assert.NoError(t, err) + config2 := ActiveSeriesCustomTrackersConfig{} + err = config2.Set(matcher2) + assert.NoError(t, err) + + asm1, err := NewActiveSeriesMatchers(config1) + assert.NoError(t, err) + asm2, err := NewActiveSeriesMatchers(config2) + assert.NoError(t, err) + assert.True(t, asm1.Equals(asm2), "matcher configs should be equal") + }) +} + func TestAmlabelMatchersToProm_HappyCase(t *testing.T) { amMatcher, err := amlabels.NewMatcher(amlabels.MatchRegexp, "foo", "bar.*") require.NoError(t, err) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 08e66b4a85..5e7c2a5f4f 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -474,7 +474,7 @@ func (i *Ingester) reloadConfig(now time.Time) { continue } newMatchers := getActiveSeriesMatchers(userID, *currentConfig) - if newMatchers != userDB.activeSeries.asm { + if !newMatchers.Equals(userDB.activeSeries.asm) { err := i.ReplaceMatchers(newMatchers, userDB) if err != nil { level.Error(i.logger).Log("msg", "failed to update config", "user", userID, "err", err) From 2821436a597eb560d7241b8a1e4fc4f1ded9c8d0 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 12:02:31 +0100 Subject: [PATCH 12/91] Changing slice allocation based on oleg's suggestion --- pkg/ingester/active_series.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index 8f3cc35855..be9f433ccf 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -57,7 +57,7 @@ func NewActiveSeries(asm *ActiveSeriesMatchers) *ActiveSeries { c.stripes[i] = activeSeriesStripe{ asm: asm, refs: map[uint64][]activeSeriesEntry{}, - activeMatching: makeIntSliceIfNotEmpty(len(asm.MatcherNames())), + activeMatching: makeIntSliceIfNotEmpty(len(asm.MatcherNames()), nil), } } @@ -73,7 +73,7 @@ func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers) { for i := 0; i < numActiveSeriesStripes; i++ { c.stripes[i].asm = asm - c.stripes[i].activeMatching = makeIntSliceIfNotEmpty(len(asm.MatcherNames())) + c.stripes[i].activeMatching = makeIntSliceIfNotEmpty(len(asm.MatcherNames()), c.stripes[i].activeMatching) } c.lastUpdate = time.Now() } @@ -120,7 +120,7 @@ func (c *ActiveSeries) clear() { // custom trackers provided (in the same order as custom trackers are defined) func (c *ActiveSeries) Active() (int, []int) { total := 0 - totalMatching := makeIntSliceIfNotEmpty(len(c.asm.MatcherNames())) + totalMatching := makeIntSliceIfNotEmpty(len(c.asm.MatcherNames()), nil) for s := 0; s < numActiveSeriesStripes; s++ { total += c.stripes[s].getTotalAndUpdateMatching(totalMatching) } @@ -236,7 +236,7 @@ func (s *activeSeriesStripe) purge(keepUntil time.Time) { defer s.mu.Unlock() active := 0 - activeMatching := makeIntSliceIfNotEmpty(len(s.activeMatching)) + activeMatching := makeIntSliceIfNotEmpty(len(s.activeMatching), s.activeMatching) oldest := int64(math.MaxInt64) for fp, entries := range s.refs { @@ -302,9 +302,17 @@ func (s *activeSeriesStripe) purge(keepUntil time.Time) { s.activeMatching = activeMatching } -func makeIntSliceIfNotEmpty(l int) []int { - if l == 0 { - return nil +func makeIntSliceIfNotEmpty(l int, prev []int) []int { + if cap(prev) < l { + if l == 0 { + return nil + } + return make([]int, l, l*2) + } + + p := prev + for i := 0; i < l; i++ { + p[i] = 0 } - return make([]int, l) + return (prev)[:l] } From 4a391276c9e47e88805a56d9a5456b7d78105bbc Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 13:39:37 +0100 Subject: [PATCH 13/91] 'interning' default matchers to avoid memory leak --- pkg/ingester/ingester.go | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 5e7c2a5f4f..c399af6781 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -197,6 +197,9 @@ type Ingester struct { metrics *ingesterMetrics logger log.Logger + // Now it is used to track default matcher + activeSeriesMatchers *ActiveSeriesMatchers + lifecycler *ring.Lifecycler limits *validation.Overrides limiter *Limiter @@ -251,13 +254,14 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus limits: limits, logger: logger, - tsdbs: make(map[string]*userTSDB), - usersMetadata: make(map[string]*userMetricsMetadata), - bucket: bucketClient, - tsdbMetrics: newTSDBMetrics(registerer), - forceCompactTrigger: make(chan requestWithUsersAndCallback), - shipTrigger: make(chan requestWithUsersAndCallback), - seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), + tsdbs: make(map[string]*userTSDB), + usersMetadata: make(map[string]*userMetricsMetadata), + bucket: bucketClient, + tsdbMetrics: newTSDBMetrics(registerer), + forceCompactTrigger: make(chan requestWithUsersAndCallback), + shipTrigger: make(chan requestWithUsersAndCallback), + seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), + activeSeriesMatchers: &getRuntimeMatchersConfig(cfg.RuntimeMatchersConfigFn).DefaultMatchers, }, nil } @@ -467,13 +471,17 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } func (i *Ingester) reloadConfig(now time.Time) { - currentConfig := i.getRuntimeMatchersConfig() + currentConfig := getRuntimeMatchersConfig(i.cfg.RuntimeMatchersConfigFn) + if !currentConfig.DefaultMatchers.Equals(i.activeSeriesMatchers) { + // only replace default matcher object if changed to avoid memory leak + i.activeSeriesMatchers = ¤tConfig.DefaultMatchers + } for _, userID := range i.getTSDBUsers() { userDB := i.getTSDB(userID) if userDB == nil { continue } - newMatchers := getActiveSeriesMatchers(userID, *currentConfig) + newMatchers := i.getActiveSeriesMatchers(userID, currentConfig) if !newMatchers.Equals(userDB.activeSeries.asm) { err := i.ReplaceMatchers(newMatchers, userDB) if err != nil { @@ -483,10 +491,10 @@ func (i *Ingester) reloadConfig(now time.Time) { } } -func getActiveSeriesMatchers(userID string, config RuntimeMatchersConfig) *ActiveSeriesMatchers { +func (i *Ingester) getActiveSeriesMatchers(userID string, config *RuntimeMatchersConfig) *ActiveSeriesMatchers { val, ok := config.TenantSpecificMatchers[userID] if !ok { - return &config.DefaultMatchers + return i.activeSeriesMatchers } return &val } @@ -497,12 +505,12 @@ func (i *Ingester) ReplaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) return nil } -func (i *Ingester) getRuntimeMatchersConfig() *RuntimeMatchersConfig { - if i.cfg.RuntimeMatchersConfigFn == nil { +func getRuntimeMatchersConfig(runtimeMatchersConfigFn func() *RuntimeMatchersConfig) *RuntimeMatchersConfig { + if runtimeMatchersConfigFn == nil { return &RuntimeMatchersConfig{} } - r := i.cfg.RuntimeMatchersConfigFn() + r := runtimeMatchersConfigFn() if r == nil { return &RuntimeMatchersConfig{} } @@ -1484,7 +1492,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - newMatchers := getActiveSeriesMatchers(userID, *i.getRuntimeMatchersConfig()) + newMatchers := i.getActiveSeriesMatchers(userID, getRuntimeMatchersConfig(i.cfg.RuntimeMatchersConfigFn)) userDB := &userTSDB{ userID: userID, From a65aaf125b256bdf23a95d0e952db0ee28b727b3 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 14:25:20 +0100 Subject: [PATCH 14/91] Support flag based config if no runtime config provided --- cmd/mimir/help-all.txt.tmpl | 2 ++ cmd/mimir/help.txt.tmpl | 2 ++ pkg/ingester/ingester.go | 59 ++++++++++++++++++++++--------------- 3 files changed, 39 insertions(+), 24 deletions(-) diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index caccf708e6..63eb1d521a 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -909,6 +909,8 @@ Usage of ./cmd/mimir/mimir: Expected number of samples per timeseries, used for preallocations. (default 10) -ingester-client.expected-timeseries int Expected number of timeseries per request, used for preallocations. (default 100) + -ingester.active-series-custom-trackers value + Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag. -ingester.active-series-metrics-enabled Enable tracking of active series and export them as metrics. (default true) -ingester.active-series-metrics-idle-timeout duration diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index ce51e1ef2d..8285865465 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -371,6 +371,8 @@ Usage of ./cmd/mimir/mimir: Expected number of samples per timeseries, used for preallocations. (default 10) -ingester-client.expected-timeseries int Expected number of timeseries per request, used for preallocations. (default 100) + -ingester.active-series-custom-trackers value + Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag. -ingester.active-series-metrics-enabled Enable tracking of active series and export them as metrics. (default true) -ingester.availability-zone string diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index c399af6781..5747dbd753 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -121,10 +121,11 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - RuntimeMatchersConfigFn func() *RuntimeMatchersConfig `yaml:"-"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesCustomTrackers ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)."` + RuntimeMatchersConfigFn func() *RuntimeMatchersConfig `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -153,6 +154,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) { f.BoolVar(&cfg.ActiveSeriesMetricsEnabled, "ingester.active-series-metrics-enabled", true, "Enable tracking of active series and export them as metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsUpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsIdleTimeout, "ingester.active-series-metrics-idle-timeout", 10*time.Minute, "After what time a series is considered to be inactive.") + f.Var(&cfg.ActiveSeriesCustomTrackers, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag.") f.BoolVar(&cfg.StreamChunksWhenUsingBlocks, "ingester.stream-chunks-when-using-blocks", true, "Stream chunks from ingesters to queriers.") f.DurationVar(&cfg.ExemplarsUpdatePeriod, "ingester.exemplars-update-period", 15*time.Second, "Period with which to update per-user max exemplars.") @@ -198,7 +200,7 @@ type Ingester struct { logger log.Logger // Now it is used to track default matcher - activeSeriesMatchers *ActiveSeriesMatchers + activeSeriesMatcher *ActiveSeriesMatchers lifecycler *ring.Lifecycler limits *validation.Overrides @@ -250,18 +252,18 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus } return &Ingester{ - cfg: cfg, - limits: limits, - logger: logger, - - tsdbs: make(map[string]*userTSDB), - usersMetadata: make(map[string]*userMetricsMetadata), - bucket: bucketClient, - tsdbMetrics: newTSDBMetrics(registerer), - forceCompactTrigger: make(chan requestWithUsersAndCallback), - shipTrigger: make(chan requestWithUsersAndCallback), - seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), - activeSeriesMatchers: &getRuntimeMatchersConfig(cfg.RuntimeMatchersConfigFn).DefaultMatchers, + cfg: cfg, + limits: limits, + logger: logger, + activeSeriesMatcher: &ActiveSeriesMatchers{}, + + tsdbs: make(map[string]*userTSDB), + usersMetadata: make(map[string]*userMetricsMetadata), + bucket: bucketClient, + tsdbMetrics: newTSDBMetrics(registerer), + forceCompactTrigger: make(chan requestWithUsersAndCallback), + shipTrigger: make(chan requestWithUsersAndCallback), + seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), }, nil } @@ -281,6 +283,12 @@ func New(cfg Config, clientConfig client.Config, limits *validation.Overrides, r i.ingestionRate = util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval) i.metrics = newIngesterMetrics(registerer, cfg.ActiveSeriesMetricsEnabled, i.getInstanceLimits, i.ingestionRate, &i.inflightPushRequests) + asm, err := NewActiveSeriesMatchers(cfg.ActiveSeriesCustomTrackers) + if err != nil { + return nil, err + } + i.activeSeriesMatcher = asm + // Replace specific metrics which we can't directly track but we need to read // them from the underlying system (ie. TSDB). if registerer != nil { @@ -472,10 +480,6 @@ func (i *Ingester) updateLoop(ctx context.Context) error { func (i *Ingester) reloadConfig(now time.Time) { currentConfig := getRuntimeMatchersConfig(i.cfg.RuntimeMatchersConfigFn) - if !currentConfig.DefaultMatchers.Equals(i.activeSeriesMatchers) { - // only replace default matcher object if changed to avoid memory leak - i.activeSeriesMatchers = ¤tConfig.DefaultMatchers - } for _, userID := range i.getTSDBUsers() { userDB := i.getTSDB(userID) if userDB == nil { @@ -492,9 +496,16 @@ func (i *Ingester) reloadConfig(now time.Time) { } func (i *Ingester) getActiveSeriesMatchers(userID string, config *RuntimeMatchersConfig) *ActiveSeriesMatchers { + if config == nil { + return i.activeSeriesMatcher + } val, ok := config.TenantSpecificMatchers[userID] if !ok { - return i.activeSeriesMatchers + if !i.activeSeriesMatcher.Equals(&config.DefaultMatchers) { + // this avoids referencing multiple instances of deserialized DefaultMatchers + i.activeSeriesMatcher = &config.DefaultMatchers + } + return i.activeSeriesMatcher } return &val } @@ -507,12 +518,12 @@ func (i *Ingester) ReplaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) func getRuntimeMatchersConfig(runtimeMatchersConfigFn func() *RuntimeMatchersConfig) *RuntimeMatchersConfig { if runtimeMatchersConfigFn == nil { - return &RuntimeMatchersConfig{} + return nil } r := runtimeMatchersConfigFn() if r == nil { - return &RuntimeMatchersConfig{} + return nil } return r From 4fd52f113ef7719bb617c2d4f85010776a375f26 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 14:48:54 +0100 Subject: [PATCH 15/91] Adding test cases for backward compatibility and overwrite check --- pkg/ingester/ingester_test.go | 84 +++++++++++++++++++++++++++++++++++ 1 file changed, 84 insertions(+) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index d3236ee15b..f801fa21b8 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5160,6 +5160,10 @@ func TestIngesterActiveSeries(t *testing.T) { }, } } + activeSeriesDefaultConfig := map[string]string{ + "bool_is_true_flagbased": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + } tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) @@ -5167,6 +5171,7 @@ func TestIngesterActiveSeries(t *testing.T) { expectedMetrics string disableActiveSeries bool RuntimeMatchersConfigFn func() *RuntimeMatchersConfig + activeSeriesConfig ActiveSeriesCustomTrackersConfig }{ "successful push, should count active series": { RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, @@ -5371,6 +5376,84 @@ func TestIngesterActiveSeries(t *testing.T) { cortex_ingester_active_series{user="test_user"} 4 ` + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + }, + }, + "should use flag based custom tracker if no runtime config specified": { + RuntimeMatchersConfigFn: nil, + activeSeriesConfig: activeSeriesDefaultConfig, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + now := time.Now() + + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(now) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="test_user"} 2 + ` + + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + }, + }, + "should use runtime matcher config if both specified": { + RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, + activeSeriesConfig: activeSeriesDefaultConfig, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + now := time.Now() + + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(now) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 + ` + // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, @@ -5386,6 +5469,7 @@ func TestIngesterActiveSeries(t *testing.T) { cfg.LifecyclerConfig.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries cfg.RuntimeMatchersConfigFn = testData.RuntimeMatchersConfigFn + cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) require.NoError(t, err) From d9870786d8e2092b20cdaed3fe545bb5cfd71d34 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 14:56:14 +0100 Subject: [PATCH 16/91] Cleaning up --- pkg/ingester/ingester.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 5747dbd753..1e550cd990 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -487,10 +487,7 @@ func (i *Ingester) reloadConfig(now time.Time) { } newMatchers := i.getActiveSeriesMatchers(userID, currentConfig) if !newMatchers.Equals(userDB.activeSeries.asm) { - err := i.ReplaceMatchers(newMatchers, userDB) - if err != nil { - level.Error(i.logger).Log("msg", "failed to update config", "user", userID, "err", err) - } + i.ReplaceMatchers(newMatchers, userDB) } } } @@ -510,10 +507,9 @@ func (i *Ingester) getActiveSeriesMatchers(userID string, config *RuntimeMatcher return &val } -func (i *Ingester) ReplaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) error { +func (i *Ingester) ReplaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) { i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.asm.names) userDB.activeSeries.ReloadSeriesMatchers(asm) - return nil } func getRuntimeMatchersConfig(runtimeMatchersConfigFn func() *RuntimeMatchersConfig) *RuntimeMatchersConfig { @@ -521,12 +517,7 @@ func getRuntimeMatchersConfig(runtimeMatchersConfigFn func() *RuntimeMatchersCon return nil } - r := runtimeMatchersConfigFn() - if r == nil { - return nil - } - - return r + return runtimeMatchersConfigFn() } func (i *Ingester) updateActiveSeries(now time.Time) { From cd7bc5ef16b2dce4d619e9a0632be609df809bbf Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 15:27:42 +0100 Subject: [PATCH 17/91] Early-exit when pointers reference the same object --- pkg/ingester/active_series_custom_tracker.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index abf0ea14d3..6966d88afb 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -88,6 +88,9 @@ type ActiveSeriesMatchers struct { } func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { + if asm == other { + return true + } if asm == nil && other != nil { return false } From 96aca48e7e4e4651df6496d532d111f7895540f2 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Feb 2022 16:48:14 +0100 Subject: [PATCH 18/91] Changing equality check to string concatenation based as it is much faster --- pkg/ingester/active_series_custom_tracker.go | 37 ++----------------- .../active_series_custom_tracker_test.go | 2 +- 2 files changed, 4 insertions(+), 35 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 6966d88afb..521c4d5fa8 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -64,6 +64,7 @@ func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml in func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*ActiveSeriesMatchers, error) { asm := &ActiveSeriesMatchers{} for name, matcher := range matchers { + asm.config += name + matcher sm, err := amlabels.ParseMatchers(matcher) if err != nil { return nil, fmt.Errorf("can't build active series matcher %s: %w", name, err) @@ -83,45 +84,13 @@ func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*Active } type ActiveSeriesMatchers struct { + config string names []string matchers []labelsMatchers } func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { - if asm == other { - return true - } - if asm == nil && other != nil { - return false - } - if asm != nil && other == nil { - return false - } - if len(asm.names) != len(other.names) { - return false - } - if len(asm.matchers) != len(other.matchers) { - return false - } - - for i, _ := range asm.names { - if asm.names[i] != other.names[i] { - return false - } - } - - for i, _ := range asm.matchers { - if len(asm.matchers[i]) != len(other.matchers[i]) { - return false - } - for j, _ := range asm.matchers[i] { - if asm.matchers[i][j].String() != other.matchers[i][j].String() { - return false - } - } - } - - return true + return asm.config == other.config } // UnmarshalYAML implements the yaml.Unmarshaler interface. diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 44da8067f3..c7fadb2514 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -193,7 +193,7 @@ func TestActiveSeriesMatcher_MalformedMatcher(t *testing.T) { func TestActiveSeriesMatcher_Equality(t *testing.T) { matcher1 := `foo:{foo="bar"};baz:{baz="bar"}` - matcher2 := `baz:{baz="bar"};foo:{foo="bar"}` + matcher2 := `foo:{foo="bar"};baz:{baz="bar"}` t.Run("Equality", func(t *testing.T) { config1 := ActiveSeriesCustomTrackersConfig{} err := config1.Set(matcher1) From b552d66b9ebefeb94851bae92c21f9e2b3541d04 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 17 Feb 2022 09:24:50 +0100 Subject: [PATCH 19/91] Matcher configuration fix + adjusting test --- pkg/ingester/active_series_custom_tracker.go | 8 +++++++- pkg/ingester/active_series_custom_tracker_test.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 521c4d5fa8..8510493377 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -64,7 +64,6 @@ func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml in func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*ActiveSeriesMatchers, error) { asm := &ActiveSeriesMatchers{} for name, matcher := range matchers { - asm.config += name + matcher sm, err := amlabels.ParseMatchers(matcher) if err != nil { return nil, fmt.Errorf("can't build active series matcher %s: %w", name, err) @@ -80,6 +79,13 @@ func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*Active // Sort the result to make it deterministic for tests. // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) + // The concatenation should happen after ordering, to ensure equality is not dependent on map traversal. + for i, name := range asm.names { + asm.config += name + for _, labelMatcher := range asm.matchers[i] { + asm.config += labelMatcher.String() + } + } return asm, nil } diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index c7fadb2514..44da8067f3 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -193,7 +193,7 @@ func TestActiveSeriesMatcher_MalformedMatcher(t *testing.T) { func TestActiveSeriesMatcher_Equality(t *testing.T) { matcher1 := `foo:{foo="bar"};baz:{baz="bar"}` - matcher2 := `foo:{foo="bar"};baz:{baz="bar"}` + matcher2 := `baz:{baz="bar"};foo:{foo="bar"}` t.Run("Equality", func(t *testing.T) { config1 := ActiveSeriesCustomTrackersConfig{} err := config1.Set(matcher1) From bb4e8b1fe0494ccd5d30b3575777223a16e89e36 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 17 Feb 2022 10:14:46 +0100 Subject: [PATCH 20/91] Adding active_series_custom_tracker tests --- .../active_series_custom_tracker_test.go | 90 +++++++++++++++---- 1 file changed, 75 insertions(+), 15 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 44da8067f3..8742bcb623 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -8,6 +8,7 @@ import ( "github.com/pkg/errors" "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" amlabels "github.com/prometheus/alertmanager/pkg/labels" "github.com/prometheus/prometheus/model/labels" @@ -192,21 +193,80 @@ func TestActiveSeriesMatcher_MalformedMatcher(t *testing.T) { } func TestActiveSeriesMatcher_Equality(t *testing.T) { - matcher1 := `foo:{foo="bar"};baz:{baz="bar"}` - matcher2 := `baz:{baz="bar"};foo:{foo="bar"}` - t.Run("Equality", func(t *testing.T) { - config1 := ActiveSeriesCustomTrackersConfig{} - err := config1.Set(matcher1) - assert.NoError(t, err) - config2 := ActiveSeriesCustomTrackersConfig{} - err = config2.Set(matcher2) - assert.NoError(t, err) - - asm1, err := NewActiveSeriesMatchers(config1) - assert.NoError(t, err) - asm2, err := NewActiveSeriesMatchers(config2) - assert.NoError(t, err) - assert.True(t, asm1.Equals(asm2), "matcher configs should be equal") + matcherSets := [][]string{ + { + `foo:{foo="bar"};baz:{baz="bar"}`, + `baz:{baz="bar"};foo:{foo="bar"}`, + ` foo:{foo="bar"};baz:{baz="bar"} `, + }, + { + `test:{test="true"}`, + }, + { + `foo:{foo="bar"};baz:{baz="bar"};extra:{extra="extra"}`, + }, + } + + for _, matcherSet := range matcherSets { + t.Run("EqualityBetweenSet", func(t *testing.T) { + var activeSeriesMatchers []*ActiveSeriesMatchers + for _, matcherConfig := range matcherSet { + config := ActiveSeriesCustomTrackersConfig{} + err := config.Set(matcherConfig) + assert.NoError(t, err) + asm, err := NewActiveSeriesMatchers(config) + assert.NoError(t, err) + activeSeriesMatchers = append(activeSeriesMatchers, asm) + } + for i := 0; i < len(activeSeriesMatchers); i++ { + for j := i + 1; j < len(activeSeriesMatchers); j++ { + assert.True(t, activeSeriesMatchers[i].Equals(activeSeriesMatchers[j]), "matcher configs should be equal") + } + } + }) + } + + t.Run("NotEqualsAcrossSets", func(t *testing.T) { + var activeSeriesMatchers []*ActiveSeriesMatchers + for _, matcherConfigs := range matcherSets { + exampleConfig := matcherConfigs[0] + config := ActiveSeriesCustomTrackersConfig{} + err := config.Set(exampleConfig) + assert.NoError(t, err) + asm, err := NewActiveSeriesMatchers(config) + assert.NoError(t, err) + activeSeriesMatchers = append(activeSeriesMatchers, asm) + } + + for i := 0; i < len(activeSeriesMatchers); i++ { + for j := i + 1; j < len(activeSeriesMatchers); j++ { + assert.False(t, activeSeriesMatchers[i].Equals(activeSeriesMatchers[j]), "matcher configs should NOT be equal") + } + } + }) + +} + +func TestActiveSeriesMatcher_Deserialization(t *testing.T) { + correct_input := ` + baz: "{baz='bar'}" + foo: "{foo='bar'}" + ` + malformed_input := + ` + baz: "123" + foo: "{foo='bar'}" + ` + t.Run("ShouldDeserializeCorrectInput", func(t *testing.T) { + asm := ActiveSeriesMatchers{} + err := yaml.Unmarshal([]byte(correct_input), &asm) + assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") + }) + + t.Run("ShouldErrorOnMalformedInput", func(t *testing.T) { + asm := ActiveSeriesMatchers{} + err := yaml.Unmarshal([]byte(malformed_input), &asm) + assert.Error(t, err, "should not deserialize malformed input") }) } From fa925be5906bc3cc215fc5d7b1b2a6a0153c7829 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 17 Feb 2022 10:45:54 +0100 Subject: [PATCH 21/91] Adding active_series_test --- pkg/ingester/active_series_test.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index 674d9442c1..c8118ad8ce 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -194,6 +194,32 @@ func TestActiveSeries_PurgeOpt(t *testing.T) { assert.Equal(t, 1, allActive) } +func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { + ls1 := []labels.Label{{Name: "a", Value: "1"}} + ls2 := []labels.Label{{Name: "a", Value: "2"}} + + asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{a=~.*}`}) + assert.NoError(t, err) + + c := NewActiveSeries(asm) + allActive, activeMatching := c.Active() + assert.Equal(t, 0, allActive) + assert.Equal(t, []int{0}, activeMatching) + + c.UpdateSeries(ls1, time.Now(), copyFn) + allActive, activeMatching = c.Active() + assert.Equal(t, 1, allActive) + assert.Equal(t, []int{1}, activeMatching) + + reloadTime := time.Now() + c.ReloadSeriesMatchers(asm) + c.UpdateSeries(ls2, time.Now(), copyFn) + allActive, activeMatching = c.Active() + assert.Equal(t, 2, allActive, "reloading should not affect general counter") + assert.Equal(t, []int{1}, activeMatching, "reloading should clear out matcher counters") + assert.True(t, c.lastUpdate.After(reloadTime)) +} + var activeSeriesTestGoroutines = []int{50, 100, 500} func BenchmarkActiveSeriesTest_single_series(b *testing.B) { From c651fca502020321cb6323c48da64c451fe9aa73 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 17 Feb 2022 14:39:24 +0100 Subject: [PATCH 22/91] Fixes based on oleg's suggestions --- pkg/ingester/active_series.go | 4 +- pkg/ingester/active_series_custom_tracker.go | 6 +- pkg/ingester/ingester.go | 14 +--- pkg/ingester/ingester_test.go | 70 +++++++++++--------- pkg/ingester/runtime_matchers.go | 13 ++-- pkg/mimir/modules.go | 2 +- pkg/mimir/runtime_config.go | 16 +++-- 7 files changed, 65 insertions(+), 60 deletions(-) diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index be9f433ccf..fd6c64effe 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -310,9 +310,9 @@ func makeIntSliceIfNotEmpty(l int, prev []int) []int { return make([]int, l, l*2) } - p := prev + p := prev[:l] for i := 0; i < l; i++ { p[i] = 0 } - return (prev)[:l] + return p } diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 8510493377..acc347ce88 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -79,13 +79,15 @@ func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*Active // Sort the result to make it deterministic for tests. // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) + var sb strings.Builder // The concatenation should happen after ordering, to ensure equality is not dependent on map traversal. for i, name := range asm.names { - asm.config += name + sb.WriteString(name) for _, labelMatcher := range asm.matchers[i] { - asm.config += labelMatcher.String() + sb.WriteString(labelMatcher.String()) } } + asm.config = sb.String() return asm, nil } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 1e550cd990..ada2a7bb86 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -125,7 +125,7 @@ type Config struct { ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` ActiveSeriesCustomTrackers ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)."` - RuntimeMatchersConfigFn func() *RuntimeMatchersConfig `yaml:"-"` + RuntimeMatchersConfigProvider *RuntimeMatchersConfigProvider `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -479,7 +479,7 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } func (i *Ingester) reloadConfig(now time.Time) { - currentConfig := getRuntimeMatchersConfig(i.cfg.RuntimeMatchersConfigFn) + currentConfig := i.cfg.RuntimeMatchersConfigProvider.Get() for _, userID := range i.getTSDBUsers() { userDB := i.getTSDB(userID) if userDB == nil { @@ -512,14 +512,6 @@ func (i *Ingester) ReplaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) userDB.activeSeries.ReloadSeriesMatchers(asm) } -func getRuntimeMatchersConfig(runtimeMatchersConfigFn func() *RuntimeMatchersConfig) *RuntimeMatchersConfig { - if runtimeMatchersConfigFn == nil { - return nil - } - - return runtimeMatchersConfigFn() -} - func (i *Ingester) updateActiveSeries(now time.Time) { purgeTime := now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout) for _, userID := range i.getTSDBUsers() { @@ -1494,7 +1486,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - newMatchers := i.getActiveSeriesMatchers(userID, getRuntimeMatchersConfig(i.cfg.RuntimeMatchersConfigFn)) + newMatchers := i.getActiveSeriesMatchers(userID, i.cfg.RuntimeMatchersConfigProvider.Get()) userDB := &userTSDB{ userID: userID, diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index f801fa21b8..c8fa07f4af 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5144,21 +5144,23 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultRuntimeMatcherConfigFn := func() *RuntimeMatchersConfig { - defaultMatchers, _ := NewActiveSeriesMatchers(map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, - }) - teamMatchers, _ := NewActiveSeriesMatchers(map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - }) - return &RuntimeMatchersConfig{ - DefaultMatchers: *defaultMatchers, - TenantSpecificMatchers: map[string]ActiveSeriesMatchers{ - "test_user": *teamMatchers, - }, - } + defaultRuntimeMatcherConfigFn := &RuntimeMatchersConfigProvider{ + func() *RuntimeMatchersConfig { + defaultMatchers, _ := NewActiveSeriesMatchers(map[string]string{ + "bool_is_true": `{bool="true"}`, + "bool_is_false": `{bool="false"}`, + }) + teamMatchers, _ := NewActiveSeriesMatchers(map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }) + return &RuntimeMatchersConfig{ + DefaultMatchers: *defaultMatchers, + TenantSpecificMatchers: map[string]ActiveSeriesMatchers{ + "test_user": *teamMatchers, + }, + } + }, } activeSeriesDefaultConfig := map[string]string{ "bool_is_true_flagbased": `{bool="true"}`, @@ -5166,15 +5168,15 @@ func TestIngesterActiveSeries(t *testing.T) { } tests := map[string]struct { - test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) - reqs []*mimirpb.WriteRequest - expectedMetrics string - disableActiveSeries bool - RuntimeMatchersConfigFn func() *RuntimeMatchersConfig - activeSeriesConfig ActiveSeriesCustomTrackersConfig + test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) + reqs []*mimirpb.WriteRequest + expectedMetrics string + disableActiveSeries bool + runtimeMatchersConfigProvider *RuntimeMatchersConfigProvider + activeSeriesConfig ActiveSeriesCustomTrackersConfig }{ "successful push, should count active series": { - RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, + runtimeMatchersConfigProvider: defaultRuntimeMatcherConfigFn, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5212,7 +5214,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should track custom matchers, removing when zero": { - RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, + runtimeMatchersConfigProvider: defaultRuntimeMatcherConfigFn, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5287,8 +5289,8 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "successful push, active series disabled": { - RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, - disableActiveSeries: true, + runtimeMatchersConfigProvider: defaultRuntimeMatcherConfigFn, + disableActiveSeries: true, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5315,8 +5317,10 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with empty runtime config": { - RuntimeMatchersConfigFn: func() *RuntimeMatchersConfig { - return nil + runtimeMatchersConfigProvider: &RuntimeMatchersConfigProvider{ + func() *RuntimeMatchersConfig { + return nil + }, }, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5349,7 +5353,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with nil matchers config function": { - RuntimeMatchersConfigFn: nil, + runtimeMatchersConfigProvider: nil, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5381,8 +5385,8 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should use flag based custom tracker if no runtime config specified": { - RuntimeMatchersConfigFn: nil, - activeSeriesConfig: activeSeriesDefaultConfig, + runtimeMatchersConfigProvider: nil, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5420,8 +5424,8 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should use runtime matcher config if both specified": { - RuntimeMatchersConfigFn: defaultRuntimeMatcherConfigFn, - activeSeriesConfig: activeSeriesDefaultConfig, + runtimeMatchersConfigProvider: defaultRuntimeMatcherConfigFn, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5468,7 +5472,7 @@ func TestIngesterActiveSeries(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.LifecyclerConfig.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - cfg.RuntimeMatchersConfigFn = testData.RuntimeMatchersConfigFn + cfg.RuntimeMatchersConfigProvider = testData.runtimeMatchersConfigProvider cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) diff --git a/pkg/ingester/runtime_matchers.go b/pkg/ingester/runtime_matchers.go index bdad142054..68396fb526 100644 --- a/pkg/ingester/runtime_matchers.go +++ b/pkg/ingester/runtime_matchers.go @@ -8,8 +8,13 @@ type RuntimeMatchersConfig struct { TenantSpecificMatchers map[string]ActiveSeriesMatchers `yaml:"tenant_matchers"` } -// UnmarshalYAML implements the yaml.Unmarshaler interface. If give -func (l *RuntimeMatchersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - type plain RuntimeMatchersConfig // type indirection to make sure we don't go into recursive loop - return unmarshal((*plain)(l)) +type RuntimeMatchersConfigProvider struct { + Getter func() *RuntimeMatchersConfig +} + +func (p *RuntimeMatchersConfigProvider) Get() *RuntimeMatchersConfig { + if p == nil || p.Getter == nil { + return nil + } + return p.Getter() } diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 98f0e86990..acb5938bbd 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -420,7 +420,7 @@ func (t *Mimir) initIngesterService() (serv services.Service, err error) { t.Cfg.Ingester.LifecyclerConfig.ListenPort = t.Cfg.Server.GRPCListenPort t.Cfg.Ingester.StreamTypeFn = ingesterChunkStreaming(t.RuntimeConfig) t.Cfg.Ingester.InstanceLimitsFn = ingesterInstanceLimits(t.RuntimeConfig) - t.Cfg.Ingester.RuntimeMatchersConfigFn = runtimeMatchersConfig(t.RuntimeConfig) + t.Cfg.Ingester.RuntimeMatchersConfigProvider = runtimeMatchersConfig(t.RuntimeConfig) t.tsdbIngesterConfig() t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger) diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index 8fd558df9c..a347490954 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -147,17 +147,19 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeMatchersConfig(manager *runtimeconfig.Manager) func() *ingester.RuntimeMatchersConfig { +func runtimeMatchersConfig(manager *runtimeconfig.Manager) *RuntimeMatchersConfigProvider { if manager == nil { return nil } - return func() *ingester.RuntimeMatchersConfig { - val := manager.GetConfig() - if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { - return cfg.RuntimeMatchersConfig - } - return nil + return &RuntimeMatchersConfigProvider{ + Getter: func() *ingester.RuntimeMatchersConfig { + val := manager.GetConfig() + if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { + return cfg.RuntimeMatchersConfig + } + return nil + }, } } From 28b4ff0763f0b00fa08fec75a5c47387d8f39473 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 17 Feb 2022 14:50:10 +0100 Subject: [PATCH 23/91] fixup --- pkg/ingester/ingester.go | 1 - pkg/mimir/runtime_config.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index ada2a7bb86..e0b92b2c86 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -199,7 +199,6 @@ type Ingester struct { metrics *ingesterMetrics logger log.Logger - // Now it is used to track default matcher activeSeriesMatcher *ActiveSeriesMatchers lifecycler *ring.Lifecycler diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index a347490954..ad58ff84e5 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -147,12 +147,12 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeMatchersConfig(manager *runtimeconfig.Manager) *RuntimeMatchersConfigProvider { +func runtimeMatchersConfig(manager *runtimeconfig.Manager) *ingester.RuntimeMatchersConfigProvider { if manager == nil { return nil } - return &RuntimeMatchersConfigProvider{ + return &ingester.RuntimeMatchersConfigProvider{ Getter: func() *ingester.RuntimeMatchersConfig { val := manager.GetConfig() if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { From e2250a9d167f8543ed1dbe09c316462bde25a13b Mon Sep 17 00:00:00 2001 From: Janos Date: Fri, 18 Feb 2022 08:26:53 +0100 Subject: [PATCH 24/91] fixup --- pkg/ingester/ingester.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index e0b92b2c86..be9fe1e88d 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -486,7 +486,7 @@ func (i *Ingester) reloadConfig(now time.Time) { } newMatchers := i.getActiveSeriesMatchers(userID, currentConfig) if !newMatchers.Equals(userDB.activeSeries.asm) { - i.ReplaceMatchers(newMatchers, userDB) + i.replaceMatchers(newMatchers, userDB) } } } @@ -506,7 +506,7 @@ func (i *Ingester) getActiveSeriesMatchers(userID string, config *RuntimeMatcher return &val } -func (i *Ingester) ReplaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) { +func (i *Ingester) replaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) { i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.asm.names) userDB.activeSeries.ReloadSeriesMatchers(asm) } From 378c86e957ed7163cad8f7a00730f99b9b46a770 Mon Sep 17 00:00:00 2001 From: Janos Date: Fri, 18 Feb 2022 17:02:58 +0100 Subject: [PATCH 25/91] Incorporating oleg's suggestions --- pkg/ingester/active_series.go | 4 ++ pkg/ingester/active_series_custom_tracker.go | 61 ++++++++++-------- .../active_series_custom_tracker_test.go | 43 ++++++------- .../active_series_runtime_overwrites.go | 27 ++++++++ ... active_series_runtime_overwrites_test.go} | 6 +- pkg/ingester/active_series_test.go | 6 +- pkg/ingester/ingester.go | 63 ++++++------------- pkg/ingester/ingester_test.go | 50 ++++++++------- pkg/ingester/runtime_matchers.go | 20 ------ pkg/mimir/runtime_config.go | 8 +-- 10 files changed, 142 insertions(+), 146 deletions(-) create mode 100644 pkg/ingester/active_series_runtime_overwrites.go rename pkg/ingester/{runtime_matchers_test.go => active_series_runtime_overwrites_test.go} (87%) delete mode 100644 pkg/ingester/runtime_matchers.go diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index fd6c64effe..6e0a6a0275 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -64,6 +64,10 @@ func NewActiveSeries(asm *ActiveSeriesMatchers) *ActiveSeries { return c } +func (c *ActiveSeries) CurrentMatchers() *ActiveSeriesMatchers { + return c.asm +} + func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers) { c.asm = asm for i := 0; i < numActiveSeriesStripes; i++ { diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index acc347ce88..608528b518 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -11,29 +11,31 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -// ActiveSeriesCustomTrackersConfig configures the additional custom trackers for active series in the ingester. -type ActiveSeriesCustomTrackersConfig map[string]string - -func (c *ActiveSeriesCustomTrackersConfig) String() string { - if *c == nil { +func (asm *ActiveSeriesMatchers) String() string { + if asm == nil { return "" } - strs := make([]string, 0, len(*c)) - for name, matcher := range *c { - strs = append(strs, fmt.Sprintf("%s:%s", name, matcher)) + var sb strings.Builder + for i, name := range asm.names { + sb.WriteString(name) + for _, labelMatcher := range asm.matchers[i] { + sb.WriteString(labelMatcher.String()) + } } - return strings.Join(strs, ";") + return sb.String() } -func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { +func (asm *ActiveSeriesMatchers) Set(s string) error { if strings.TrimSpace(s) == "" { return nil } - if *c == nil { - *c = map[string]string{} + if len(asm.names) != 0 { + return fmt.Errorf("can't provide active series custom trackers flag multple times") } + c := map[string]string{} + pairs := strings.Split(s, ";") for i, p := range pairs { split := strings.SplitN(p, ":", 2) @@ -44,26 +46,33 @@ func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { if len(name) == 0 || len(matcher) == 0 { return fmt.Errorf("semicolon-separated values should be :, but one of the sides was empty in the value %d: %q", i, p) } - if _, ok := (*c)[name]; ok { + if _, ok := c[name]; ok { return fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) } - (*c)[name] = matcher + c[name] = matcher + } + + a, err := NewActiveSeriesMatchers(c) + if err != nil { + return err } + *asm = *a + return nil } -func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { +func (c *ActiveSeriesMatchers) ExampleDoc() (comment string, yaml interface{}) { return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, - ActiveSeriesCustomTrackersConfig{ + map[string]string{ "dev": `{namespace=~"dev-.*"}`, "prod": `{namespace=~"prod-.*"}`, } } -func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*ActiveSeriesMatchers, error) { +func NewActiveSeriesMatchers(matchersConfig map[string]string) (*ActiveSeriesMatchers, error) { asm := &ActiveSeriesMatchers{} - for name, matcher := range matchers { + for name, matcher := range matchersConfig { sm, err := amlabels.ParseMatchers(matcher) if err != nil { return nil, fmt.Errorf("can't build active series matcher %s: %w", name, err) @@ -79,15 +88,9 @@ func NewActiveSeriesMatchers(matchers ActiveSeriesCustomTrackersConfig) (*Active // Sort the result to make it deterministic for tests. // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) - var sb strings.Builder // The concatenation should happen after ordering, to ensure equality is not dependent on map traversal. - for i, name := range asm.names { - sb.WriteString(name) - for _, labelMatcher := range asm.matchers[i] { - sb.WriteString(labelMatcher.String()) - } - } - asm.config = sb.String() + asm.config = asm.String() + return asm, nil } @@ -98,12 +101,16 @@ type ActiveSeriesMatchers struct { } func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { + if asm == nil || other == nil { + return asm == other + } return asm.config == other.config } // UnmarshalYAML implements the yaml.Unmarshaler interface. +// ActiveSeriesMatchers are marshaled in yaml as a map, with matcher names as keys and strings as matchers definitions. func (asm *ActiveSeriesMatchers) UnmarshalYAML(unmarshal func(interface{}) error) error { - m := ActiveSeriesCustomTrackersConfig{} + m := map[string]string{} err := unmarshal(&m) if err != nil { return err diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 8742bcb623..7a91c096f8 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -16,16 +16,21 @@ import ( ) func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { + safeActiveSeriesMatchers := func(input map[string]string) *ActiveSeriesMatchers { + res, err := NewActiveSeriesMatchers(input) + assert.NoError(t, err) + return res + } for _, tc := range []struct { name string flags []string - expected ActiveSeriesCustomTrackersConfig + expected *ActiveSeriesMatchers error error }{ { name: "empty flag value produces empty config", flags: []string{`-ingester.active-series-custom-trackers=`}, - expected: nil, + expected: &ActiveSeriesMatchers{}, }, { name: "empty matcher fails", @@ -55,22 +60,22 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "one matcher", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`}, - expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`}, + expected: safeActiveSeriesMatchers(map[string]string{`foo`: `{foo="bar"}`}), }, { name: "whitespaces are trimmed from name and matcher", flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, - expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`}, + expected: safeActiveSeriesMatchers(map[string]string{`foo`: `{foo="bar"}`}), }, { name: "two matchers in one flag value", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};baz:{baz="bar"}`}, - expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}, + expected: safeActiveSeriesMatchers(map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), }, { - name: "two matchers in two flag values", - flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, - expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}, + name: "two matchers in two flag values", + flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, + error: errors.New("invalid value \"baz:{baz=\\\"bar\\\"}\" for flag -ingester.active-series-custom-trackers: can't provide active series custom trackers flag multple times"), }, { name: "two matchers with same name in same flag", @@ -80,27 +85,27 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "two matchers with same name in separate flags", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=foo:{boo="bam"}`}, - error: errors.New(`invalid value "foo:{boo=\"bam\"}" for flag -ingester.active-series-custom-trackers: matcher "foo" for active series custom trackers is provided twice`), + error: errors.New("invalid value \"foo:{boo=\\\"bam\\\"}\" for flag -ingester.active-series-custom-trackers: can't provide active series custom trackers flag multple times"), }, } { t.Run(tc.name, func(t *testing.T) { flagSet := flag.NewFlagSet("test", flag.ContinueOnError) - var config ActiveSeriesCustomTrackersConfig + var config ActiveSeriesMatchers flagSet.Var(&config, "ingester.active-series-custom-trackers", "...usage docs...") err := flagSet.Parse(tc.flags) if tc.error != nil { assert.EqualError(t, err, tc.error.Error()) } else { - assert.Equal(t, tc.expected, config) + assert.True(t, tc.expected.Equals(&config)) } }) } } func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { - config := ActiveSeriesCustomTrackersConfig{ + config := map[string]string{ "bar_starts_with_1": `{bar=~"1.*"}`, "does_not_have_foo_label": `{foo=""}`, "has_foo_and_bar_starts_with_1": `{foo!="", bar=~"1.*"}`, @@ -182,7 +187,7 @@ func TestActiveSeriesMatcher_MalformedMatcher(t *testing.T) { `{foo=~"}`, } { t.Run(matcher, func(t *testing.T) { - config := ActiveSeriesCustomTrackersConfig{ + config := map[string]string{ "malformed": matcher, } @@ -211,10 +216,8 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { t.Run("EqualityBetweenSet", func(t *testing.T) { var activeSeriesMatchers []*ActiveSeriesMatchers for _, matcherConfig := range matcherSet { - config := ActiveSeriesCustomTrackersConfig{} - err := config.Set(matcherConfig) - assert.NoError(t, err) - asm, err := NewActiveSeriesMatchers(config) + asm := &ActiveSeriesMatchers{} + err := asm.Set(matcherConfig) assert.NoError(t, err) activeSeriesMatchers = append(activeSeriesMatchers, asm) } @@ -230,10 +233,8 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { var activeSeriesMatchers []*ActiveSeriesMatchers for _, matcherConfigs := range matcherSets { exampleConfig := matcherConfigs[0] - config := ActiveSeriesCustomTrackersConfig{} - err := config.Set(exampleConfig) - assert.NoError(t, err) - asm, err := NewActiveSeriesMatchers(config) + asm := &ActiveSeriesMatchers{} + err := asm.Set(exampleConfig) assert.NoError(t, err) activeSeriesMatchers = append(activeSeriesMatchers, asm) } diff --git a/pkg/ingester/active_series_runtime_overwrites.go b/pkg/ingester/active_series_runtime_overwrites.go new file mode 100644 index 0000000000..6ce3fcb508 --- /dev/null +++ b/pkg/ingester/active_series_runtime_overwrites.go @@ -0,0 +1,27 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package ingester + +// ActiveSeriesCustomTrackersOverrides holds the definition of custom tracking rules. +type ActiveSeriesCustomTrackersOverrides struct { + Default *ActiveSeriesMatchers `yaml:"default"` + TenantSpecific map[string]*ActiveSeriesMatchers `yaml:"tenant_specific"` +} + +func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersForUser(userID string) *ActiveSeriesMatchers { + if tenantspecific, ok := asmo.TenantSpecific[userID]; ok { + return tenantspecific + } + return asmo.Default +} + +type ActiveSeriesCustomTrackersOverridesProvider struct { + Getter func() *ActiveSeriesCustomTrackersOverrides +} + +func (p *ActiveSeriesCustomTrackersOverridesProvider) Get() *ActiveSeriesCustomTrackersOverrides { + if p == nil || p.Getter == nil { + return nil + } + return p.Getter() +} diff --git a/pkg/ingester/runtime_matchers_test.go b/pkg/ingester/active_series_runtime_overwrites_test.go similarity index 87% rename from pkg/ingester/runtime_matchers_test.go rename to pkg/ingester/active_series_runtime_overwrites_test.go index 7f37d47e36..86616119d4 100644 --- a/pkg/ingester/runtime_matchers_test.go +++ b/pkg/ingester/active_series_runtime_overwrites_test.go @@ -10,12 +10,12 @@ import ( ) func TestRuntimeMatchersUnmarshal(t *testing.T) { - r := RuntimeMatchersConfig{} + r := ActiveSeriesCustomTrackersOverrides{} input := ` -default_matchers: +default: integrations/apolloserver: "{job='integrations/apollo-server'}" integrations/caddy: "{job='integrations/caddy'}" -tenant_matchers: +tenant_specific: 1: team_A: "{grafanacloud_team='team_a'}" team_B: "{grafanacloud_team='team_b'}" diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index c8118ad8ce..f4a0624007 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -50,7 +50,7 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { ls2 := []labels.Label{{Name: "a", Value: "2"}} ls3 := []labels.Label{{Name: "a", Value: "3"}} - asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{a=~"2|3"}`}) + asm, err := NewActiveSeriesMatchers(map[string]string{"foo": `{a=~"2|3"}`}) require.NoError(t, err) c := NewActiveSeries(asm) @@ -133,7 +133,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { {{Name: "_", Value: "KiqbryhzUpn"}, {Name: "__name__", Value: "logs"}}, } - asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{_=~"y.*"}`}) + asm, err := NewActiveSeriesMatchers(map[string]string{"foo": `{_=~"y.*"}`}) require.NoError(t, err) // Run the same test for increasing TTL values @@ -198,7 +198,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{a=~.*}`}) + asm, err := NewActiveSeriesMatchers(map[string]string{"foo": `{a=~.*}`}) assert.NoError(t, err) c := NewActiveSeries(asm) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index be9fe1e88d..95c9ed16c0 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -121,11 +121,11 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackers ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)."` - RuntimeMatchersConfigProvider *RuntimeMatchersConfigProvider `yaml:"-"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesCustomTrackers ActiveSeriesMatchers `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)."` + RuntimeMatchersConfigProvider *ActiveSeriesCustomTrackersOverridesProvider `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -251,10 +251,9 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus } return &Ingester{ - cfg: cfg, - limits: limits, - logger: logger, - activeSeriesMatcher: &ActiveSeriesMatchers{}, + cfg: cfg, + limits: limits, + logger: logger, tsdbs: make(map[string]*userTSDB), usersMetadata: make(map[string]*userMetricsMetadata), @@ -282,12 +281,6 @@ func New(cfg Config, clientConfig client.Config, limits *validation.Overrides, r i.ingestionRate = util_math.NewEWMARate(0.2, instanceIngestionRateTickInterval) i.metrics = newIngesterMetrics(registerer, cfg.ActiveSeriesMetricsEnabled, i.getInstanceLimits, i.ingestionRate, &i.inflightPushRequests) - asm, err := NewActiveSeriesMatchers(cfg.ActiveSeriesCustomTrackers) - if err != nil { - return nil, err - } - i.activeSeriesMatcher = asm - // Replace specific metrics which we can't directly track but we need to read // them from the underlying system (ie. TSDB). if registerer != nil { @@ -466,7 +459,6 @@ func (i *Ingester) updateLoop(ctx context.Context) error { i.applyExemplarsSettings() case <-activeSeriesTickerChan: - i.reloadConfig(time.Now()) i.updateActiveSeries(time.Now()) case <-ctx.Done(): @@ -477,37 +469,15 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) reloadConfig(now time.Time) { - currentConfig := i.cfg.RuntimeMatchersConfigProvider.Get() - for _, userID := range i.getTSDBUsers() { - userDB := i.getTSDB(userID) - if userDB == nil { - continue - } - newMatchers := i.getActiveSeriesMatchers(userID, currentConfig) - if !newMatchers.Equals(userDB.activeSeries.asm) { - i.replaceMatchers(newMatchers, userDB) - } - } -} - -func (i *Ingester) getActiveSeriesMatchers(userID string, config *RuntimeMatchersConfig) *ActiveSeriesMatchers { - if config == nil { - return i.activeSeriesMatcher - } - val, ok := config.TenantSpecificMatchers[userID] - if !ok { - if !i.activeSeriesMatcher.Equals(&config.DefaultMatchers) { - // this avoids referencing multiple instances of deserialized DefaultMatchers - i.activeSeriesMatcher = &config.DefaultMatchers - } - return i.activeSeriesMatcher +func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers { + if cfg := i.cfg.RuntimeMatchersConfigProvider.Get(); cfg != nil { + return cfg.MatchersForUser(userID) } - return &val + return &i.cfg.ActiveSeriesCustomTrackers } func (i *Ingester) replaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) { - i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.asm.names) + i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatchers().names) userDB.activeSeries.ReloadSeriesMatchers(asm) } @@ -519,6 +489,11 @@ func (i *Ingester) updateActiveSeries(now time.Time) { continue } + newMatchers := i.getActiveSeriesMatchers(userID) + if !newMatchers.Equals(userDB.activeSeries.CurrentMatchers()) { + i.replaceMatchers(newMatchers, userDB) + } + userDB.activeSeries.Purge(purgeTime) allActive, activeMatching := userDB.activeSeries.Active() if allActive > 0 { @@ -1485,7 +1460,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - newMatchers := i.getActiveSeriesMatchers(userID, i.cfg.RuntimeMatchersConfigProvider.Get()) + newMatchers := i.getActiveSeriesMatchers(userID) userDB := &userTSDB{ userID: userID, diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index c8fa07f4af..fbf25b6153 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5144,39 +5144,41 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultRuntimeMatcherConfigFn := &RuntimeMatchersConfigProvider{ - func() *RuntimeMatchersConfig { - defaultMatchers, _ := NewActiveSeriesMatchers(map[string]string{ + defaultRuntimeMatcherConfigFn := &ActiveSeriesCustomTrackersOverridesProvider{ + func() *ActiveSeriesCustomTrackersOverrides { + defaultMatchers, err := NewActiveSeriesMatchers(map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }) - teamMatchers, _ := NewActiveSeriesMatchers(map[string]string{ + assert.NoError(t, err) + teamMatchers, err := NewActiveSeriesMatchers(map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }) - return &RuntimeMatchersConfig{ - DefaultMatchers: *defaultMatchers, - TenantSpecificMatchers: map[string]ActiveSeriesMatchers{ - "test_user": *teamMatchers, + assert.NoError(t, err) + return &ActiveSeriesCustomTrackersOverrides{ + Default: defaultMatchers, + TenantSpecific: map[string]*ActiveSeriesMatchers{ + "test_user": teamMatchers, }, } }, } - activeSeriesDefaultConfig := map[string]string{ + activeSeriesDefaultConfig, err := NewActiveSeriesMatchers(map[string]string{ "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, - } - + }) + assert.NoError(t, err) tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest expectedMetrics string disableActiveSeries bool - runtimeMatchersConfigProvider *RuntimeMatchersConfigProvider - activeSeriesConfig ActiveSeriesCustomTrackersConfig + activeSeriesOverridesProvider *ActiveSeriesCustomTrackersOverridesProvider + activeSeriesConfig ActiveSeriesMatchers }{ "successful push, should count active series": { - runtimeMatchersConfigProvider: defaultRuntimeMatcherConfigFn, + activeSeriesOverridesProvider: defaultRuntimeMatcherConfigFn, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5214,7 +5216,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should track custom matchers, removing when zero": { - runtimeMatchersConfigProvider: defaultRuntimeMatcherConfigFn, + activeSeriesOverridesProvider: defaultRuntimeMatcherConfigFn, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5289,7 +5291,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "successful push, active series disabled": { - runtimeMatchersConfigProvider: defaultRuntimeMatcherConfigFn, + activeSeriesOverridesProvider: defaultRuntimeMatcherConfigFn, disableActiveSeries: true, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5317,8 +5319,8 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with empty runtime config": { - runtimeMatchersConfigProvider: &RuntimeMatchersConfigProvider{ - func() *RuntimeMatchersConfig { + activeSeriesOverridesProvider: &ActiveSeriesCustomTrackersOverridesProvider{ + func() *ActiveSeriesCustomTrackersOverrides { return nil }, }, @@ -5353,7 +5355,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with nil matchers config function": { - runtimeMatchersConfigProvider: nil, + activeSeriesOverridesProvider: nil, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5385,8 +5387,8 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should use flag based custom tracker if no runtime config specified": { - runtimeMatchersConfigProvider: nil, - activeSeriesConfig: activeSeriesDefaultConfig, + activeSeriesOverridesProvider: nil, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5424,8 +5426,8 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should use runtime matcher config if both specified": { - runtimeMatchersConfigProvider: defaultRuntimeMatcherConfigFn, - activeSeriesConfig: activeSeriesDefaultConfig, + activeSeriesOverridesProvider: defaultRuntimeMatcherConfigFn, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5472,7 +5474,7 @@ func TestIngesterActiveSeries(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.LifecyclerConfig.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - cfg.RuntimeMatchersConfigProvider = testData.runtimeMatchersConfigProvider + cfg.RuntimeMatchersConfigProvider = testData.activeSeriesOverridesProvider cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) diff --git a/pkg/ingester/runtime_matchers.go b/pkg/ingester/runtime_matchers.go deleted file mode 100644 index 68396fb526..0000000000 --- a/pkg/ingester/runtime_matchers.go +++ /dev/null @@ -1,20 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package ingester - -// RuntimeMatchers holds the definition of custom tracking rules -type RuntimeMatchersConfig struct { - DefaultMatchers ActiveSeriesMatchers `yaml:"default_matchers"` - TenantSpecificMatchers map[string]ActiveSeriesMatchers `yaml:"tenant_matchers"` -} - -type RuntimeMatchersConfigProvider struct { - Getter func() *RuntimeMatchersConfig -} - -func (p *RuntimeMatchersConfigProvider) Get() *RuntimeMatchersConfig { - if p == nil || p.Getter == nil { - return nil - } - return p.Getter() -} diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index ad58ff84e5..b33b569a7d 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -35,7 +35,7 @@ type runtimeConfigValues struct { IngesterLimits *ingester.InstanceLimits `yaml:"ingester_limits"` - RuntimeMatchersConfig *ingester.RuntimeMatchersConfig `yaml:"runtime_matchers"` + RuntimeMatchersConfig *ingester.ActiveSeriesCustomTrackersOverrides `yaml:"runtime_matchers"` } // runtimeConfigTenantLimits provides per-tenant limit overrides based on a runtimeconfig.Manager @@ -147,13 +147,13 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeMatchersConfig(manager *runtimeconfig.Manager) *ingester.RuntimeMatchersConfigProvider { +func runtimeMatchersConfig(manager *runtimeconfig.Manager) *ingester.ActiveSeriesCustomTrackersOverridesProvider { if manager == nil { return nil } - return &ingester.RuntimeMatchersConfigProvider{ - Getter: func() *ingester.RuntimeMatchersConfig { + return &ingester.ActiveSeriesCustomTrackersOverridesProvider{ + Getter: func() *ingester.ActiveSeriesCustomTrackersOverrides { val := manager.GetConfig() if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { return cfg.RuntimeMatchersConfig From 858863a6dba4fa2c2c605e52bb518096fdaca506 Mon Sep 17 00:00:00 2001 From: Janos Date: Mon, 21 Feb 2022 14:22:12 +0100 Subject: [PATCH 26/91] Adding more tests --- pkg/ingester/active_series.go | 4 +- pkg/ingester/active_series_custom_tracker.go | 6 +- pkg/ingester/active_series_test.go | 2 +- pkg/ingester/ingester.go | 10 +- pkg/ingester/ingester_test.go | 217 ++++++++++++++++++- 5 files changed, 222 insertions(+), 17 deletions(-) diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index 6e0a6a0275..560f8ab348 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -68,7 +68,7 @@ func (c *ActiveSeries) CurrentMatchers() *ActiveSeriesMatchers { return c.asm } -func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers) { +func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers, reloadTime time.Time) { c.asm = asm for i := 0; i < numActiveSeriesStripes; i++ { c.stripes[i].mu.Lock() @@ -79,7 +79,7 @@ func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers) { c.stripes[i].asm = asm c.stripes[i].activeMatching = makeIntSliceIfNotEmpty(len(asm.MatcherNames()), c.stripes[i].activeMatching) } - c.lastUpdate = time.Now() + c.lastUpdate = reloadTime } // Updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 608528b518..debc8f9ee2 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -89,13 +89,13 @@ func NewActiveSeriesMatchers(matchersConfig map[string]string) (*ActiveSeriesMat // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) // The concatenation should happen after ordering, to ensure equality is not dependent on map traversal. - asm.config = asm.String() + asm.key = asm.String() return asm, nil } type ActiveSeriesMatchers struct { - config string + key string names []string matchers []labelsMatchers } @@ -104,7 +104,7 @@ func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { if asm == nil || other == nil { return asm == other } - return asm.config == other.config + return asm.key == other.key } // UnmarshalYAML implements the yaml.Unmarshaler interface. diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index f4a0624007..980dee9582 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -212,7 +212,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { assert.Equal(t, []int{1}, activeMatching) reloadTime := time.Now() - c.ReloadSeriesMatchers(asm) + c.ReloadSeriesMatchers(asm, reloadTime) c.UpdateSeries(ls2, time.Now(), copyFn) allActive, activeMatching = c.Active() assert.Equal(t, 2, allActive, "reloading should not affect general counter") diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 95c9ed16c0..42879ff4fc 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -476,9 +476,9 @@ func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers return &i.cfg.ActiveSeriesCustomTrackers } -func (i *Ingester) replaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) { +func (i *Ingester) replaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB, now time.Time) { i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatchers().names) - userDB.activeSeries.ReloadSeriesMatchers(asm) + userDB.activeSeries.ReloadSeriesMatchers(asm, now) } func (i *Ingester) updateActiveSeries(now time.Time) { @@ -491,7 +491,7 @@ func (i *Ingester) updateActiveSeries(now time.Time) { newMatchers := i.getActiveSeriesMatchers(userID) if !newMatchers.Equals(userDB.activeSeries.CurrentMatchers()) { - i.replaceMatchers(newMatchers, userDB) + i.replaceMatchers(newMatchers, userDB, now) } userDB.activeSeries.Purge(purgeTime) @@ -502,8 +502,8 @@ func (i *Ingester) updateActiveSeries(now time.Time) { i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) } if userDB.activeSeries.lastUpdate.Before(purgeTime) { - // Do not publish metrics until the new matcher setup had time to catch up - // LastUpdate is Zero when it never get updated + // Do not publish metrics until the new matcher setup had time to catch up. + // LastUpdate is Zero when it never get updated. for idx, name := range userDB.activeSeries.asm.names { // We only set the metrics for matchers that actually exist, to avoid increasing cardinality with zero valued metrics. if activeMatching[idx] > 0 { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index fbf25b6153..10d06ead65 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5144,7 +5144,7 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultRuntimeMatcherConfigFn := &ActiveSeriesCustomTrackersOverridesProvider{ + defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { defaultMatchers, err := NewActiveSeriesMatchers(map[string]string{ "bool_is_true": `{bool="true"}`, @@ -5178,7 +5178,7 @@ func TestIngesterActiveSeries(t *testing.T) { activeSeriesConfig ActiveSeriesMatchers }{ "successful push, should count active series": { - activeSeriesOverridesProvider: defaultRuntimeMatcherConfigFn, + activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5216,7 +5216,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should track custom matchers, removing when zero": { - activeSeriesOverridesProvider: defaultRuntimeMatcherConfigFn, + activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5291,7 +5291,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "successful push, active series disabled": { - activeSeriesOverridesProvider: defaultRuntimeMatcherConfigFn, + activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, disableActiveSeries: true, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5354,7 +5354,7 @@ func TestIngesterActiveSeries(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, }, - "should not fail with nil matchers config function": { + "should not fail with nil provider": { activeSeriesOverridesProvider: nil, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5426,7 +5426,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should use runtime matcher config if both specified": { - activeSeriesOverridesProvider: defaultRuntimeMatcherConfigFn, + activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5492,6 +5492,211 @@ func TestIngesterActiveSeries(t *testing.T) { } } +func TestIngesterActiveSeriesConfigChanges(t *testing.T) { + labelsToPush := []labels.Labels{ + labels.FromStrings(labels.MetricName, "test_metric", "bool", "false", "team", "a"), + labels.FromStrings(labels.MetricName, "test_metric", "bool", "false", "team", "b"), + labels.FromStrings(labels.MetricName, "test_metric", "bool", "true", "team", "a"), + labels.FromStrings(labels.MetricName, "test_metric", "bool", "true", "team", "b"), + } + + req := func(lbls labels.Labels, t time.Time) *mimirpb.WriteRequest { + return mimirpb.ToWriteRequest( + []labels.Labels{lbls}, + []mimirpb.Sample{{Value: 1, TimestampMs: t.UnixMilli()}}, + nil, + nil, + mimirpb.API, + ) + } + + metricNames := []string{ + "cortex_ingester_active_series", + "cortex_ingester_active_series_custom_tracker", + } + userID := "test_user" + + defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ + func() *ActiveSeriesCustomTrackersOverrides { + defaultMatchers, err := NewActiveSeriesMatchers(map[string]string{ + "bool_is_true": `{bool="true"}`, + "bool_is_false": `{bool="false"}`, + }) + assert.NoError(t, err) + teamMatchers, err := NewActiveSeriesMatchers(map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }) + assert.NoError(t, err) + return &ActiveSeriesCustomTrackersOverrides{ + Default: defaultMatchers, + TenantSpecific: map[string]*ActiveSeriesMatchers{ + "test_user": teamMatchers, + }, + } + }, + } + activeSeriesDefaultConfig, err := NewActiveSeriesMatchers(map[string]string{ + "bool_is_true_flagbased": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + }) + assert.NoError(t, err) + + tests := map[string]struct { + test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) + reqs []*mimirpb.WriteRequest + expectedMetrics string + activeSeriesOverridesProvider *ActiveSeriesCustomTrackersOverridesProvider + activeSeriesConfig ActiveSeriesMatchers + }{ + "flag based config to runtime overwrite": { + activeSeriesOverridesProvider: nil, + activeSeriesConfig: *activeSeriesDefaultConfig, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + firstPushTime := time.Now() + + for _, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + //offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now())) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(firstPushTime) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="test_user"} 2 + ` + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + + // "Remove" runtime configs + ingester.cfg.RuntimeMatchersConfigProvider = defaultCustomTrackersOverridesProvider + ingester.updateActiveSeries(firstPushTime) + expectedMetrics = ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + ` + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + // After the MetricsTimeout is over the default configuration should be used to expose metrics + time.Sleep(time.Millisecond) + secondPushtime := time.Now() + for _, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + _, err := ingester.Push(ctx, req(label, time.Now())) + require.NoError(t, err) + } + time.Sleep(time.Millisecond) + ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) + expectedMetrics = ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + ` + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + time.Sleep(time.Millisecond) + }, + }, + "runtime overwrite to flag based config": { + activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, + activeSeriesConfig: *activeSeriesDefaultConfig, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + firstPushTime := time.Now() + + for _, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + //offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now())) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(firstPushTime) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + ` + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + + // "Remove" runtime configs + ingester.cfg.RuntimeMatchersConfigProvider = nil + ingester.updateActiveSeries(firstPushTime) + expectedMetrics = ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + ` + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + // After the MetricsTimeout is over the default configuration should be used to expose metrics + time.Sleep(time.Millisecond) + secondPushtime := time.Now() + for _, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + _, err := ingester.Push(ctx, req(label, time.Now())) + require.NoError(t, err) + } + time.Sleep(time.Millisecond) + ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) + expectedMetrics = ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="test_user"} 2 + ` + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + time.Sleep(time.Millisecond) + }, + }, + } + + for testName, testData := range tests { + t.Run(testName, func(t *testing.T) { + registry := prometheus.NewRegistry() + + // Create a mocked ingester + cfg := defaultIngesterTestConfig(t) + cfg.LifecyclerConfig.JoinAfter = 0 + cfg.ActiveSeriesMetricsEnabled = true + cfg.RuntimeMatchersConfigProvider = testData.activeSeriesOverridesProvider + cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig + + ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) + require.NoError(t, err) + require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) + defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck + + // Wait until the ingester is healthy + test.Poll(t, 100*time.Millisecond, 1, func() interface{} { + return ing.lifecycler.HealthyInstancesCount() + }) + + testData.test(t, ing, registry) + }) + } +} + func TestGetIgnoreSeriesLimitForMetricNamesMap(t *testing.T) { cfg := Config{} From 66e6ea137c5eaaa7fb87db55b9dbd35d4c752610 Mon Sep 17 00:00:00 2001 From: Janos Date: Mon, 21 Feb 2022 14:28:50 +0100 Subject: [PATCH 27/91] cleanup --- ....go => active_series_runtime_overrides.go} | 0 ...> active_series_runtime_overrides_test.go} | 2 +- pkg/ingester/ingester.go | 12 ++++----- pkg/ingester/ingester_test.go | 27 +++++++++---------- pkg/mimir/modules.go | 2 +- pkg/mimir/runtime_config.go | 6 ++--- 6 files changed, 23 insertions(+), 26 deletions(-) rename pkg/ingester/{active_series_runtime_overwrites.go => active_series_runtime_overrides.go} (100%) rename pkg/ingester/{active_series_runtime_overwrites_test.go => active_series_runtime_overrides_test.go} (90%) diff --git a/pkg/ingester/active_series_runtime_overwrites.go b/pkg/ingester/active_series_runtime_overrides.go similarity index 100% rename from pkg/ingester/active_series_runtime_overwrites.go rename to pkg/ingester/active_series_runtime_overrides.go diff --git a/pkg/ingester/active_series_runtime_overwrites_test.go b/pkg/ingester/active_series_runtime_overrides_test.go similarity index 90% rename from pkg/ingester/active_series_runtime_overwrites_test.go rename to pkg/ingester/active_series_runtime_overrides_test.go index 86616119d4..01a3cf706b 100644 --- a/pkg/ingester/active_series_runtime_overwrites_test.go +++ b/pkg/ingester/active_series_runtime_overrides_test.go @@ -9,7 +9,7 @@ import ( "gopkg.in/yaml.v2" ) -func TestRuntimeMatchersUnmarshal(t *testing.T) { +func TestRuntimeOverridesUnmarshal(t *testing.T) { r := ActiveSeriesCustomTrackersOverrides{} input := ` default: diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 42879ff4fc..e6579bd43c 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -121,11 +121,11 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackers ActiveSeriesMatchers `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)."` - RuntimeMatchersConfigProvider *ActiveSeriesCustomTrackersOverridesProvider `yaml:"-"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesCustomTrackers ActiveSeriesMatchers `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)."` + ActiveSeriesCustomTrackersOverrides *ActiveSeriesCustomTrackersOverridesProvider `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -470,7 +470,7 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers { - if cfg := i.cfg.RuntimeMatchersConfigProvider.Get(); cfg != nil { + if cfg := i.cfg.ActiveSeriesCustomTrackersOverrides.Get(); cfg != nil { return cfg.MatchersForUser(userID) } return &i.cfg.ActiveSeriesCustomTrackers diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 10d06ead65..8bc9dd4c92 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5474,7 +5474,7 @@ func TestIngesterActiveSeries(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.LifecyclerConfig.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - cfg.RuntimeMatchersConfigProvider = testData.activeSeriesOverridesProvider + cfg.ActiveSeriesCustomTrackersOverrides = testData.activeSeriesOverridesProvider cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) @@ -5557,7 +5557,6 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { for _, label := range labelsToPush { ctx := user.InjectOrgID(context.Background(), userID) - //offset := time.Duration(len(labelsToPush) - i) _, err := ingester.Push(ctx, req(label, time.Now())) require.NoError(t, err) } @@ -5577,8 +5576,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // "Remove" runtime configs - ingester.cfg.RuntimeMatchersConfigProvider = defaultCustomTrackersOverridesProvider + // Add new runtime configs + ingester.cfg.ActiveSeriesCustomTrackersOverrides = defaultCustomTrackersOverridesProvider ingester.updateActiveSeries(firstPushTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5586,15 +5585,15 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cortex_ingester_active_series{user="test_user"} 4 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // After the MetricsTimeout is over the default configuration should be used to expose metrics - time.Sleep(time.Millisecond) + secondPushtime := time.Now() + // Sleep here to ensure that the second batch of push happens after purgeTime to have entries for custom trackers + time.Sleep(time.Millisecond) for _, label := range labelsToPush { ctx := user.InjectOrgID(context.Background(), userID) _, err := ingester.Push(ctx, req(label, time.Now())) require.NoError(t, err) } - time.Sleep(time.Millisecond) ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5606,7 +5605,6 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - time.Sleep(time.Millisecond) }, }, "runtime overwrite to flag based config": { @@ -5637,8 +5635,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // "Remove" runtime configs - ingester.cfg.RuntimeMatchersConfigProvider = nil + // Remove runtime configs + ingester.cfg.ActiveSeriesCustomTrackersOverrides = nil ingester.updateActiveSeries(firstPushTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5646,15 +5644,15 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cortex_ingester_active_series{user="test_user"} 4 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // After the MetricsTimeout is over the default configuration should be used to expose metrics - time.Sleep(time.Millisecond) + secondPushtime := time.Now() + // Sleep here to ensure that the second batch of push happens after purgeTime to have entries for custom trackers + time.Sleep(time.Millisecond) for _, label := range labelsToPush { ctx := user.InjectOrgID(context.Background(), userID) _, err := ingester.Push(ctx, req(label, time.Now())) require.NoError(t, err) } - time.Sleep(time.Millisecond) ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5666,7 +5664,6 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="test_user"} 2 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - time.Sleep(time.Millisecond) }, }, } @@ -5679,7 +5676,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.LifecyclerConfig.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = true - cfg.RuntimeMatchersConfigProvider = testData.activeSeriesOverridesProvider + cfg.ActiveSeriesCustomTrackersOverrides = testData.activeSeriesOverridesProvider cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index acb5938bbd..9f70bcb505 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -420,7 +420,7 @@ func (t *Mimir) initIngesterService() (serv services.Service, err error) { t.Cfg.Ingester.LifecyclerConfig.ListenPort = t.Cfg.Server.GRPCListenPort t.Cfg.Ingester.StreamTypeFn = ingesterChunkStreaming(t.RuntimeConfig) t.Cfg.Ingester.InstanceLimitsFn = ingesterInstanceLimits(t.RuntimeConfig) - t.Cfg.Ingester.RuntimeMatchersConfigProvider = runtimeMatchersConfig(t.RuntimeConfig) + t.Cfg.Ingester.ActiveSeriesCustomTrackersOverrides = runtimeActiveSeriesCustomTrackersOverrides(t.RuntimeConfig) t.tsdbIngesterConfig() t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger) diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index b33b569a7d..18a0633ad0 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -35,7 +35,7 @@ type runtimeConfigValues struct { IngesterLimits *ingester.InstanceLimits `yaml:"ingester_limits"` - RuntimeMatchersConfig *ingester.ActiveSeriesCustomTrackersOverrides `yaml:"runtime_matchers"` + ActiveSeriesCustomTrackersOverrides *ingester.ActiveSeriesCustomTrackersOverrides `yaml:"active_series_custom_trackers_overrides"` } // runtimeConfigTenantLimits provides per-tenant limit overrides based on a runtimeconfig.Manager @@ -147,7 +147,7 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeMatchersConfig(manager *runtimeconfig.Manager) *ingester.ActiveSeriesCustomTrackersOverridesProvider { +func runtimeActiveSeriesCustomTrackersOverrides(manager *runtimeconfig.Manager) *ingester.ActiveSeriesCustomTrackersOverridesProvider { if manager == nil { return nil } @@ -156,7 +156,7 @@ func runtimeMatchersConfig(manager *runtimeconfig.Manager) *ingester.ActiveSerie Getter: func() *ingester.ActiveSeriesCustomTrackersOverrides { val := manager.GetConfig() if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { - return cfg.RuntimeMatchersConfig + return cfg.ActiveSeriesCustomTrackersOverrides } return nil }, From 1fcd42c58f49858300dcb91d022a65c663b1c5ff Mon Sep 17 00:00:00 2001 From: Janos Date: Mon, 21 Feb 2022 16:52:54 +0100 Subject: [PATCH 28/91] Adding testcase for empty default overwrite and fixing this corner case --- pkg/ingester/ingester.go | 8 ++++-- pkg/ingester/ingester_test.go | 54 +++++++++++++++++++++++++++++++++-- 2 files changed, 58 insertions(+), 4 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index e6579bd43c..68624f62fc 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -470,10 +470,14 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers { + var matchers *ActiveSeriesMatchers if cfg := i.cfg.ActiveSeriesCustomTrackersOverrides.Get(); cfg != nil { - return cfg.MatchersForUser(userID) + matchers = cfg.MatchersForUser(userID) } - return &i.cfg.ActiveSeriesCustomTrackers + if matchers == nil { + matchers = &i.cfg.ActiveSeriesCustomTrackers + } + return matchers } func (i *Ingester) replaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB, now time.Time) { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 8bc9dd4c92..278d1275c3 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5460,6 +5460,56 @@ func TestIngesterActiveSeries(t *testing.T) { cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 ` + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + }, + }, + "should revert to flag based default if only tenant-specific overwrite is present": { + activeSeriesOverridesProvider: &ActiveSeriesCustomTrackersOverridesProvider{ + func() *ActiveSeriesCustomTrackersOverrides { + teamMatchers, err := NewActiveSeriesMatchers(map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }) + assert.NoError(t, err) + return &ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*ActiveSeriesMatchers{ + "test_user": teamMatchers, + }} + }, + }, + activeSeriesConfig: *activeSeriesDefaultConfig, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + now := time.Now() + + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(now) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + ` + // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, @@ -5549,7 +5599,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { activeSeriesOverridesProvider *ActiveSeriesCustomTrackersOverridesProvider activeSeriesConfig ActiveSeriesMatchers }{ - "flag based config to runtime overwrite": { + "overwrite flag based config with runtime overwrite": { activeSeriesOverridesProvider: nil, activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { @@ -5607,7 +5657,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, }, - "runtime overwrite to flag based config": { + "remove runtime overwrite and revert to flag based config": { activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { From 3dd657edf86f29cee3a0a2ea4dbe9cd6eec62176 Mon Sep 17 00:00:00 2001 From: Janos Date: Mon, 21 Feb 2022 17:36:04 +0100 Subject: [PATCH 29/91] Test fixes --- pkg/ingester/active_series_test.go | 2 +- pkg/ingester/ingester_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index 980dee9582..52677e9440 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -217,7 +217,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { allActive, activeMatching = c.Active() assert.Equal(t, 2, allActive, "reloading should not affect general counter") assert.Equal(t, []int{1}, activeMatching, "reloading should clear out matcher counters") - assert.True(t, c.lastUpdate.After(reloadTime)) + assert.True(t, c.lastUpdate.Equal(reloadTime)) } var activeSeriesTestGoroutines = []int{50, 100, 500} diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 73209b4fdc..5dd8553c36 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5724,7 +5724,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { // Create a mocked ingester cfg := defaultIngesterTestConfig(t) - cfg.LifecyclerConfig.JoinAfter = 0 + cfg.IngesterRing.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = true cfg.ActiveSeriesCustomTrackersOverrides = testData.activeSeriesOverridesProvider cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig From 5209623cfddc290fc9f64bb488de9218628f058e Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 08:44:03 +0100 Subject: [PATCH 30/91] Test fixes, lint fixes --- cmd/mimir/help-all.txt.tmpl | 2 +- cmd/mimir/help.txt.tmpl | 4 ++++ pkg/ingester/active_series_custom_tracker.go | 2 +- pkg/ingester/active_series_custom_tracker_test.go | 8 ++++---- pkg/ingester/ingester.go | 4 +--- tools/doc-generator/parser.go | 3 --- 6 files changed, 11 insertions(+), 12 deletions(-) diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 429a3800f6..51ff2c3672 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -758,7 +758,7 @@ Usage of ./cmd/mimir/mimir: -http.prometheus-http-prefix string HTTP URL path under which the Prometheus api will be served. (default "/prometheus") -ingester.active-series-custom-trackers value - Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag. + Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be by providing multiple semicolon-separated values to a single flag. -ingester.active-series-metrics-enabled Enable tracking of active series and export them as metrics. (default true) -ingester.active-series-metrics-idle-timeout duration diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index d3add4d334..6c644b9764 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -267,6 +267,10 @@ Usage of ./cmd/mimir/mimir: Print basic help. -help-all Print help, also including advanced and experimental parameters. + -ingester.active-series-custom-trackers value + Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be by providing multiple semicolon-separated values to a single flag. + -ingester.active-series-metrics-enabled + Enable tracking of active series and export them as metrics. (default true) -ingester.max-global-metadata-per-metric int The maximum number of metadata per metric, across the cluster. 0 to disable. -ingester.max-global-metadata-per-user int diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index debc8f9ee2..5c3edaae79 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -61,7 +61,7 @@ func (asm *ActiveSeriesMatchers) Set(s string) error { return nil } -func (c *ActiveSeriesMatchers) ExampleDoc() (comment string, yaml interface{}) { +func (asm *ActiveSeriesMatchers) ExampleDoc() (comment string, yaml interface{}) { return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, map[string]string{ diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 7a91c096f8..f42851692c 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -249,24 +249,24 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { } func TestActiveSeriesMatcher_Deserialization(t *testing.T) { - correct_input := ` + correctInput := ` baz: "{baz='bar'}" foo: "{foo='bar'}" ` - malformed_input := + malformedInput := ` baz: "123" foo: "{foo='bar'}" ` t.Run("ShouldDeserializeCorrectInput", func(t *testing.T) { asm := ActiveSeriesMatchers{} - err := yaml.Unmarshal([]byte(correct_input), &asm) + err := yaml.Unmarshal([]byte(correctInput), &asm) assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") }) t.Run("ShouldErrorOnMalformedInput", func(t *testing.T) { asm := ActiveSeriesMatchers{} - err := yaml.Unmarshal([]byte(malformed_input), &asm) + err := yaml.Unmarshal([]byte(malformedInput), &asm) assert.Error(t, err, "should not deserialize malformed input") }) } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 864bdffb3d..de755157e6 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -153,7 +153,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { f.BoolVar(&cfg.ActiveSeriesMetricsEnabled, "ingester.active-series-metrics-enabled", true, "Enable tracking of active series and export them as metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsUpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsIdleTimeout, "ingester.active-series-metrics-idle-timeout", 10*time.Minute, "After what time a series is considered to be inactive.") - f.Var(&cfg.ActiveSeriesCustomTrackers, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag.") + f.Var(&cfg.ActiveSeriesCustomTrackers, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be by providing multiple semicolon-separated values to a single flag.") f.BoolVar(&cfg.StreamChunksWhenUsingBlocks, "ingester.stream-chunks-when-using-blocks", true, "Stream chunks from ingesters to queriers.") f.DurationVar(&cfg.ExemplarsUpdatePeriod, "ingester.exemplars-update-period", 15*time.Second, "Period with which to update per-user max exemplars.") @@ -198,8 +198,6 @@ type Ingester struct { metrics *ingesterMetrics logger log.Logger - activeSeriesMatcher *ActiveSeriesMatchers - lifecycler *ring.Lifecycler limits *validation.Overrides limiter *Limiter diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index 8437fd55d4..c6fe8b85ff 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -22,7 +22,6 @@ import ( "github.com/prometheus/prometheus/model/relabel" "github.com/weaveworks/common/logging" - "github.com/grafana/mimir/pkg/ingester" "github.com/grafana/mimir/pkg/util/fieldcategory" ) @@ -292,8 +291,6 @@ func getFieldType(t reflect.Type) (string, error) { return "string", nil case reflect.TypeOf([]*relabel.Config{}).String(): return "relabel_config...", nil - case reflect.TypeOf(ingester.ActiveSeriesCustomTrackersConfig{}).String(): - return "map of tracker name (string) to matcher (string)", nil } // Fallback to auto-detection of built-in data types From 5da5bce050117834c7b4e494d0e798ddbc6b5083 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 09:27:40 +0100 Subject: [PATCH 31/91] Adding more testcases --- .../active_series_runtime_overrides_test.go | 74 +++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/pkg/ingester/active_series_runtime_overrides_test.go b/pkg/ingester/active_series_runtime_overrides_test.go index 01a3cf706b..2294faf106 100644 --- a/pkg/ingester/active_series_runtime_overrides_test.go +++ b/pkg/ingester/active_series_runtime_overrides_test.go @@ -5,6 +5,7 @@ package ingester import ( "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v2" ) @@ -23,3 +24,76 @@ tenant_specific: require.NoError(t, yaml.UnmarshalStrict([]byte(input), &r)) } + +func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { + overridesReference := &ActiveSeriesCustomTrackersOverrides{} + tests := map[string]struct { + provider *ActiveSeriesCustomTrackersOverridesProvider + expected *ActiveSeriesCustomTrackersOverrides + }{ + "nil provider returns nil": { + provider: nil, + expected: nil, + }, + "nil getter returns nil": { + provider: &ActiveSeriesCustomTrackersOverridesProvider{}, + expected: nil, + }, + "getter is called": { + provider: &ActiveSeriesCustomTrackersOverridesProvider{ + Getter: func() *ActiveSeriesCustomTrackersOverrides { + return overridesReference + }, + }, + expected: overridesReference, + }, + } + + for name, testData := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, testData.expected, testData.provider.Get()) + }) + } +} + +func TestMatchersForUser(t *testing.T) { + defaultMatchers, err := NewActiveSeriesMatchers( + map[string]string{ + "foo": `{foo="bar"}`, + "bar": `{baz="bar"}`, + }) + + assert.NoError(t, err) + tenantSpecificMatchers, err := NewActiveSeriesMatchers( + map[string]string{ + "team_a": `{team="team_a"}`, + "team_b": `{team="team_b"}`, + }, + ) + assert.NoError(t, err) + activeSeriesCustomTrackersOverrides := &ActiveSeriesCustomTrackersOverrides{ + Default: defaultMatchers, + TenantSpecific: map[string]*ActiveSeriesMatchers{ + "1": tenantSpecificMatchers, + }, + } + tests := map[string]struct { + userID string + expected *ActiveSeriesMatchers + }{ + "User with no override should return deafult": { + userID: "5", + expected: defaultMatchers, + }, + "User with override should return override": { + userID: "1", + expected: tenantSpecificMatchers, + }, + } + for name, testData := range tests { + t.Run(name, func(t *testing.T) { + matchersForUser := activeSeriesCustomTrackersOverrides.MatchersForUser(testData.userID) + assert.True(t, testData.expected.Equals(matchersForUser)) + }) + } +} From 203d66edc11ff6277878bfb6c75bc4a7f9ef0e23 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 09:32:51 +0100 Subject: [PATCH 32/91] Changing assert.NoError to require.NoError when immediate halting is required --- pkg/ingester/active_series_custom_tracker_test.go | 6 +++--- .../active_series_runtime_overrides_test.go | 4 ++-- pkg/ingester/active_series_test.go | 2 +- pkg/ingester/ingester_test.go | 14 +++++++------- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index f42851692c..62865311cd 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -18,7 +18,7 @@ import ( func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { safeActiveSeriesMatchers := func(input map[string]string) *ActiveSeriesMatchers { res, err := NewActiveSeriesMatchers(input) - assert.NoError(t, err) + require.NoError(t, err) return res } for _, tc := range []struct { @@ -218,7 +218,7 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { for _, matcherConfig := range matcherSet { asm := &ActiveSeriesMatchers{} err := asm.Set(matcherConfig) - assert.NoError(t, err) + require.NoError(t, err) activeSeriesMatchers = append(activeSeriesMatchers, asm) } for i := 0; i < len(activeSeriesMatchers); i++ { @@ -235,7 +235,7 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { exampleConfig := matcherConfigs[0] asm := &ActiveSeriesMatchers{} err := asm.Set(exampleConfig) - assert.NoError(t, err) + require.NoError(t, err) activeSeriesMatchers = append(activeSeriesMatchers, asm) } diff --git a/pkg/ingester/active_series_runtime_overrides_test.go b/pkg/ingester/active_series_runtime_overrides_test.go index 2294faf106..26b8bf4d60 100644 --- a/pkg/ingester/active_series_runtime_overrides_test.go +++ b/pkg/ingester/active_series_runtime_overrides_test.go @@ -63,14 +63,14 @@ func TestMatchersForUser(t *testing.T) { "bar": `{baz="bar"}`, }) - assert.NoError(t, err) + require.NoError(t, err) tenantSpecificMatchers, err := NewActiveSeriesMatchers( map[string]string{ "team_a": `{team="team_a"}`, "team_b": `{team="team_b"}`, }, ) - assert.NoError(t, err) + require.NoError(t, err) activeSeriesCustomTrackersOverrides := &ActiveSeriesCustomTrackersOverrides{ Default: defaultMatchers, TenantSpecific: map[string]*ActiveSeriesMatchers{ diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index 52677e9440..7bd9168f9f 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -199,7 +199,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { ls2 := []labels.Label{{Name: "a", Value: "2"}} asm, err := NewActiveSeriesMatchers(map[string]string{"foo": `{a=~.*}`}) - assert.NoError(t, err) + require.NoError(t, err) c := NewActiveSeries(asm) allActive, activeMatching := c.Active() diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 5dd8553c36..3f8dc88103 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5150,12 +5150,12 @@ func TestIngesterActiveSeries(t *testing.T) { "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }) - assert.NoError(t, err) + require.NoError(t, err) teamMatchers, err := NewActiveSeriesMatchers(map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }) - assert.NoError(t, err) + require.NoError(t, err) return &ActiveSeriesCustomTrackersOverrides{ Default: defaultMatchers, TenantSpecific: map[string]*ActiveSeriesMatchers{ @@ -5168,7 +5168,7 @@ func TestIngesterActiveSeries(t *testing.T) { "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, }) - assert.NoError(t, err) + require.NoError(t, err) tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest @@ -5471,7 +5471,7 @@ func TestIngesterActiveSeries(t *testing.T) { "team_a": `{team="a"}`, "team_b": `{team="b"}`, }) - assert.NoError(t, err) + require.NoError(t, err) return &ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*ActiveSeriesMatchers{ "test_user": teamMatchers, }} @@ -5572,12 +5572,12 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }) - assert.NoError(t, err) + require.NoError(t, err) teamMatchers, err := NewActiveSeriesMatchers(map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }) - assert.NoError(t, err) + require.NoError(t, err) return &ActiveSeriesCustomTrackersOverrides{ Default: defaultMatchers, TenantSpecific: map[string]*ActiveSeriesMatchers{ @@ -5590,7 +5590,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, }) - assert.NoError(t, err) + require.NoError(t, err) tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) From 60f67ce486fadbf7e039830fecf2a020ae960212 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 09:45:13 +0100 Subject: [PATCH 33/91] lint fix and merge resolve fix --- pkg/ingester/active_series_runtime_overrides_test.go | 2 +- pkg/ingester/ingester.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/active_series_runtime_overrides_test.go b/pkg/ingester/active_series_runtime_overrides_test.go index 26b8bf4d60..1f1bfd021f 100644 --- a/pkg/ingester/active_series_runtime_overrides_test.go +++ b/pkg/ingester/active_series_runtime_overrides_test.go @@ -81,7 +81,7 @@ func TestMatchersForUser(t *testing.T) { userID string expected *ActiveSeriesMatchers }{ - "User with no override should return deafult": { + "User with no override should return default": { userID: "5", expected: defaultMatchers, }, diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index de755157e6..3b46f38335 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -121,7 +121,7 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` ActiveSeriesCustomTrackers ActiveSeriesMatchers `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` From 40ea21001e2fb67d113d5e0e2ee476b7556db51c Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 11:22:57 +0100 Subject: [PATCH 34/91] Reintroducing ActiveSeriesCustomTrackerConfig because ingester config should not contain structs --- pkg/ingester/active_series_custom_tracker.go | 59 +++++++++++-------- .../active_series_custom_tracker_test.go | 39 ++++++------ pkg/ingester/ingester.go | 28 +++++---- pkg/ingester/ingester_test.go | 28 ++++----- tools/doc-generator/parser.go | 3 + 5 files changed, 86 insertions(+), 71 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 5c3edaae79..a2a4fa2927 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -11,31 +11,28 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -func (asm *ActiveSeriesMatchers) String() string { - if asm == nil { +type ActiveSeriesCustomTrackersConfig map[string]string + +func (c *ActiveSeriesCustomTrackersConfig) String() string { + if *c == nil { return "" } - var sb strings.Builder - for i, name := range asm.names { - sb.WriteString(name) - for _, labelMatcher := range asm.matchers[i] { - sb.WriteString(labelMatcher.String()) - } + strs := make([]string, 0, len(*c)) + for name, matcher := range *c { + strs = append(strs, fmt.Sprintf("%s:%s", name, matcher)) } - return sb.String() + return strings.Join(strs, ";") } -func (asm *ActiveSeriesMatchers) Set(s string) error { +func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { if strings.TrimSpace(s) == "" { return nil } - if len(asm.names) != 0 { - return fmt.Errorf("can't provide active series custom trackers flag multple times") + if *c == nil { + *c = map[string]string{} } - c := map[string]string{} - pairs := strings.Split(s, ";") for i, p := range pairs { split := strings.SplitN(p, ":", 2) @@ -46,31 +43,26 @@ func (asm *ActiveSeriesMatchers) Set(s string) error { if len(name) == 0 || len(matcher) == 0 { return fmt.Errorf("semicolon-separated values should be :, but one of the sides was empty in the value %d: %q", i, p) } - if _, ok := c[name]; ok { + if _, ok := (*c)[name]; ok { return fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) } - c[name] = matcher - } - - a, err := NewActiveSeriesMatchers(c) - if err != nil { - return err + (*c)[name] = matcher } - *asm = *a - return nil + _, err := NewActiveSeriesMatchers(*c) + return err } -func (asm *ActiveSeriesMatchers) ExampleDoc() (comment string, yaml interface{}) { +func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, - map[string]string{ + ActiveSeriesCustomTrackersConfig{ "dev": `{namespace=~"dev-.*"}`, "prod": `{namespace=~"prod-.*"}`, } } -func NewActiveSeriesMatchers(matchersConfig map[string]string) (*ActiveSeriesMatchers, error) { +func NewActiveSeriesMatchers(matchersConfig ActiveSeriesCustomTrackersConfig) (*ActiveSeriesMatchers, error) { asm := &ActiveSeriesMatchers{} for name, matcher := range matchersConfig { sm, err := amlabels.ParseMatchers(matcher) @@ -94,6 +86,21 @@ func NewActiveSeriesMatchers(matchersConfig map[string]string) (*ActiveSeriesMat return asm, nil } +func (asm *ActiveSeriesMatchers) String() string { + if asm == nil { + return "" + } + + var sb strings.Builder + for i, name := range asm.names { + sb.WriteString(name) + for _, labelMatcher := range asm.matchers[i] { + sb.WriteString(labelMatcher.String()) + } + } + return sb.String() +} + type ActiveSeriesMatchers struct { key string names []string diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 62865311cd..18eb2d0a66 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -16,21 +16,16 @@ import ( ) func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { - safeActiveSeriesMatchers := func(input map[string]string) *ActiveSeriesMatchers { - res, err := NewActiveSeriesMatchers(input) - require.NoError(t, err) - return res - } for _, tc := range []struct { name string flags []string - expected *ActiveSeriesMatchers + expected ActiveSeriesCustomTrackersConfig error error }{ { name: "empty flag value produces empty config", flags: []string{`-ingester.active-series-custom-trackers=`}, - expected: &ActiveSeriesMatchers{}, + expected: nil, }, { name: "empty matcher fails", @@ -60,22 +55,22 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "one matcher", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`}, - expected: safeActiveSeriesMatchers(map[string]string{`foo`: `{foo="bar"}`}), + expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`}, }, { name: "whitespaces are trimmed from name and matcher", flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, - expected: safeActiveSeriesMatchers(map[string]string{`foo`: `{foo="bar"}`}), + expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`}, }, { name: "two matchers in one flag value", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};baz:{baz="bar"}`}, - expected: safeActiveSeriesMatchers(map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), + expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}, }, { - name: "two matchers in two flag values", - flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, - error: errors.New("invalid value \"baz:{baz=\\\"bar\\\"}\" for flag -ingester.active-series-custom-trackers: can't provide active series custom trackers flag multple times"), + name: "two matchers in two flag values", + flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, + expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}, }, { name: "two matchers with same name in same flag", @@ -85,20 +80,20 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "two matchers with same name in separate flags", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=foo:{boo="bam"}`}, - error: errors.New("invalid value \"foo:{boo=\\\"bam\\\"}\" for flag -ingester.active-series-custom-trackers: can't provide active series custom trackers flag multple times"), + error: errors.New(`invalid value "foo:{boo=\"bam\"}" for flag -ingester.active-series-custom-trackers: matcher "foo" for active series custom trackers is provided twice`), }, } { t.Run(tc.name, func(t *testing.T) { flagSet := flag.NewFlagSet("test", flag.ContinueOnError) - var config ActiveSeriesMatchers + var config ActiveSeriesCustomTrackersConfig flagSet.Var(&config, "ingester.active-series-custom-trackers", "...usage docs...") err := flagSet.Parse(tc.flags) if tc.error != nil { assert.EqualError(t, err, tc.error.Error()) } else { - assert.True(t, tc.expected.Equals(&config)) + assert.Equal(t, tc.expected, config) } }) } @@ -216,8 +211,10 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { t.Run("EqualityBetweenSet", func(t *testing.T) { var activeSeriesMatchers []*ActiveSeriesMatchers for _, matcherConfig := range matcherSet { - asm := &ActiveSeriesMatchers{} - err := asm.Set(matcherConfig) + config := ActiveSeriesCustomTrackersConfig{} + err := config.Set(matcherConfig) + require.NoError(t, err) + asm, err := NewActiveSeriesMatchers(config) require.NoError(t, err) activeSeriesMatchers = append(activeSeriesMatchers, asm) } @@ -233,8 +230,10 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { var activeSeriesMatchers []*ActiveSeriesMatchers for _, matcherConfigs := range matcherSets { exampleConfig := matcherConfigs[0] - asm := &ActiveSeriesMatchers{} - err := asm.Set(exampleConfig) + config := ActiveSeriesCustomTrackersConfig{} + err := config.Set(exampleConfig) + require.NoError(t, err) + asm, err := NewActiveSeriesMatchers(config) require.NoError(t, err) activeSeriesMatchers = append(activeSeriesMatchers, asm) } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 3b46f38335..46d6257160 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -124,7 +124,8 @@ type Config struct { ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackers ActiveSeriesMatchers `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` + ActiveSeriesCustomTrackers ActiveSeriesMatchers `yaml:"-"` + ActiveSeriesCustomTrackersConfig ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` ActiveSeriesCustomTrackersOverrides *ActiveSeriesCustomTrackersOverridesProvider `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -153,7 +154,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { f.BoolVar(&cfg.ActiveSeriesMetricsEnabled, "ingester.active-series-metrics-enabled", true, "Enable tracking of active series and export them as metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsUpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsIdleTimeout, "ingester.active-series-metrics-idle-timeout", 10*time.Minute, "After what time a series is considered to be inactive.") - f.Var(&cfg.ActiveSeriesCustomTrackers, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be by providing multiple semicolon-separated values to a single flag.") + f.Var(&cfg.ActiveSeriesCustomTrackersConfig, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be by providing multiple semicolon-separated values to a single flag.") f.BoolVar(&cfg.StreamChunksWhenUsingBlocks, "ingester.stream-chunks-when-using-blocks", true, "Stream chunks from ingesters to queriers.") f.DurationVar(&cfg.ExemplarsUpdatePeriod, "ingester.exemplars-update-period", 15*time.Second, "Period with which to update per-user max exemplars.") @@ -235,6 +236,7 @@ type Ingester struct { // Rate of pushed samples. Used to limit global samples push rate. ingestionRate *util_math.EwmaRate inflightPushRequests atomic.Int64 + activeSeriesMatchers ActiveSeriesMatchers } func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { @@ -247,18 +249,24 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus return nil, errors.Wrap(err, "failed to create the bucket client") } + asm, err := NewActiveSeriesMatchers(cfg.ActiveSeriesCustomTrackersConfig) + if err != nil { + return nil, errors.Wrap(err, "failed to parse active series matchers config") + } + return &Ingester{ cfg: cfg, limits: limits, logger: logger, - tsdbs: make(map[string]*userTSDB), - usersMetadata: make(map[string]*userMetricsMetadata), - bucket: bucketClient, - tsdbMetrics: newTSDBMetrics(registerer), - forceCompactTrigger: make(chan requestWithUsersAndCallback), - shipTrigger: make(chan requestWithUsersAndCallback), - seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), + tsdbs: make(map[string]*userTSDB), + usersMetadata: make(map[string]*userMetricsMetadata), + bucket: bucketClient, + tsdbMetrics: newTSDBMetrics(registerer), + forceCompactTrigger: make(chan requestWithUsersAndCallback), + shipTrigger: make(chan requestWithUsersAndCallback), + seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), + activeSeriesMatchers: *asm, }, nil } @@ -472,7 +480,7 @@ func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers matchers = cfg.MatchersForUser(userID) } if matchers == nil { - matchers = &i.cfg.ActiveSeriesCustomTrackers + matchers = &i.activeSeriesMatchers } return matchers } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 3f8dc88103..7927aa347a 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5164,18 +5164,17 @@ func TestIngesterActiveSeries(t *testing.T) { } }, } - activeSeriesDefaultConfig, err := NewActiveSeriesMatchers(map[string]string{ + activeSeriesDefaultConfig := ActiveSeriesCustomTrackersConfig{ "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, - }) - require.NoError(t, err) + } tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest expectedMetrics string disableActiveSeries bool activeSeriesOverridesProvider *ActiveSeriesCustomTrackersOverridesProvider - activeSeriesConfig ActiveSeriesMatchers + activeSeriesConfig ActiveSeriesCustomTrackersConfig }{ "successful push, should count active series": { activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, @@ -5388,7 +5387,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should use flag based custom tracker if no runtime config specified": { activeSeriesOverridesProvider: nil, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5427,7 +5426,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should use runtime matcher config if both specified": { activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5477,7 +5476,7 @@ func TestIngesterActiveSeries(t *testing.T) { }} }, }, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5525,7 +5524,7 @@ func TestIngesterActiveSeries(t *testing.T) { cfg.IngesterRing.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries cfg.ActiveSeriesCustomTrackersOverrides = testData.activeSeriesOverridesProvider - cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig + cfg.ActiveSeriesCustomTrackersConfig = testData.activeSeriesConfig ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) require.NoError(t, err) @@ -5586,22 +5585,21 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { } }, } - activeSeriesDefaultConfig, err := NewActiveSeriesMatchers(map[string]string{ + activeSeriesDefaultConfig := ActiveSeriesCustomTrackersConfig{ "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, - }) - require.NoError(t, err) + } tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest expectedMetrics string activeSeriesOverridesProvider *ActiveSeriesCustomTrackersOverridesProvider - activeSeriesConfig ActiveSeriesMatchers + activeSeriesConfig ActiveSeriesCustomTrackersConfig }{ "overwrite flag based config with runtime overwrite": { activeSeriesOverridesProvider: nil, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5659,7 +5657,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { }, "remove runtime overwrite and revert to flag based config": { activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5727,7 +5725,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cfg.IngesterRing.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = true cfg.ActiveSeriesCustomTrackersOverrides = testData.activeSeriesOverridesProvider - cfg.ActiveSeriesCustomTrackers = testData.activeSeriesConfig + cfg.ActiveSeriesCustomTrackersConfig = testData.activeSeriesConfig ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) require.NoError(t, err) diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index c6fe8b85ff..8437fd55d4 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -22,6 +22,7 @@ import ( "github.com/prometheus/prometheus/model/relabel" "github.com/weaveworks/common/logging" + "github.com/grafana/mimir/pkg/ingester" "github.com/grafana/mimir/pkg/util/fieldcategory" ) @@ -291,6 +292,8 @@ func getFieldType(t reflect.Type) (string, error) { return "string", nil case reflect.TypeOf([]*relabel.Config{}).String(): return "relabel_config...", nil + case reflect.TypeOf(ingester.ActiveSeriesCustomTrackersConfig{}).String(): + return "map of tracker name (string) to matcher (string)", nil } // Fallback to auto-detection of built-in data types From c8ee2c9f1061bad3ca3cd40ba17ac1bf932e7ce5 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 12:20:59 +0100 Subject: [PATCH 35/91] Adding more context to asm.key comment --- pkg/ingester/active_series_custom_tracker.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index a2a4fa2927..360a7ca5a9 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -81,6 +81,7 @@ func NewActiveSeriesMatchers(matchersConfig ActiveSeriesCustomTrackersConfig) (* // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) // The concatenation should happen after ordering, to ensure equality is not dependent on map traversal. + // The string representation is saved to ensure quick equaility checks. asm.key = asm.String() return asm, nil @@ -115,15 +116,15 @@ func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { } // UnmarshalYAML implements the yaml.Unmarshaler interface. -// ActiveSeriesMatchers are marshaled in yaml as a map, with matcher names as keys and strings as matchers definitions. +// ActiveSeriesMatchers are marshaled in yaml as a ActiveSeriesCustomTrackersConfig, with matcher names as keys and strings as matchers definitions. func (asm *ActiveSeriesMatchers) UnmarshalYAML(unmarshal func(interface{}) error) error { - m := map[string]string{} - err := unmarshal(&m) + c := ActiveSeriesCustomTrackersConfig{} + err := unmarshal(&c) if err != nil { return err } var newMatchers *ActiveSeriesMatchers - newMatchers, err = NewActiveSeriesMatchers(m) + newMatchers, err = NewActiveSeriesMatchers(c) if err != nil { return err } From 8cde5965d7ce5ac8072718382293969c7abf351e Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 12:27:06 +0100 Subject: [PATCH 36/91] Changing back flag based description as it allows multiple flags --- cmd/mimir/help-all.txt.tmpl | 2 +- cmd/mimir/help.txt.tmpl | 4 ---- pkg/ingester/ingester.go | 2 +- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 51ff2c3672..98481d8bc0 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -758,7 +758,7 @@ Usage of ./cmd/mimir/mimir: -http.prometheus-http-prefix string HTTP URL path under which the Prometheus api will be served. (default "/prometheus") -ingester.active-series-custom-trackers value - Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be by providing multiple semicolon-separated values to a single flag. + Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or by providing multiple semicolon-separated values to a single flag. -ingester.active-series-metrics-enabled Enable tracking of active series and export them as metrics. (default true) -ingester.active-series-metrics-idle-timeout duration diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index 6c644b9764..d3add4d334 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -267,10 +267,6 @@ Usage of ./cmd/mimir/mimir: Print basic help. -help-all Print help, also including advanced and experimental parameters. - -ingester.active-series-custom-trackers value - Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be by providing multiple semicolon-separated values to a single flag. - -ingester.active-series-metrics-enabled - Enable tracking of active series and export them as metrics. (default true) -ingester.max-global-metadata-per-metric int The maximum number of metadata per metric, across the cluster. 0 to disable. -ingester.max-global-metadata-per-user int diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 46d6257160..6f83091595 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -154,7 +154,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { f.BoolVar(&cfg.ActiveSeriesMetricsEnabled, "ingester.active-series-metrics-enabled", true, "Enable tracking of active series and export them as metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsUpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsIdleTimeout, "ingester.active-series-metrics-idle-timeout", 10*time.Minute, "After what time a series is considered to be inactive.") - f.Var(&cfg.ActiveSeriesCustomTrackersConfig, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be by providing multiple semicolon-separated values to a single flag.") + f.Var(&cfg.ActiveSeriesCustomTrackersConfig, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be provided either providing the flag multiple times or by providing multiple semicolon-separated values to a single flag.") f.BoolVar(&cfg.StreamChunksWhenUsingBlocks, "ingester.stream-chunks-when-using-blocks", true, "Stream chunks from ingesters to queriers.") f.DurationVar(&cfg.ExemplarsUpdatePeriod, "ingester.exemplars-update-period", 15*time.Second, "Period with which to update per-user max exemplars.") From 47c7a0c6fcc2623e57a15b6b6b50dce26c1f33a3 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 12:39:02 +0100 Subject: [PATCH 37/91] changing back to exact wording --- pkg/ingester/ingester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 6f83091595..4c2b206d8e 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -154,7 +154,7 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { f.BoolVar(&cfg.ActiveSeriesMetricsEnabled, "ingester.active-series-metrics-enabled", true, "Enable tracking of active series and export them as metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsUpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsIdleTimeout, "ingester.active-series-metrics-idle-timeout", 10*time.Minute, "After what time a series is considered to be inactive.") - f.Var(&cfg.ActiveSeriesCustomTrackersConfig, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be provided either providing the flag multiple times or by providing multiple semicolon-separated values to a single flag.") + f.Var(&cfg.ActiveSeriesCustomTrackersConfig, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag.") f.BoolVar(&cfg.StreamChunksWhenUsingBlocks, "ingester.stream-chunks-when-using-blocks", true, "Stream chunks from ingesters to queriers.") f.DurationVar(&cfg.ExemplarsUpdatePeriod, "ingester.exemplars-update-period", 15*time.Second, "Period with which to update per-user max exemplars.") From 9e1ef8ed02beb76e91e4afebd687f18114ac5d94 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 13:24:14 +0100 Subject: [PATCH 38/91] fixup --- cmd/mimir/help-all.txt.tmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 98481d8bc0..429a3800f6 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -758,7 +758,7 @@ Usage of ./cmd/mimir/mimir: -http.prometheus-http-prefix string HTTP URL path under which the Prometheus api will be served. (default "/prometheus") -ingester.active-series-custom-trackers value - Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or by providing multiple semicolon-separated values to a single flag. + Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag. -ingester.active-series-metrics-enabled Enable tracking of active series and export them as metrics. (default true) -ingester.active-series-metrics-idle-timeout duration From bbf54a5393ca8a9fe9d9e133d22485501574e25d Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 13:51:33 +0100 Subject: [PATCH 39/91] Using config type instead of map[string]string in tests, cleanup --- .../active_series_runtime_overrides_test.go | 4 +-- pkg/ingester/active_series_test.go | 6 ++-- pkg/ingester/ingester.go | 30 +++++++++---------- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pkg/ingester/active_series_runtime_overrides_test.go b/pkg/ingester/active_series_runtime_overrides_test.go index 1f1bfd021f..da270ed0ac 100644 --- a/pkg/ingester/active_series_runtime_overrides_test.go +++ b/pkg/ingester/active_series_runtime_overrides_test.go @@ -58,14 +58,14 @@ func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { func TestMatchersForUser(t *testing.T) { defaultMatchers, err := NewActiveSeriesMatchers( - map[string]string{ + ActiveSeriesCustomTrackersConfig{ "foo": `{foo="bar"}`, "bar": `{baz="bar"}`, }) require.NoError(t, err) tenantSpecificMatchers, err := NewActiveSeriesMatchers( - map[string]string{ + ActiveSeriesCustomTrackersConfig{ "team_a": `{team="team_a"}`, "team_b": `{team="team_b"}`, }, diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index 7bd9168f9f..d3d018f661 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -50,7 +50,7 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { ls2 := []labels.Label{{Name: "a", Value: "2"}} ls3 := []labels.Label{{Name: "a", Value: "3"}} - asm, err := NewActiveSeriesMatchers(map[string]string{"foo": `{a=~"2|3"}`}) + asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{a=~"2|3"}`}) require.NoError(t, err) c := NewActiveSeries(asm) @@ -133,7 +133,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { {{Name: "_", Value: "KiqbryhzUpn"}, {Name: "__name__", Value: "logs"}}, } - asm, err := NewActiveSeriesMatchers(map[string]string{"foo": `{_=~"y.*"}`}) + asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{_=~"y.*"}`}) require.NoError(t, err) // Run the same test for increasing TTL values @@ -198,7 +198,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - asm, err := NewActiveSeriesMatchers(map[string]string{"foo": `{a=~.*}`}) + asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{a=~.*}`}) require.NoError(t, err) c := NewActiveSeries(asm) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 4c2b206d8e..24f031f0b1 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -124,7 +124,6 @@ type Config struct { ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackers ActiveSeriesMatchers `yaml:"-"` ActiveSeriesCustomTrackersConfig ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` ActiveSeriesCustomTrackersOverrides *ActiveSeriesCustomTrackersOverridesProvider `yaml:"-"` @@ -199,6 +198,8 @@ type Ingester struct { metrics *ingesterMetrics logger log.Logger + activeSeriesMatcher ActiveSeriesMatchers + lifecycler *ring.Lifecycler limits *validation.Overrides limiter *Limiter @@ -236,7 +237,6 @@ type Ingester struct { // Rate of pushed samples. Used to limit global samples push rate. ingestionRate *util_math.EwmaRate inflightPushRequests atomic.Int64 - activeSeriesMatchers ActiveSeriesMatchers } func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { @@ -255,18 +255,18 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus } return &Ingester{ - cfg: cfg, - limits: limits, - logger: logger, - - tsdbs: make(map[string]*userTSDB), - usersMetadata: make(map[string]*userMetricsMetadata), - bucket: bucketClient, - tsdbMetrics: newTSDBMetrics(registerer), - forceCompactTrigger: make(chan requestWithUsersAndCallback), - shipTrigger: make(chan requestWithUsersAndCallback), - seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), - activeSeriesMatchers: *asm, + cfg: cfg, + limits: limits, + logger: logger, + activeSeriesMatcher: *asm, + + tsdbs: make(map[string]*userTSDB), + usersMetadata: make(map[string]*userMetricsMetadata), + bucket: bucketClient, + tsdbMetrics: newTSDBMetrics(registerer), + forceCompactTrigger: make(chan requestWithUsersAndCallback), + shipTrigger: make(chan requestWithUsersAndCallback), + seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), }, nil } @@ -480,7 +480,7 @@ func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers matchers = cfg.MatchersForUser(userID) } if matchers == nil { - matchers = &i.activeSeriesMatchers + matchers = &i.activeSeriesMatcher } return matchers } From 42870e53af9a6ffa860cf85a72edbe88a8323d8c Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Feb 2022 15:12:14 +0100 Subject: [PATCH 40/91] Adding test case for metric cleanup --- pkg/ingester/ingester_test.go | 42 +++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 7927aa347a..7f3a0effd1 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5214,6 +5214,48 @@ func TestIngesterActiveSeries(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, }, + "should cleanup metrics when tsdb closed": { + activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + now := time.Now() + + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + for i, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID2) + offset := time.Duration(len(labelsToPush) - i) + _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(now) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 + ` + + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + // close tsdbs and check for cleanup + ingester.closeAllTSDB() + expectedMetrics = "" + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + }, + }, "should track custom matchers, removing when zero": { activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { From 79304c5b11626dbeef78d86c36d6aaf69fdf885d Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 23 Feb 2022 08:08:50 +0100 Subject: [PATCH 41/91] Changing ActiveSeriesCustomTrackerConfig to contain []labelMatchers instead of strings --- pkg/ingester/active_series.go | 1 + pkg/ingester/active_series_custom_tracker.go | 118 ++++++++++-------- .../active_series_custom_tracker_test.go | 51 ++++---- .../active_series_runtime_overrides.go | 11 +- .../active_series_runtime_overrides_test.go | 32 ++--- pkg/ingester/active_series_test.go | 9 +- pkg/ingester/ingester.go | 2 +- pkg/ingester/ingester_test.go | 62 +++++---- pkg/mimir/metrics-activity.log | Bin 0 -> 1048576 bytes 9 files changed, 158 insertions(+), 128 deletions(-) create mode 100644 pkg/mimir/metrics-activity.log diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index 560f8ab348..bddd47c52d 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -311,6 +311,7 @@ func makeIntSliceIfNotEmpty(l int, prev []int) []int { if l == 0 { return nil } + // The allocation is bigger than the required capacity to save time in cases when the number of matchers are just slightly increasing. return make([]int, l, l*2) } diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 360a7ca5a9..402aab6bcc 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -11,18 +11,26 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -type ActiveSeriesCustomTrackersConfig map[string]string +type ActiveSeriesCustomTrackersConfig map[string]labelsMatchers func (c *ActiveSeriesCustomTrackersConfig) String() string { if *c == nil { return "" } + keys := make([]string, len(*c)) + for name := range *c { + keys = append(keys, name) + } + sort.Strings(keys) - strs := make([]string, 0, len(*c)) - for name, matcher := range *c { - strs = append(strs, fmt.Sprintf("%s:%s", name, matcher)) + var sb strings.Builder + for _, name := range keys { + sb.WriteString(name) + for _, labelMatcher := range (*c)[name] { + sb.WriteString(labelMatcher.String()) + } } - return strings.Join(strs, ";") + return sb.String() } func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { @@ -30,9 +38,10 @@ func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { return nil } if *c == nil { - *c = map[string]string{} + *c = ActiveSeriesCustomTrackersConfig{} } + source := map[string]string{} pairs := strings.Split(s, ";") for i, p := range pairs { split := strings.SplitN(p, ":", 2) @@ -43,28 +52,45 @@ func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { if len(name) == 0 || len(matcher) == 0 { return fmt.Errorf("semicolon-separated values should be :, but one of the sides was empty in the value %d: %q", i, p) } + if _, ok := source[name]; ok { + return fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) + } + source[name] = matcher + } + config, err := NewActiveSeriesCustomTrackersConfig(source) + if err != nil { + return err + } + for name, matchers := range *config { + // This check is when the value comes from multiple flags if _, ok := (*c)[name]; ok { return fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) } - (*c)[name] = matcher + (*c)[name] = matchers } - _, err := NewActiveSeriesMatchers(*c) - return err + return nil } -func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { - return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + - ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, - ActiveSeriesCustomTrackersConfig{ - "dev": `{namespace=~"dev-.*"}`, - "prod": `{namespace=~"prod-.*"}`, - } +// UnmarshalYAML implements the yaml.Unmarshaler interface. +// ActiveSeriesCustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. +func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + stringMap := map[string]string{} + err := unmarshal(&stringMap) + if err != nil { + return err + } + config, err := NewActiveSeriesCustomTrackersConfig(stringMap) + if err != nil { + return err + } + *c = *config + return nil } -func NewActiveSeriesMatchers(matchersConfig ActiveSeriesCustomTrackersConfig) (*ActiveSeriesMatchers, error) { - asm := &ActiveSeriesMatchers{} - for name, matcher := range matchersConfig { +func NewActiveSeriesCustomTrackersConfig(m map[string]string) (*ActiveSeriesCustomTrackersConfig, error) { + c := ActiveSeriesCustomTrackersConfig{} + for name, matcher := range m { sm, err := amlabels.ParseMatchers(matcher) if err != nil { return nil, fmt.Errorf("can't build active series matcher %s: %w", name, err) @@ -73,33 +99,34 @@ func NewActiveSeriesMatchers(matchersConfig ActiveSeriesCustomTrackersConfig) (* for i, m := range sm { matchers[i] = amlabelMatcherToProm(m) } + c[name] = matchers + } + return &c, nil +} +func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { + return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, + map[string]string{ + "dev": `{namespace=~"dev-.*"}`, + "prod": `{namespace=~"prod-.*"}`, + } +} + +func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) *ActiveSeriesMatchers { + asm := &ActiveSeriesMatchers{} + for name, matchers := range *matchersConfig { asm.matchers = append(asm.matchers, matchers) asm.names = append(asm.names, name) } // Sort the result to make it deterministic for tests. // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) - // The concatenation should happen after ordering, to ensure equality is not dependent on map traversal. - // The string representation is saved to ensure quick equaility checks. - asm.key = asm.String() - return asm, nil -} + // ActiveSeriesCustomTrackersConfig String is ordering keys to be useful for comparison. + asm.key = matchersConfig.String() -func (asm *ActiveSeriesMatchers) String() string { - if asm == nil { - return "" - } - - var sb strings.Builder - for i, name := range asm.names { - sb.WriteString(name) - for _, labelMatcher := range asm.matchers[i] { - sb.WriteString(labelMatcher.String()) - } - } - return sb.String() + return asm } type ActiveSeriesMatchers struct { @@ -115,23 +142,6 @@ func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { return asm.key == other.key } -// UnmarshalYAML implements the yaml.Unmarshaler interface. -// ActiveSeriesMatchers are marshaled in yaml as a ActiveSeriesCustomTrackersConfig, with matcher names as keys and strings as matchers definitions. -func (asm *ActiveSeriesMatchers) UnmarshalYAML(unmarshal func(interface{}) error) error { - c := ActiveSeriesCustomTrackersConfig{} - err := unmarshal(&c) - if err != nil { - return err - } - var newMatchers *ActiveSeriesMatchers - newMatchers, err = NewActiveSeriesMatchers(c) - if err != nil { - return err - } - *asm = *newMatchers - return nil -} - func (asm *ActiveSeriesMatchers) MatcherNames() []string { return asm.names } diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 18eb2d0a66..214ca679ef 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -15,17 +15,23 @@ import ( "github.com/stretchr/testify/assert" ) +func safeLabelMatchers(t *testing.T, source map[string]string) *ActiveSeriesCustomTrackersConfig { + m, err := NewActiveSeriesCustomTrackersConfig(source) + require.NoError(t, err) + return m +} + func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { for _, tc := range []struct { name string flags []string - expected ActiveSeriesCustomTrackersConfig + expected string error error }{ { name: "empty flag value produces empty config", flags: []string{`-ingester.active-series-custom-trackers=`}, - expected: nil, + expected: "", }, { name: "empty matcher fails", @@ -55,22 +61,22 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "one matcher", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`}, - expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`}, + expected: safeLabelMatchers(t, map[string]string{`foo`: `{foo="bar"}`}).String(), }, { name: "whitespaces are trimmed from name and matcher", flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, - expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`}, + expected: safeLabelMatchers(t, map[string]string{`foo`: `{foo="bar"}`}).String(), }, { name: "two matchers in one flag value", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};baz:{baz="bar"}`}, - expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}, + expected: safeLabelMatchers(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}).String(), }, { name: "two matchers in two flag values", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, - expected: ActiveSeriesCustomTrackersConfig{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}, + expected: safeLabelMatchers(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}).String(), }, { name: "two matchers with same name in same flag", @@ -93,22 +99,21 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { if tc.error != nil { assert.EqualError(t, err, tc.error.Error()) } else { - assert.Equal(t, tc.expected, config) + assert.Equal(t, tc.expected, config.String()) } }) } } func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { - config := map[string]string{ + config := safeLabelMatchers(t, map[string]string{ "bar_starts_with_1": `{bar=~"1.*"}`, "does_not_have_foo_label": `{foo=""}`, "has_foo_and_bar_starts_with_1": `{foo!="", bar=~"1.*"}`, "has_foo_label": `{foo!=""}`, - } + }) - asm, err := NewActiveSeriesMatchers(config) - require.NoError(t, err) + asm := NewActiveSeriesMatchers(config) for _, tc := range []struct { series labels.Labels @@ -176,7 +181,7 @@ func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { } } -func TestActiveSeriesMatcher_MalformedMatcher(t *testing.T) { +func TestActiveSeriesCustomTrackersConfigs_MalformedMatcher(t *testing.T) { for _, matcher := range []string{ `{foo}`, `{foo=~"}`, @@ -186,7 +191,7 @@ func TestActiveSeriesMatcher_MalformedMatcher(t *testing.T) { "malformed": matcher, } - _, err := NewActiveSeriesMatchers(config) + _, err := NewActiveSeriesCustomTrackersConfig(config) assert.Error(t, err) }) } @@ -211,11 +216,10 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { t.Run("EqualityBetweenSet", func(t *testing.T) { var activeSeriesMatchers []*ActiveSeriesMatchers for _, matcherConfig := range matcherSet { - config := ActiveSeriesCustomTrackersConfig{} + config := &ActiveSeriesCustomTrackersConfig{} err := config.Set(matcherConfig) require.NoError(t, err) - asm, err := NewActiveSeriesMatchers(config) - require.NoError(t, err) + asm := NewActiveSeriesMatchers(config) activeSeriesMatchers = append(activeSeriesMatchers, asm) } for i := 0; i < len(activeSeriesMatchers); i++ { @@ -230,11 +234,10 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { var activeSeriesMatchers []*ActiveSeriesMatchers for _, matcherConfigs := range matcherSets { exampleConfig := matcherConfigs[0] - config := ActiveSeriesCustomTrackersConfig{} + config := &ActiveSeriesCustomTrackersConfig{} err := config.Set(exampleConfig) require.NoError(t, err) - asm, err := NewActiveSeriesMatchers(config) - require.NoError(t, err) + asm := NewActiveSeriesMatchers(config) activeSeriesMatchers = append(activeSeriesMatchers, asm) } @@ -247,7 +250,7 @@ func TestActiveSeriesMatcher_Equality(t *testing.T) { } -func TestActiveSeriesMatcher_Deserialization(t *testing.T) { +func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { correctInput := ` baz: "{baz='bar'}" foo: "{foo='bar'}" @@ -258,14 +261,14 @@ func TestActiveSeriesMatcher_Deserialization(t *testing.T) { foo: "{foo='bar'}" ` t.Run("ShouldDeserializeCorrectInput", func(t *testing.T) { - asm := ActiveSeriesMatchers{} - err := yaml.Unmarshal([]byte(correctInput), &asm) + config := ActiveSeriesCustomTrackersConfig{} + err := yaml.Unmarshal([]byte(correctInput), &config) assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") }) t.Run("ShouldErrorOnMalformedInput", func(t *testing.T) { - asm := ActiveSeriesMatchers{} - err := yaml.Unmarshal([]byte(malformedInput), &asm) + config := ActiveSeriesCustomTrackersConfig{} + err := yaml.Unmarshal([]byte(malformedInput), &config) assert.Error(t, err, "should not deserialize malformed input") }) } diff --git a/pkg/ingester/active_series_runtime_overrides.go b/pkg/ingester/active_series_runtime_overrides.go index 6ce3fcb508..69dfaf9fdb 100644 --- a/pkg/ingester/active_series_runtime_overrides.go +++ b/pkg/ingester/active_series_runtime_overrides.go @@ -4,15 +4,18 @@ package ingester // ActiveSeriesCustomTrackersOverrides holds the definition of custom tracking rules. type ActiveSeriesCustomTrackersOverrides struct { - Default *ActiveSeriesMatchers `yaml:"default"` - TenantSpecific map[string]*ActiveSeriesMatchers `yaml:"tenant_specific"` + Default *ActiveSeriesCustomTrackersConfig `yaml:"default"` + TenantSpecific map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_specific"` } func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersForUser(userID string) *ActiveSeriesMatchers { if tenantspecific, ok := asmo.TenantSpecific[userID]; ok { - return tenantspecific + return NewActiveSeriesMatchers(tenantspecific) } - return asmo.Default + if asmo.Default == nil { + return nil + } + return NewActiveSeriesMatchers(asmo.Default) } type ActiveSeriesCustomTrackersOverridesProvider struct { diff --git a/pkg/ingester/active_series_runtime_overrides_test.go b/pkg/ingester/active_series_runtime_overrides_test.go index da270ed0ac..af78297882 100644 --- a/pkg/ingester/active_series_runtime_overrides_test.go +++ b/pkg/ingester/active_series_runtime_overrides_test.go @@ -57,23 +57,23 @@ func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { } func TestMatchersForUser(t *testing.T) { - defaultMatchers, err := NewActiveSeriesMatchers( - ActiveSeriesCustomTrackersConfig{ - "foo": `{foo="bar"}`, - "bar": `{baz="bar"}`, - }) + safeLabelMatchers := func(source map[string]string) *ActiveSeriesCustomTrackersConfig { + m, err := NewActiveSeriesCustomTrackersConfig(source) + require.NoError(t, err) + return m + } + defaultMatchers := safeLabelMatchers(map[string]string{ + "foo": `{foo="bar"}`, + "bar": `{baz="bar"}`, + }) - require.NoError(t, err) - tenantSpecificMatchers, err := NewActiveSeriesMatchers( - ActiveSeriesCustomTrackersConfig{ - "team_a": `{team="team_a"}`, - "team_b": `{team="team_b"}`, - }, - ) - require.NoError(t, err) + tenantSpecificMatchers := safeLabelMatchers(map[string]string{ + "team_a": `{team="team_a"}`, + "team_b": `{team="team_b"}`, + }) activeSeriesCustomTrackersOverrides := &ActiveSeriesCustomTrackersOverrides{ Default: defaultMatchers, - TenantSpecific: map[string]*ActiveSeriesMatchers{ + TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ "1": tenantSpecificMatchers, }, } @@ -83,11 +83,11 @@ func TestMatchersForUser(t *testing.T) { }{ "User with no override should return default": { userID: "5", - expected: defaultMatchers, + expected: NewActiveSeriesMatchers(defaultMatchers), }, "User with override should return override": { userID: "1", - expected: tenantSpecificMatchers, + expected: NewActiveSeriesMatchers(tenantSpecificMatchers), }, } for name, testData := range tests { diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index d3d018f661..f3e240f8ff 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -50,8 +50,9 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { ls2 := []labels.Label{{Name: "a", Value: "2"}} ls3 := []labels.Label{{Name: "a", Value: "3"}} - asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{a=~"2|3"}`}) + config, err := NewActiveSeriesCustomTrackersConfig(map[string]string{"foo": `{a=~"2|3"}`}) require.NoError(t, err) + asm := NewActiveSeriesMatchers(config) c := NewActiveSeries(asm) allActive, activeMatching := c.Active() @@ -133,8 +134,9 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { {{Name: "_", Value: "KiqbryhzUpn"}, {Name: "__name__", Value: "logs"}}, } - asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{_=~"y.*"}`}) + config, err := NewActiveSeriesCustomTrackersConfig(map[string]string{"foo": `{_=~"y.*"}`}) require.NoError(t, err) + asm := NewActiveSeriesMatchers(config) // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { @@ -198,8 +200,9 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - asm, err := NewActiveSeriesMatchers(ActiveSeriesCustomTrackersConfig{"foo": `{a=~.*}`}) + config, err := NewActiveSeriesCustomTrackersConfig(map[string]string{"foo": `{a=~.*}`}) require.NoError(t, err) + asm := NewActiveSeriesMatchers(config) c := NewActiveSeries(asm) allActive, activeMatching := c.Active() diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 24f031f0b1..b0b7216e2a 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -249,7 +249,7 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus return nil, errors.Wrap(err, "failed to create the bucket client") } - asm, err := NewActiveSeriesMatchers(cfg.ActiveSeriesCustomTrackersConfig) + asm := NewActiveSeriesMatchers(&cfg.ActiveSeriesCustomTrackersConfig) if err != nil { return nil, errors.Wrap(err, "failed to parse active series matchers config") } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 7f3a0effd1..332ba9aaa3 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5146,28 +5146,33 @@ func TestIngesterActiveSeries(t *testing.T) { defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { - defaultMatchers, err := NewActiveSeriesMatchers(map[string]string{ + defaultConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }) require.NoError(t, err) - teamMatchers, err := NewActiveSeriesMatchers(map[string]string{ + + tenantConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }) require.NoError(t, err) + return &ActiveSeriesCustomTrackersOverrides{ - Default: defaultMatchers, - TenantSpecific: map[string]*ActiveSeriesMatchers{ - "test_user": teamMatchers, + Default: defaultConfig, + TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + "test_user": tenantConfig, }, } }, } - activeSeriesDefaultConfig := ActiveSeriesCustomTrackersConfig{ - "bool_is_true_flagbased": `{bool="true"}`, - "bool_is_false_flagbased": `{bool="false"}`, - } + + activeSeriesDefaultConfig, err := NewActiveSeriesCustomTrackersConfig( + map[string]string{ + "bool_is_true_flagbased": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + }) + require.NoError(t, err) tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest @@ -5429,7 +5434,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should use flag based custom tracker if no runtime config specified": { activeSeriesOverridesProvider: nil, - activeSeriesConfig: activeSeriesDefaultConfig, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5468,7 +5473,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should use runtime matcher config if both specified": { activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, - activeSeriesConfig: activeSeriesDefaultConfig, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5508,17 +5513,17 @@ func TestIngesterActiveSeries(t *testing.T) { "should revert to flag based default if only tenant-specific overwrite is present": { activeSeriesOverridesProvider: &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { - teamMatchers, err := NewActiveSeriesMatchers(map[string]string{ + tenantConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }) require.NoError(t, err) - return &ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*ActiveSeriesMatchers{ - "test_user": teamMatchers, + return &ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + "test_user": tenantConfig, }} }, }, - activeSeriesConfig: activeSeriesDefaultConfig, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5609,28 +5614,33 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { - defaultMatchers, err := NewActiveSeriesMatchers(map[string]string{ + defaultConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }) require.NoError(t, err) - teamMatchers, err := NewActiveSeriesMatchers(map[string]string{ + + tenantConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }) require.NoError(t, err) + return &ActiveSeriesCustomTrackersOverrides{ - Default: defaultMatchers, - TenantSpecific: map[string]*ActiveSeriesMatchers{ - "test_user": teamMatchers, + Default: defaultConfig, + TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + "test_user": tenantConfig, }, } }, } - activeSeriesDefaultConfig := ActiveSeriesCustomTrackersConfig{ - "bool_is_true_flagbased": `{bool="true"}`, - "bool_is_false_flagbased": `{bool="false"}`, - } + + activeSeriesDefaultConfig, err := NewActiveSeriesCustomTrackersConfig( + map[string]string{ + "bool_is_true_flagbased": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + }) + require.NoError(t, err) tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) @@ -5641,7 +5651,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { }{ "overwrite flag based config with runtime overwrite": { activeSeriesOverridesProvider: nil, - activeSeriesConfig: activeSeriesDefaultConfig, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5699,7 +5709,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { }, "remove runtime overwrite and revert to flag based config": { activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, - activeSeriesConfig: activeSeriesDefaultConfig, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() diff --git a/pkg/mimir/metrics-activity.log b/pkg/mimir/metrics-activity.log new file mode 100644 index 0000000000000000000000000000000000000000..9e0f96a2a253b173cb45b41868209a5d043e1437 GIT binary patch literal 1048576 zcmeIuF#!Mo0K%a4Pi+Wah(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ T0|pEjFkrxd0RsjM82APT0Pp|- literal 0 HcmV?d00001 From c076c04a09400914a0172d33f2dad41c012508f8 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 23 Feb 2022 12:45:46 +0100 Subject: [PATCH 42/91] Adding assumption to deserialization test, and adding comment to String() --- pkg/ingester/active_series_custom_tracker.go | 1 + pkg/ingester/active_series_custom_tracker_test.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 402aab6bcc..8b976c3a9f 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -21,6 +21,7 @@ func (c *ActiveSeriesCustomTrackersConfig) String() string { for name := range *c { keys = append(keys, name) } + // The map is traversed in an ordered fashion to make String representaton stable and comparable. sort.Strings(keys) var sb strings.Builder diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 214ca679ef..16250f0137 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -264,6 +264,12 @@ func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { config := ActiveSeriesCustomTrackersConfig{} err := yaml.Unmarshal([]byte(correctInput), &config) assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") + expectedConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ + "baz": "{baz='bar'}", + "foo": "{foo='bar'}", + }) + require.NoError(t, err) + assert.Equal(t, expectedConfig.String(), config.String()) }) t.Run("ShouldErrorOnMalformedInput", func(t *testing.T) { From 20cb06f7b02dea90715ff3c78469d4ccd049769b Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 23 Feb 2022 13:20:23 +0100 Subject: [PATCH 43/91] Trying to speed up comparison --- .../reference-configuration-parameters.md | 31 +++++++----- pkg/ingester/active_series_custom_tracker.go | 50 +++++++++++-------- tools/doc-generator/parser.go | 2 +- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/docs/sources/configuration/reference-configuration-parameters.md b/docs/sources/configuration/reference-configuration-parameters.md index f570fa7171..a92eb0b837 100644 --- a/docs/sources/configuration/reference-configuration-parameters.md +++ b/docs/sources/configuration/reference-configuration-parameters.md @@ -794,19 +794,24 @@ ring: # CLI flag: -ingester.active-series-metrics-idle-timeout [active_series_metrics_idle_timeout: | default = 10m] -# (advanced) Additional custom trackers for active metrics. If there are active -# series matching a provided matcher (map value), the count will be exposed in -# the custom trackers metric labeled using the tracker name (map key). Zero -# valued counts are not exposed (and removed when they go back to zero). -# Example: -# The following configuration will count the active series coming from dev and -# prod namespaces for each tenant and label them as {name="dev"} and -# {name="prod"} in the cortex_ingester_active_series_custom_tracker metric. -# active_series_custom_trackers: -# dev: '{namespace=~"dev-.*"}' -# prod: '{namespace=~"prod-.*"}' -# CLI flag: -ingester.active-series-custom-trackers -[active_series_custom_trackers: | default = ] +# Additional custom trackers for active metrics. If there are active series +# matching a provided matcher (map value), the count will be exposed in the +# custom trackers metric labeled using the tracker name (map key). Zero valued +# counts are not exposed (and removed when they go back to zero). +active_series_custom_trackers: + # Additional active series metrics, matching the provided matchers. Matchers + # should be in form :, like 'foobar:{foo="bar"}'. Multiple + # matchers can be provided either providing the flag multiple times or + # providing multiple semicolon-separated values to a single flag. + # Example: + # The following configuration will count the active series coming from dev + # and prod namespaces for each tenant and label them as {name="dev"} and + # {name="prod"} in the cortex_ingester_active_series_custom_tracker metric. + # active_series_custom_trackers: + # dev: '{namespace=~"dev-.*"}' + # prod: '{namespace=~"prod-.*"}' + # CLI flag: -ingester.active-series-custom-trackers + [active_series_custom_trackers: | default = ] # (experimental) Period with which to update per-user max exemplars. # CLI flag: -ingester.exemplars-update-period diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 8b976c3a9f..56470b63d4 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -11,14 +11,19 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -type ActiveSeriesCustomTrackersConfig map[string]labelsMatchers +type ActiveSeriesCustomTrackersConfigValue map[string]labelsMatchers + +type ActiveSeriesCustomTrackersConfig struct { + config ActiveSeriesCustomTrackersConfigValue `yaml:"active_series_custom_trackers"` + key string +} func (c *ActiveSeriesCustomTrackersConfig) String() string { - if *c == nil { + if (*c).config == nil { return "" } - keys := make([]string, len(*c)) - for name := range *c { + keys := make([]string, len((*c).config)) + for name := range (*c).config { keys = append(keys, name) } // The map is traversed in an ordered fashion to make String representaton stable and comparable. @@ -27,7 +32,7 @@ func (c *ActiveSeriesCustomTrackersConfig) String() string { var sb strings.Builder for _, name := range keys { sb.WriteString(name) - for _, labelMatcher := range (*c)[name] { + for _, labelMatcher := range ((*c).config)[name] { sb.WriteString(labelMatcher.String()) } } @@ -38,8 +43,9 @@ func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { if strings.TrimSpace(s) == "" { return nil } - if *c == nil { + if (*c).config == nil { *c = ActiveSeriesCustomTrackersConfig{} + (*c).config = map[string]labelsMatchers{} } source := map[string]string{} @@ -62,13 +68,14 @@ func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { if err != nil { return err } - for name, matchers := range *config { + for name, matchers := range (*config).config { // This check is when the value comes from multiple flags - if _, ok := (*c)[name]; ok { + if _, ok := (*c).config[name]; ok { return fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) } - (*c)[name] = matchers + (*c).config[name] = matchers } + (*c).key = (*c).String() return nil } @@ -91,6 +98,7 @@ func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interfac func NewActiveSeriesCustomTrackersConfig(m map[string]string) (*ActiveSeriesCustomTrackersConfig, error) { c := ActiveSeriesCustomTrackersConfig{} + c.config = map[string]labelsMatchers{} for name, matcher := range m { sm, err := amlabels.ParseMatchers(matcher) if err != nil { @@ -100,12 +108,13 @@ func NewActiveSeriesCustomTrackersConfig(m map[string]string) (*ActiveSeriesCust for i, m := range sm { matchers[i] = amlabelMatcherToProm(m) } - c[name] = matchers + c.config[name] = matchers } + c.key = c.String() return &c, nil } -func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { +func (c *ActiveSeriesCustomTrackersConfigValue) ExampleDoc() (comment string, yaml interface{}) { return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, map[string]string{ @@ -116,26 +125,19 @@ func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml in func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) *ActiveSeriesMatchers { asm := &ActiveSeriesMatchers{} - for name, matchers := range *matchersConfig { + for name, matchers := range (*matchersConfig).config { asm.matchers = append(asm.matchers, matchers) asm.names = append(asm.names, name) } // Sort the result to make it deterministic for tests. // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) - - // ActiveSeriesCustomTrackersConfig String is ordering keys to be useful for comparison. - asm.key = matchersConfig.String() + // ActiveSeriesCustomTrackersConfig.key is suitable for fast equality checks. + asm.key = matchersConfig.key return asm } -type ActiveSeriesMatchers struct { - key string - names []string - matchers []labelsMatchers -} - func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { if asm == nil || other == nil { return asm == other @@ -143,6 +145,12 @@ func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { return asm.key == other.key } +type ActiveSeriesMatchers struct { + key string + names []string + matchers []labelsMatchers +} + func (asm *ActiveSeriesMatchers) MatcherNames() []string { return asm.names } diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index 8437fd55d4..2c855b9b14 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -292,7 +292,7 @@ func getFieldType(t reflect.Type) (string, error) { return "string", nil case reflect.TypeOf([]*relabel.Config{}).String(): return "relabel_config...", nil - case reflect.TypeOf(ingester.ActiveSeriesCustomTrackersConfig{}).String(): + case reflect.TypeOf(ingester.ActiveSeriesCustomTrackersConfigValue{}).String(): return "map of tracker name (string) to matcher (string)", nil } From 7d0792c9ff019cebda0b495954fdf17b4652d16d Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Wed, 23 Feb 2022 15:11:59 +0100 Subject: [PATCH 44/91] Make custom trackers be unmarshaled as a map Instead of being a struct that holds a map, which an unnecessary wrapping. We need to tweak the docs parser for that, checking first whether a struct field is a custom field, in that case it's not a block. Signed-off-by: Oleg Zaytsev --- .../reference-configuration-parameters.md | 31 ++++++-------- pkg/ingester/active_series_custom_tracker.go | 25 ++++++------ tools/doc-generator/parser.go | 40 +++++++++++-------- 3 files changed, 50 insertions(+), 46 deletions(-) diff --git a/docs/sources/configuration/reference-configuration-parameters.md b/docs/sources/configuration/reference-configuration-parameters.md index a92eb0b837..f570fa7171 100644 --- a/docs/sources/configuration/reference-configuration-parameters.md +++ b/docs/sources/configuration/reference-configuration-parameters.md @@ -794,24 +794,19 @@ ring: # CLI flag: -ingester.active-series-metrics-idle-timeout [active_series_metrics_idle_timeout: | default = 10m] -# Additional custom trackers for active metrics. If there are active series -# matching a provided matcher (map value), the count will be exposed in the -# custom trackers metric labeled using the tracker name (map key). Zero valued -# counts are not exposed (and removed when they go back to zero). -active_series_custom_trackers: - # Additional active series metrics, matching the provided matchers. Matchers - # should be in form :, like 'foobar:{foo="bar"}'. Multiple - # matchers can be provided either providing the flag multiple times or - # providing multiple semicolon-separated values to a single flag. - # Example: - # The following configuration will count the active series coming from dev - # and prod namespaces for each tenant and label them as {name="dev"} and - # {name="prod"} in the cortex_ingester_active_series_custom_tracker metric. - # active_series_custom_trackers: - # dev: '{namespace=~"dev-.*"}' - # prod: '{namespace=~"prod-.*"}' - # CLI flag: -ingester.active-series-custom-trackers - [active_series_custom_trackers: | default = ] +# (advanced) Additional custom trackers for active metrics. If there are active +# series matching a provided matcher (map value), the count will be exposed in +# the custom trackers metric labeled using the tracker name (map key). Zero +# valued counts are not exposed (and removed when they go back to zero). +# Example: +# The following configuration will count the active series coming from dev and +# prod namespaces for each tenant and label them as {name="dev"} and +# {name="prod"} in the cortex_ingester_active_series_custom_tracker metric. +# active_series_custom_trackers: +# dev: '{namespace=~"dev-.*"}' +# prod: '{namespace=~"prod-.*"}' +# CLI flag: -ingester.active-series-custom-trackers +[active_series_custom_trackers: | default = ] # (experimental) Period with which to update per-user max exemplars. # CLI flag: -ingester.exemplars-update-period diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 56470b63d4..0ade8a459d 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -11,13 +11,23 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -type ActiveSeriesCustomTrackersConfigValue map[string]labelsMatchers - +// ActiveSeriesCustomTrackersConfig configures active series custom trackers. +// It can be set using a flag, or parsed from yaml. type ActiveSeriesCustomTrackersConfig struct { - config ActiveSeriesCustomTrackersConfigValue `yaml:"active_series_custom_trackers"` + config map[string]labelsMatchers key string } +// ExampleDoc provides an example doc for this config, especially valuable since it's custom-unmarshaled. +func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { + return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, + map[string]string{ + "dev": `{namespace=~"dev-.*"}`, + "prod": `{namespace=~"prod-.*"}`, + } +} + func (c *ActiveSeriesCustomTrackersConfig) String() string { if (*c).config == nil { return "" @@ -114,15 +124,6 @@ func NewActiveSeriesCustomTrackersConfig(m map[string]string) (*ActiveSeriesCust return &c, nil } -func (c *ActiveSeriesCustomTrackersConfigValue) ExampleDoc() (comment string, yaml interface{}) { - return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + - ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, - map[string]string{ - "dev": `{namespace=~"dev-.*"}`, - "prod": `{namespace=~"prod-.*"}`, - } -} - func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) *ActiveSeriesMatchers { asm := &ActiveSeriesMatchers{} for name, matchers := range (*matchersConfig).config { diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index 2c855b9b14..7d7d6f7278 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -165,8 +165,8 @@ func parseConfig(block *configBlock, cfg interface{}, flags map[uintptr]*flag.Fl continue } - // Recursively re-iterate if it's a struct - if field.Type.Kind() == reflect.Struct { + // Recursively re-iterate if it's a struct and + if _, custom := getCustomFieldType(field.Type); field.Type.Kind() == reflect.Struct && !custom { // Check whether the sub-block is a root config block rootName, rootDesc, isRoot := isRootBlock(field.Type) @@ -280,20 +280,8 @@ func getFieldName(field reflect.StructField) string { } func getFieldType(t reflect.Type) (string, error) { - // Handle custom data types used in the config - switch t.String() { - case reflect.TypeOf(&url.URL{}).String(): - return "url", nil - case reflect.TypeOf(time.Duration(0)).String(): - return "duration", nil - case reflect.TypeOf(flagext.StringSliceCSV{}).String(): - return "string", nil - case reflect.TypeOf(flagext.CIDRSliceCSV{}).String(): - return "string", nil - case reflect.TypeOf([]*relabel.Config{}).String(): - return "relabel_config...", nil - case reflect.TypeOf(ingester.ActiveSeriesCustomTrackersConfigValue{}).String(): - return "map of tracker name (string) to matcher (string)", nil + if custom, ok := getCustomFieldType(t); ok { + return custom, nil } // Fallback to auto-detection of built-in data types @@ -347,6 +335,26 @@ func getFieldType(t reflect.Type) (string, error) { } } +func getCustomFieldType(t reflect.Type) (string, bool) { + // Handle custom data types used in the config + switch t.String() { + case reflect.TypeOf(&url.URL{}).String(): + return "url", true + case reflect.TypeOf(time.Duration(0)).String(): + return "duration", true + case reflect.TypeOf(flagext.StringSliceCSV{}).String(): + return "string", true + case reflect.TypeOf(flagext.CIDRSliceCSV{}).String(): + return "string", true + case reflect.TypeOf([]*relabel.Config{}).String(): + return "relabel_config...", true + case reflect.TypeOf(ingester.ActiveSeriesCustomTrackersConfig{}).String(): + return "map of tracker name (string) to matcher (string)", true + default: + return "", false + } +} + func getFieldFlag(field reflect.StructField, fieldValue reflect.Value, flags map[uintptr]*flag.Flag) (*flag.Flag, error) { if isAbsentInCLI(field) { return nil, nil From 2bc4da1652b30b142db7564e95fff902cdb33e83 Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Wed, 23 Feb 2022 15:56:30 +0100 Subject: [PATCH 45/91] Refactor config parsing, make string key static Instead of calculating string key, we calculate it once when we change the config map (either parsing a flag or unmarshaling yaml) Also, unexport the config constructor: we don't expect anyone to call it from outside. Also renamed `safe...` to `must...` as that's the golang convention. Signed-off-by: Oleg Zaytsev --- pkg/ingester/active_series_custom_tracker.go | 89 +++++++++++-------- .../active_series_custom_tracker_test.go | 32 ++++--- .../active_series_runtime_overrides_test.go | 12 +-- pkg/ingester/active_series_test.go | 12 +-- pkg/ingester/ingester_test.go | 74 ++++++--------- 5 files changed, 100 insertions(+), 119 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 0ade8a459d..61868610bd 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -15,7 +15,7 @@ import ( // It can be set using a flag, or parsed from yaml. type ActiveSeriesCustomTrackersConfig struct { config map[string]labelsMatchers - key string + string string } // ExampleDoc provides an example doc for this config, especially valuable since it's custom-unmarshaled. @@ -29,65 +29,81 @@ func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml in } func (c *ActiveSeriesCustomTrackersConfig) String() string { - if (*c).config == nil { + return c.string +} + +func activeSeriesCustomTrackersConfigString(cfg map[string]labelsMatchers) string { + if len(cfg) == 0 { return "" } - keys := make([]string, len((*c).config)) - for name := range (*c).config { + + keys := make([]string, len(cfg)) + for name := range cfg { keys = append(keys, name) } - // The map is traversed in an ordered fashion to make String representaton stable and comparable. + // The map is traversed in an ordered fashion to make String representation stable and comparable. sort.Strings(keys) var sb strings.Builder for _, name := range keys { sb.WriteString(name) - for _, labelMatcher := range ((*c).config)[name] { + for _, labelMatcher := range cfg[name] { sb.WriteString(labelMatcher.String()) } } + return sb.String() } func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { - if strings.TrimSpace(s) == "" { + f, err := activeSeriesCustomTrackerFlagValueToMap(s) + if err != nil { + return err + } + + nc, err := newActiveSeriesCustomTrackersConfig(f) + if err != nil { + return err + } + + if len(c.config) == 0 { + // First flag, just set whatever we parsed. + // This includes an updated string. + *c = nc return nil } - if (*c).config == nil { - *c = ActiveSeriesCustomTrackersConfig{} - (*c).config = map[string]labelsMatchers{} + + // Not the first flag, merge checking for duplications. + for name := range nc.config { + if _, ok := c.config[name]; ok { + return fmt.Errorf("matcher %q for active series custom trackers is provided more than once", name) + } + c.config[name] = nc.config[name] } + // Recalculate the string after merging. + c.string = activeSeriesCustomTrackersConfigString(c.config) + return nil +} + +func activeSeriesCustomTrackerFlagValueToMap(s string) (map[string]string, error) { source := map[string]string{} pairs := strings.Split(s, ";") for i, p := range pairs { split := strings.SplitN(p, ":", 2) if len(split) != 2 { - return fmt.Errorf("value should be :[;:]*, but colon was not found in the value %d: %q", i, p) + return nil, fmt.Errorf("value should be :[;:]*, but colon was not found in the value %d: %q", i, p) } name, matcher := strings.TrimSpace(split[0]), strings.TrimSpace(split[1]) if len(name) == 0 || len(matcher) == 0 { - return fmt.Errorf("semicolon-separated values should be :, but one of the sides was empty in the value %d: %q", i, p) + return nil, fmt.Errorf("semicolon-separated values should be :, but one of the sides was empty in the value %d: %q", i, p) } if _, ok := source[name]; ok { - return fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) + return nil, fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) } source[name] = matcher } - config, err := NewActiveSeriesCustomTrackersConfig(source) - if err != nil { - return err - } - for name, matchers := range (*config).config { - // This check is when the value comes from multiple flags - if _, ok := (*c).config[name]; ok { - return fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) - } - (*c).config[name] = matchers - } - (*c).key = (*c).String() - - return nil + return source, nil } // UnmarshalYAML implements the yaml.Unmarshaler interface. @@ -98,21 +114,16 @@ func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interfac if err != nil { return err } - config, err := NewActiveSeriesCustomTrackersConfig(stringMap) - if err != nil { - return err - } - *c = *config - return nil + *c, err = newActiveSeriesCustomTrackersConfig(stringMap) + return err } -func NewActiveSeriesCustomTrackersConfig(m map[string]string) (*ActiveSeriesCustomTrackersConfig, error) { - c := ActiveSeriesCustomTrackersConfig{} +func newActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCustomTrackersConfig, err error) { c.config = map[string]labelsMatchers{} for name, matcher := range m { sm, err := amlabels.ParseMatchers(matcher) if err != nil { - return nil, fmt.Errorf("can't build active series matcher %s: %w", name, err) + return c, fmt.Errorf("can't build active series matcher %s: %w", name, err) } matchers := make(labelsMatchers, len(sm)) for i, m := range sm { @@ -120,8 +131,8 @@ func NewActiveSeriesCustomTrackersConfig(m map[string]string) (*ActiveSeriesCust } c.config[name] = matchers } - c.key = c.String() - return &c, nil + c.string = activeSeriesCustomTrackersConfigString(c.config) + return c, nil } func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) *ActiveSeriesMatchers { @@ -134,7 +145,7 @@ func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) * // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) // ActiveSeriesCustomTrackersConfig.key is suitable for fast equality checks. - asm.key = matchersConfig.key + asm.key = matchersConfig.string return asm } diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 16250f0137..51694cb56e 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -15,23 +15,23 @@ import ( "github.com/stretchr/testify/assert" ) -func safeLabelMatchers(t *testing.T, source map[string]string) *ActiveSeriesCustomTrackersConfig { - m, err := NewActiveSeriesCustomTrackersConfig(source) +func mustNewActiveSeriesCustomTrackersConfig(t *testing.T, source map[string]string) *ActiveSeriesCustomTrackersConfig { + m, err := newActiveSeriesCustomTrackersConfig(source) require.NoError(t, err) - return m + return &m } func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { for _, tc := range []struct { name string flags []string - expected string + expected *ActiveSeriesCustomTrackersConfig error error }{ { name: "empty flag value produces empty config", flags: []string{`-ingester.active-series-custom-trackers=`}, - expected: "", + expected: &ActiveSeriesCustomTrackersConfig{}, }, { name: "empty matcher fails", @@ -61,22 +61,22 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "one matcher", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`}, - expected: safeLabelMatchers(t, map[string]string{`foo`: `{foo="bar"}`}).String(), + expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`}), }, { name: "whitespaces are trimmed from name and matcher", flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, - expected: safeLabelMatchers(t, map[string]string{`foo`: `{foo="bar"}`}).String(), + expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`}), }, { name: "two matchers in one flag value", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};baz:{baz="bar"}`}, - expected: safeLabelMatchers(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}).String(), + expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), }, { name: "two matchers in two flag values", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, - expected: safeLabelMatchers(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}).String(), + expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), }, { name: "two matchers with same name in same flag", @@ -86,7 +86,7 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "two matchers with same name in separate flags", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=foo:{boo="bam"}`}, - error: errors.New(`invalid value "foo:{boo=\"bam\"}" for flag -ingester.active-series-custom-trackers: matcher "foo" for active series custom trackers is provided twice`), + error: errors.New(`invalid value "foo:{boo=\"bam\"}" for flag -ingester.active-series-custom-trackers: matcher "foo" for active series custom trackers is provided more than once`), }, } { t.Run(tc.name, func(t *testing.T) { @@ -99,21 +99,19 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { if tc.error != nil { assert.EqualError(t, err, tc.error.Error()) } else { - assert.Equal(t, tc.expected, config.String()) + assert.Equal(t, tc.expected, &config) } }) } } func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { - config := safeLabelMatchers(t, map[string]string{ + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ "bar_starts_with_1": `{bar=~"1.*"}`, "does_not_have_foo_label": `{foo=""}`, "has_foo_and_bar_starts_with_1": `{foo!="", bar=~"1.*"}`, "has_foo_label": `{foo!=""}`, - }) - - asm := NewActiveSeriesMatchers(config) + })) for _, tc := range []struct { series labels.Labels @@ -191,7 +189,7 @@ func TestActiveSeriesCustomTrackersConfigs_MalformedMatcher(t *testing.T) { "malformed": matcher, } - _, err := NewActiveSeriesCustomTrackersConfig(config) + _, err := newActiveSeriesCustomTrackersConfig(config) assert.Error(t, err) }) } @@ -264,7 +262,7 @@ func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { config := ActiveSeriesCustomTrackersConfig{} err := yaml.Unmarshal([]byte(correctInput), &config) assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") - expectedConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ + expectedConfig, err := newActiveSeriesCustomTrackersConfig(map[string]string{ "baz": "{baz='bar'}", "foo": "{foo='bar'}", }) diff --git a/pkg/ingester/active_series_runtime_overrides_test.go b/pkg/ingester/active_series_runtime_overrides_test.go index af78297882..4c8b223c28 100644 --- a/pkg/ingester/active_series_runtime_overrides_test.go +++ b/pkg/ingester/active_series_runtime_overrides_test.go @@ -57,26 +57,22 @@ func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { } func TestMatchersForUser(t *testing.T) { - safeLabelMatchers := func(source map[string]string) *ActiveSeriesCustomTrackersConfig { - m, err := NewActiveSeriesCustomTrackersConfig(source) - require.NoError(t, err) - return m - } - defaultMatchers := safeLabelMatchers(map[string]string{ + defaultMatchers := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ "foo": `{foo="bar"}`, "bar": `{baz="bar"}`, }) - - tenantSpecificMatchers := safeLabelMatchers(map[string]string{ + tenantSpecificMatchers := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ "team_a": `{team="team_a"}`, "team_b": `{team="team_b"}`, }) + activeSeriesCustomTrackersOverrides := &ActiveSeriesCustomTrackersOverrides{ Default: defaultMatchers, TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ "1": tenantSpecificMatchers, }, } + tests := map[string]struct { userID string expected *ActiveSeriesMatchers diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index f3e240f8ff..697a2e03e7 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -50,9 +50,7 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { ls2 := []labels.Label{{Name: "a", Value: "2"}} ls3 := []labels.Label{{Name: "a", Value: "3"}} - config, err := NewActiveSeriesCustomTrackersConfig(map[string]string{"foo": `{a=~"2|3"}`}) - require.NoError(t, err) - asm := NewActiveSeriesMatchers(config) + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{"foo": `{a=~"2|3"}`})) c := NewActiveSeries(asm) allActive, activeMatching := c.Active() @@ -134,9 +132,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { {{Name: "_", Value: "KiqbryhzUpn"}, {Name: "__name__", Value: "logs"}}, } - config, err := NewActiveSeriesCustomTrackersConfig(map[string]string{"foo": `{_=~"y.*"}`}) - require.NoError(t, err) - asm := NewActiveSeriesMatchers(config) + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{"foo": `{_=~"y.*"}`})) // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { @@ -200,9 +196,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - config, err := NewActiveSeriesCustomTrackersConfig(map[string]string{"foo": `{a=~.*}`}) - require.NoError(t, err) - asm := NewActiveSeriesMatchers(config) + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{"foo": `{a=~.*}`})) c := NewActiveSeries(asm) allActive, activeMatching := c.Active() diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 332ba9aaa3..33d58550f1 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5146,33 +5146,25 @@ func TestIngesterActiveSeries(t *testing.T) { defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { - defaultConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, - }) - require.NoError(t, err) - - tenantConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - }) - require.NoError(t, err) - return &ActiveSeriesCustomTrackersOverrides{ - Default: defaultConfig, + Default: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "bool_is_true": `{bool="true"}`, + "bool_is_false": `{bool="false"}`, + }), TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ - "test_user": tenantConfig, + "test_user": mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }), }, } }, } - activeSeriesDefaultConfig, err := NewActiveSeriesCustomTrackersConfig( - map[string]string{ - "bool_is_true_flagbased": `{bool="true"}`, - "bool_is_false_flagbased": `{bool="false"}`, - }) - require.NoError(t, err) + activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "bool_is_true_flagbased": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + }) tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest @@ -5513,13 +5505,11 @@ func TestIngesterActiveSeries(t *testing.T) { "should revert to flag based default if only tenant-specific overwrite is present": { activeSeriesOverridesProvider: &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { - tenantConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - }) - require.NoError(t, err) return &ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ - "test_user": tenantConfig, + "test_user": mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }), }} }, }, @@ -5614,33 +5604,25 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { - defaultConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, - }) - require.NoError(t, err) - - tenantConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - }) - require.NoError(t, err) - return &ActiveSeriesCustomTrackersOverrides{ - Default: defaultConfig, + Default: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "bool_is_true": `{bool="true"}`, + "bool_is_false": `{bool="false"}`, + }), TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ - "test_user": tenantConfig, + "test_user": mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }), }, } }, } - activeSeriesDefaultConfig, err := NewActiveSeriesCustomTrackersConfig( - map[string]string{ - "bool_is_true_flagbased": `{bool="true"}`, - "bool_is_false_flagbased": `{bool="false"}`, - }) - require.NoError(t, err) + activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "bool_is_true_flagbased": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + }) tests := map[string]struct { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) From 23e84ebc45e5a1d7996b247fc3db95d8a1887275 Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Wed, 23 Feb 2022 16:04:36 +0100 Subject: [PATCH 46/91] No need to store asm.key since config.string is static now Signed-off-by: Oleg Zaytsev --- pkg/ingester/active_series_custom_tracker.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 61868610bd..3714e281d5 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -136,7 +136,7 @@ func newActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCus } func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) *ActiveSeriesMatchers { - asm := &ActiveSeriesMatchers{} + asm := &ActiveSeriesMatchers{cfg: matchersConfig} for name, matchers := range (*matchersConfig).config { asm.matchers = append(asm.matchers, matchers) asm.names = append(asm.names, name) @@ -144,9 +144,6 @@ func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) * // Sort the result to make it deterministic for tests. // Order doesn't matter for the functionality as long as the order remains consistent during the execution of the program. sort.Sort(asm) - // ActiveSeriesCustomTrackersConfig.key is suitable for fast equality checks. - asm.key = matchersConfig.string - return asm } @@ -154,11 +151,11 @@ func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { if asm == nil || other == nil { return asm == other } - return asm.key == other.key + return asm.cfg.String() == other.cfg.String() } type ActiveSeriesMatchers struct { - key string + cfg *ActiveSeriesCustomTrackersConfig names []string matchers []labelsMatchers } From ed1bdfa7a0a537fda2bf6217e75a15a2e557247f Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Wed, 23 Feb 2022 16:15:17 +0100 Subject: [PATCH 47/91] Make config.String() be a valid flag value. Signed-off-by: Oleg Zaytsev --- pkg/ingester/active_series_custom_tracker.go | 29 ++++++++++++++----- .../active_series_custom_tracker_test.go | 13 +++++++-- 2 files changed, 32 insertions(+), 10 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 3714e281d5..49f4b85664 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -14,6 +14,7 @@ import ( // ActiveSeriesCustomTrackersConfig configures active series custom trackers. // It can be set using a flag, or parsed from yaml. type ActiveSeriesCustomTrackersConfig struct { + source map[string]string config map[string]labelsMatchers string string } @@ -28,34 +29,44 @@ func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml in } } +// String is a canonical representation of the config, it is compatible with flag definition. +// String is also needed to implement flag.Value. func (c *ActiveSeriesCustomTrackersConfig) String() string { return c.string } -func activeSeriesCustomTrackersConfigString(cfg map[string]labelsMatchers) string { +func activeSeriesCustomTrackersConfigString(cfg map[string]string) string { if len(cfg) == 0 { return "" } - keys := make([]string, len(cfg)) + keys := make([]string, 0, len(cfg)) for name := range cfg { keys = append(keys, name) } + // The map is traversed in an ordered fashion to make String representation stable and comparable. sort.Strings(keys) var sb strings.Builder - for _, name := range keys { - sb.WriteString(name) - for _, labelMatcher := range cfg[name] { - sb.WriteString(labelMatcher.String()) + for i, name := range keys { + if i > 0 { + sb.WriteByte(';') } + sb.WriteString(name) + sb.WriteByte(':') + sb.WriteString(cfg[name]) } return sb.String() } +// Set implements flag.Value, and is used to set the config value from a flag value provided as string. func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { + if strings.TrimSpace(s) == "" { + return nil + } + f, err := activeSeriesCustomTrackerFlagValueToMap(s) if err != nil { return err @@ -79,10 +90,11 @@ func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { return fmt.Errorf("matcher %q for active series custom trackers is provided more than once", name) } c.config[name] = nc.config[name] + c.source[name] = f[name] } // Recalculate the string after merging. - c.string = activeSeriesCustomTrackersConfigString(c.config) + c.string = activeSeriesCustomTrackersConfigString(c.source) return nil } @@ -119,6 +131,7 @@ func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interfac } func newActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCustomTrackersConfig, err error) { + c.source = m c.config = map[string]labelsMatchers{} for name, matcher := range m { sm, err := amlabels.ParseMatchers(matcher) @@ -131,7 +144,7 @@ func newActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCus } c.config[name] = matchers } - c.string = activeSeriesCustomTrackersConfigString(c.config) + c.string = activeSeriesCustomTrackersConfigString(c.source) return c, nil } diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 51694cb56e..fd83f3bab7 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -98,9 +98,18 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { if tc.error != nil { assert.EqualError(t, err, tc.error.Error()) - } else { - assert.Equal(t, tc.expected, &config) + return } + + require.Equal(t, tc.expected, &config) + + // Check that ActiveSeriesCustomTrackersConfig.String() value is a valid flag value. + flagSetAgain := flag.NewFlagSet("test-string", flag.ContinueOnError) + var configAgain ActiveSeriesCustomTrackersConfig + flagSetAgain.Var(&configAgain, "ingester.active-series-custom-trackers", "...usage docs...") + require.NoError(t, flagSetAgain.Parse([]string{"-ingester.active-series-custom-trackers=" + config.String()})) + + require.Equal(t, tc.expected, &configAgain) }) } } From b490ca8e078f6a6dfce2f582066cb73841e8c336 Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Wed, 23 Feb 2022 16:19:26 +0100 Subject: [PATCH 48/91] Move config to same file as overrides, and rename file. Signed-off-by: Oleg Zaytsev --- pkg/ingester/active_series_custom_tracker.go | 139 ------------- .../active_series_custom_tracker_test.go | 95 --------- .../active_series_custom_trackers_config.go | 175 ++++++++++++++++ ...tive_series_custom_trackers_config_test.go | 190 ++++++++++++++++++ .../active_series_runtime_overrides.go | 30 --- .../active_series_runtime_overrides_test.go | 95 --------- 6 files changed, 365 insertions(+), 359 deletions(-) create mode 100644 pkg/ingester/active_series_custom_trackers_config.go create mode 100644 pkg/ingester/active_series_custom_trackers_config_test.go delete mode 100644 pkg/ingester/active_series_runtime_overrides.go delete mode 100644 pkg/ingester/active_series_runtime_overrides_test.go diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index 49f4b85664..a64f027f19 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -3,151 +3,12 @@ package ingester import ( - "fmt" "sort" - "strings" amlabels "github.com/prometheus/alertmanager/pkg/labels" "github.com/prometheus/prometheus/model/labels" ) -// ActiveSeriesCustomTrackersConfig configures active series custom trackers. -// It can be set using a flag, or parsed from yaml. -type ActiveSeriesCustomTrackersConfig struct { - source map[string]string - config map[string]labelsMatchers - string string -} - -// ExampleDoc provides an example doc for this config, especially valuable since it's custom-unmarshaled. -func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { - return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + - ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, - map[string]string{ - "dev": `{namespace=~"dev-.*"}`, - "prod": `{namespace=~"prod-.*"}`, - } -} - -// String is a canonical representation of the config, it is compatible with flag definition. -// String is also needed to implement flag.Value. -func (c *ActiveSeriesCustomTrackersConfig) String() string { - return c.string -} - -func activeSeriesCustomTrackersConfigString(cfg map[string]string) string { - if len(cfg) == 0 { - return "" - } - - keys := make([]string, 0, len(cfg)) - for name := range cfg { - keys = append(keys, name) - } - - // The map is traversed in an ordered fashion to make String representation stable and comparable. - sort.Strings(keys) - - var sb strings.Builder - for i, name := range keys { - if i > 0 { - sb.WriteByte(';') - } - sb.WriteString(name) - sb.WriteByte(':') - sb.WriteString(cfg[name]) - } - - return sb.String() -} - -// Set implements flag.Value, and is used to set the config value from a flag value provided as string. -func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { - if strings.TrimSpace(s) == "" { - return nil - } - - f, err := activeSeriesCustomTrackerFlagValueToMap(s) - if err != nil { - return err - } - - nc, err := newActiveSeriesCustomTrackersConfig(f) - if err != nil { - return err - } - - if len(c.config) == 0 { - // First flag, just set whatever we parsed. - // This includes an updated string. - *c = nc - return nil - } - - // Not the first flag, merge checking for duplications. - for name := range nc.config { - if _, ok := c.config[name]; ok { - return fmt.Errorf("matcher %q for active series custom trackers is provided more than once", name) - } - c.config[name] = nc.config[name] - c.source[name] = f[name] - } - - // Recalculate the string after merging. - c.string = activeSeriesCustomTrackersConfigString(c.source) - return nil -} - -func activeSeriesCustomTrackerFlagValueToMap(s string) (map[string]string, error) { - source := map[string]string{} - pairs := strings.Split(s, ";") - for i, p := range pairs { - split := strings.SplitN(p, ":", 2) - if len(split) != 2 { - return nil, fmt.Errorf("value should be :[;:]*, but colon was not found in the value %d: %q", i, p) - } - name, matcher := strings.TrimSpace(split[0]), strings.TrimSpace(split[1]) - if len(name) == 0 || len(matcher) == 0 { - return nil, fmt.Errorf("semicolon-separated values should be :, but one of the sides was empty in the value %d: %q", i, p) - } - if _, ok := source[name]; ok { - return nil, fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) - } - source[name] = matcher - } - return source, nil -} - -// UnmarshalYAML implements the yaml.Unmarshaler interface. -// ActiveSeriesCustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. -func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { - stringMap := map[string]string{} - err := unmarshal(&stringMap) - if err != nil { - return err - } - *c, err = newActiveSeriesCustomTrackersConfig(stringMap) - return err -} - -func newActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCustomTrackersConfig, err error) { - c.source = m - c.config = map[string]labelsMatchers{} - for name, matcher := range m { - sm, err := amlabels.ParseMatchers(matcher) - if err != nil { - return c, fmt.Errorf("can't build active series matcher %s: %w", name, err) - } - matchers := make(labelsMatchers, len(sm)) - for i, m := range sm { - matchers[i] = amlabelMatcherToProm(m) - } - c.config[name] = matchers - } - c.string = activeSeriesCustomTrackersConfigString(c.source) - return c, nil -} - func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) *ActiveSeriesMatchers { asm := &ActiveSeriesMatchers{cfg: matchersConfig} for name, matchers := range (*matchersConfig).config { diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index fd83f3bab7..f1e5f3f44f 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -3,10 +3,8 @@ package ingester import ( - "flag" "testing" - "github.com/pkg/errors" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" @@ -21,99 +19,6 @@ func mustNewActiveSeriesCustomTrackersConfig(t *testing.T, source map[string]str return &m } -func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { - for _, tc := range []struct { - name string - flags []string - expected *ActiveSeriesCustomTrackersConfig - error error - }{ - { - name: "empty flag value produces empty config", - flags: []string{`-ingester.active-series-custom-trackers=`}, - expected: &ActiveSeriesCustomTrackersConfig{}, - }, - { - name: "empty matcher fails", - flags: []string{`-ingester.active-series-custom-trackers=foo:`}, - error: errors.New(`invalid value "foo:" for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: "foo:"`), - }, - { - name: "empty whitespace-only matcher fails", - flags: []string{`-ingester.active-series-custom-trackers=foo: `}, - error: errors.New(`invalid value "foo: " for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: "foo: "`), - }, - { - name: "second empty whitespace-only matcher fails", - flags: []string{`-ingester.active-series-custom-trackers=foo: ;bar:{}`}, - error: errors.New(`invalid value "foo: ;bar:{}" for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: "foo: "`), - }, - { - name: "empty name fails", - flags: []string{`-ingester.active-series-custom-trackers=:{}`}, - error: errors.New(`invalid value ":{}" for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: ":{}"`), - }, - { - name: "empty whitespace-only name fails", - flags: []string{`-ingester.active-series-custom-trackers= :{}`}, - error: errors.New(`invalid value " :{}" for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: " :{}"`), - }, - { - name: "one matcher", - flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`}), - }, - { - name: "whitespaces are trimmed from name and matcher", - flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, - expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`}), - }, - { - name: "two matchers in one flag value", - flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};baz:{baz="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), - }, - { - name: "two matchers in two flag values", - flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), - }, - { - name: "two matchers with same name in same flag", - flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};foo:{boo="bam"}`}, - error: errors.New(`invalid value "foo:{foo=\"bar\"};foo:{boo=\"bam\"}" for flag -ingester.active-series-custom-trackers: matcher "foo" for active series custom trackers is provided twice`), - }, - { - name: "two matchers with same name in separate flags", - flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=foo:{boo="bam"}`}, - error: errors.New(`invalid value "foo:{boo=\"bam\"}" for flag -ingester.active-series-custom-trackers: matcher "foo" for active series custom trackers is provided more than once`), - }, - } { - t.Run(tc.name, func(t *testing.T) { - flagSet := flag.NewFlagSet("test", flag.ContinueOnError) - - var config ActiveSeriesCustomTrackersConfig - flagSet.Var(&config, "ingester.active-series-custom-trackers", "...usage docs...") - err := flagSet.Parse(tc.flags) - - if tc.error != nil { - assert.EqualError(t, err, tc.error.Error()) - return - } - - require.Equal(t, tc.expected, &config) - - // Check that ActiveSeriesCustomTrackersConfig.String() value is a valid flag value. - flagSetAgain := flag.NewFlagSet("test-string", flag.ContinueOnError) - var configAgain ActiveSeriesCustomTrackersConfig - flagSetAgain.Var(&configAgain, "ingester.active-series-custom-trackers", "...usage docs...") - require.NoError(t, flagSetAgain.Parse([]string{"-ingester.active-series-custom-trackers=" + config.String()})) - - require.Equal(t, tc.expected, &configAgain) - }) - } -} - func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ "bar_starts_with_1": `{bar=~"1.*"}`, diff --git a/pkg/ingester/active_series_custom_trackers_config.go b/pkg/ingester/active_series_custom_trackers_config.go new file mode 100644 index 0000000000..7d77b63d26 --- /dev/null +++ b/pkg/ingester/active_series_custom_trackers_config.go @@ -0,0 +1,175 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package ingester + +import ( + "fmt" + "sort" + "strings" + + amlabels "github.com/prometheus/alertmanager/pkg/labels" +) + +// ActiveSeriesCustomTrackersConfig configures active series custom trackers. +// It can be set using a flag, or parsed from yaml. +type ActiveSeriesCustomTrackersConfig struct { + source map[string]string + config map[string]labelsMatchers + string string +} + +// ExampleDoc provides an example doc for this config, especially valuable since it's custom-unmarshaled. +func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { + return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, + map[string]string{ + "dev": `{namespace=~"dev-.*"}`, + "prod": `{namespace=~"prod-.*"}`, + } +} + +// String is a canonical representation of the config, it is compatible with flag definition. +// String is also needed to implement flag.Value. +func (c *ActiveSeriesCustomTrackersConfig) String() string { + return c.string +} + +func activeSeriesCustomTrackersConfigString(cfg map[string]string) string { + if len(cfg) == 0 { + return "" + } + + keys := make([]string, 0, len(cfg)) + for name := range cfg { + keys = append(keys, name) + } + + // The map is traversed in an ordered fashion to make String representation stable and comparable. + sort.Strings(keys) + + var sb strings.Builder + for i, name := range keys { + if i > 0 { + sb.WriteByte(';') + } + sb.WriteString(name) + sb.WriteByte(':') + sb.WriteString(cfg[name]) + } + + return sb.String() +} + +// Set implements flag.Value, and is used to set the config value from a flag value provided as string. +func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { + if strings.TrimSpace(s) == "" { + return nil + } + + f, err := activeSeriesCustomTrackerFlagValueToMap(s) + if err != nil { + return err + } + + nc, err := newActiveSeriesCustomTrackersConfig(f) + if err != nil { + return err + } + + if len(c.config) == 0 { + // First flag, just set whatever we parsed. + // This includes an updated string. + *c = nc + return nil + } + + // Not the first flag, merge checking for duplications. + for name := range nc.config { + if _, ok := c.config[name]; ok { + return fmt.Errorf("matcher %q for active series custom trackers is provided more than once", name) + } + c.config[name] = nc.config[name] + c.source[name] = f[name] + } + + // Recalculate the string after merging. + c.string = activeSeriesCustomTrackersConfigString(c.source) + return nil +} + +func activeSeriesCustomTrackerFlagValueToMap(s string) (map[string]string, error) { + source := map[string]string{} + pairs := strings.Split(s, ";") + for i, p := range pairs { + split := strings.SplitN(p, ":", 2) + if len(split) != 2 { + return nil, fmt.Errorf("value should be :[;:]*, but colon was not found in the value %d: %q", i, p) + } + name, matcher := strings.TrimSpace(split[0]), strings.TrimSpace(split[1]) + if len(name) == 0 || len(matcher) == 0 { + return nil, fmt.Errorf("semicolon-separated values should be :, but one of the sides was empty in the value %d: %q", i, p) + } + if _, ok := source[name]; ok { + return nil, fmt.Errorf("matcher %q for active series custom trackers is provided twice", name) + } + source[name] = matcher + } + return source, nil +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +// ActiveSeriesCustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. +func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + stringMap := map[string]string{} + err := unmarshal(&stringMap) + if err != nil { + return err + } + *c, err = newActiveSeriesCustomTrackersConfig(stringMap) + return err +} + +func newActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCustomTrackersConfig, err error) { + c.source = m + c.config = map[string]labelsMatchers{} + for name, matcher := range m { + sm, err := amlabels.ParseMatchers(matcher) + if err != nil { + return c, fmt.Errorf("can't build active series matcher %s: %w", name, err) + } + matchers := make(labelsMatchers, len(sm)) + for i, m := range sm { + matchers[i] = amlabelMatcherToProm(m) + } + c.config[name] = matchers + } + c.string = activeSeriesCustomTrackersConfigString(c.source) + return c, nil +} + +// ActiveSeriesCustomTrackersOverrides holds the definition of custom tracking rules. +type ActiveSeriesCustomTrackersOverrides struct { + Default *ActiveSeriesCustomTrackersConfig `yaml:"default"` + TenantSpecific map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_specific"` +} + +func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersForUser(userID string) *ActiveSeriesMatchers { + if tenantspecific, ok := asmo.TenantSpecific[userID]; ok { + return NewActiveSeriesMatchers(tenantspecific) + } + if asmo.Default == nil { + return nil + } + return NewActiveSeriesMatchers(asmo.Default) +} + +type ActiveSeriesCustomTrackersOverridesProvider struct { + Getter func() *ActiveSeriesCustomTrackersOverrides +} + +func (p *ActiveSeriesCustomTrackersOverridesProvider) Get() *ActiveSeriesCustomTrackersOverrides { + if p == nil || p.Getter == nil { + return nil + } + return p.Getter() +} diff --git a/pkg/ingester/active_series_custom_trackers_config_test.go b/pkg/ingester/active_series_custom_trackers_config_test.go new file mode 100644 index 0000000000..19bbc82c4f --- /dev/null +++ b/pkg/ingester/active_series_custom_trackers_config_test.go @@ -0,0 +1,190 @@ +// SPDX-License-Identifier: AGPL-3.0-only + +package ingester + +import ( + "flag" + "testing" + + "github.com/pkg/errors" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v2" +) + +func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { + for _, tc := range []struct { + name string + flags []string + expected *ActiveSeriesCustomTrackersConfig + error error + }{ + { + name: "empty flag value produces empty config", + flags: []string{`-ingester.active-series-custom-trackers=`}, + expected: &ActiveSeriesCustomTrackersConfig{}, + }, + { + name: "empty matcher fails", + flags: []string{`-ingester.active-series-custom-trackers=foo:`}, + error: errors.New(`invalid value "foo:" for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: "foo:"`), + }, + { + name: "empty whitespace-only matcher fails", + flags: []string{`-ingester.active-series-custom-trackers=foo: `}, + error: errors.New(`invalid value "foo: " for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: "foo: "`), + }, + { + name: "second empty whitespace-only matcher fails", + flags: []string{`-ingester.active-series-custom-trackers=foo: ;bar:{}`}, + error: errors.New(`invalid value "foo: ;bar:{}" for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: "foo: "`), + }, + { + name: "empty name fails", + flags: []string{`-ingester.active-series-custom-trackers=:{}`}, + error: errors.New(`invalid value ":{}" for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: ":{}"`), + }, + { + name: "empty whitespace-only name fails", + flags: []string{`-ingester.active-series-custom-trackers= :{}`}, + error: errors.New(`invalid value " :{}" for flag -ingester.active-series-custom-trackers: semicolon-separated values should be :, but one of the sides was empty in the value 0: " :{}"`), + }, + { + name: "one matcher", + flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`}, + expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`}), + }, + { + name: "whitespaces are trimmed from name and matcher", + flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, + expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`}), + }, + { + name: "two matchers in one flag value", + flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};baz:{baz="bar"}`}, + expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), + }, + { + name: "two matchers in two flag values", + flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, + expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), + }, + { + name: "two matchers with same name in same flag", + flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};foo:{boo="bam"}`}, + error: errors.New(`invalid value "foo:{foo=\"bar\"};foo:{boo=\"bam\"}" for flag -ingester.active-series-custom-trackers: matcher "foo" for active series custom trackers is provided twice`), + }, + { + name: "two matchers with same name in separate flags", + flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=foo:{boo="bam"}`}, + error: errors.New(`invalid value "foo:{boo=\"bam\"}" for flag -ingester.active-series-custom-trackers: matcher "foo" for active series custom trackers is provided more than once`), + }, + } { + t.Run(tc.name, func(t *testing.T) { + flagSet := flag.NewFlagSet("test", flag.ContinueOnError) + + var config ActiveSeriesCustomTrackersConfig + flagSet.Var(&config, "ingester.active-series-custom-trackers", "...usage docs...") + err := flagSet.Parse(tc.flags) + + if tc.error != nil { + assert.EqualError(t, err, tc.error.Error()) + return + } + + require.Equal(t, tc.expected, &config) + + // Check that ActiveSeriesCustomTrackersConfig.String() value is a valid flag value. + flagSetAgain := flag.NewFlagSet("test-string", flag.ContinueOnError) + var configAgain ActiveSeriesCustomTrackersConfig + flagSetAgain.Var(&configAgain, "ingester.active-series-custom-trackers", "...usage docs...") + require.NoError(t, flagSetAgain.Parse([]string{"-ingester.active-series-custom-trackers=" + config.String()})) + + require.Equal(t, tc.expected, &configAgain) + }) + } +} + +func TestRuntimeOverridesUnmarshal(t *testing.T) { + r := ActiveSeriesCustomTrackersOverrides{} + input := ` +default: + integrations/apolloserver: "{job='integrations/apollo-server'}" + integrations/caddy: "{job='integrations/caddy'}" +tenant_specific: + 1: + team_A: "{grafanacloud_team='team_a'}" + team_B: "{grafanacloud_team='team_b'}" +` + + require.NoError(t, yaml.UnmarshalStrict([]byte(input), &r)) +} + +func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { + overridesReference := &ActiveSeriesCustomTrackersOverrides{} + tests := map[string]struct { + provider *ActiveSeriesCustomTrackersOverridesProvider + expected *ActiveSeriesCustomTrackersOverrides + }{ + "nil provider returns nil": { + provider: nil, + expected: nil, + }, + "nil getter returns nil": { + provider: &ActiveSeriesCustomTrackersOverridesProvider{}, + expected: nil, + }, + "getter is called": { + provider: &ActiveSeriesCustomTrackersOverridesProvider{ + Getter: func() *ActiveSeriesCustomTrackersOverrides { + return overridesReference + }, + }, + expected: overridesReference, + }, + } + + for name, testData := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, testData.expected, testData.provider.Get()) + }) + } +} + +func TestMatchersForUser(t *testing.T) { + defaultMatchers := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "foo": `{foo="bar"}`, + "bar": `{baz="bar"}`, + }) + tenantSpecificMatchers := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "team_a": `{team="team_a"}`, + "team_b": `{team="team_b"}`, + }) + + activeSeriesCustomTrackersOverrides := &ActiveSeriesCustomTrackersOverrides{ + Default: defaultMatchers, + TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + "1": tenantSpecificMatchers, + }, + } + + tests := map[string]struct { + userID string + expected *ActiveSeriesMatchers + }{ + "User with no override should return default": { + userID: "5", + expected: NewActiveSeriesMatchers(defaultMatchers), + }, + "User with override should return override": { + userID: "1", + expected: NewActiveSeriesMatchers(tenantSpecificMatchers), + }, + } + for name, testData := range tests { + t.Run(name, func(t *testing.T) { + matchersForUser := activeSeriesCustomTrackersOverrides.MatchersForUser(testData.userID) + assert.True(t, testData.expected.Equals(matchersForUser)) + }) + } +} diff --git a/pkg/ingester/active_series_runtime_overrides.go b/pkg/ingester/active_series_runtime_overrides.go deleted file mode 100644 index 69dfaf9fdb..0000000000 --- a/pkg/ingester/active_series_runtime_overrides.go +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package ingester - -// ActiveSeriesCustomTrackersOverrides holds the definition of custom tracking rules. -type ActiveSeriesCustomTrackersOverrides struct { - Default *ActiveSeriesCustomTrackersConfig `yaml:"default"` - TenantSpecific map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_specific"` -} - -func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersForUser(userID string) *ActiveSeriesMatchers { - if tenantspecific, ok := asmo.TenantSpecific[userID]; ok { - return NewActiveSeriesMatchers(tenantspecific) - } - if asmo.Default == nil { - return nil - } - return NewActiveSeriesMatchers(asmo.Default) -} - -type ActiveSeriesCustomTrackersOverridesProvider struct { - Getter func() *ActiveSeriesCustomTrackersOverrides -} - -func (p *ActiveSeriesCustomTrackersOverridesProvider) Get() *ActiveSeriesCustomTrackersOverrides { - if p == nil || p.Getter == nil { - return nil - } - return p.Getter() -} diff --git a/pkg/ingester/active_series_runtime_overrides_test.go b/pkg/ingester/active_series_runtime_overrides_test.go deleted file mode 100644 index 4c8b223c28..0000000000 --- a/pkg/ingester/active_series_runtime_overrides_test.go +++ /dev/null @@ -1,95 +0,0 @@ -// SPDX-License-Identifier: AGPL-3.0-only - -package ingester - -import ( - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v2" -) - -func TestRuntimeOverridesUnmarshal(t *testing.T) { - r := ActiveSeriesCustomTrackersOverrides{} - input := ` -default: - integrations/apolloserver: "{job='integrations/apollo-server'}" - integrations/caddy: "{job='integrations/caddy'}" -tenant_specific: - 1: - team_A: "{grafanacloud_team='team_a'}" - team_B: "{grafanacloud_team='team_b'}" -` - - require.NoError(t, yaml.UnmarshalStrict([]byte(input), &r)) -} - -func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { - overridesReference := &ActiveSeriesCustomTrackersOverrides{} - tests := map[string]struct { - provider *ActiveSeriesCustomTrackersOverridesProvider - expected *ActiveSeriesCustomTrackersOverrides - }{ - "nil provider returns nil": { - provider: nil, - expected: nil, - }, - "nil getter returns nil": { - provider: &ActiveSeriesCustomTrackersOverridesProvider{}, - expected: nil, - }, - "getter is called": { - provider: &ActiveSeriesCustomTrackersOverridesProvider{ - Getter: func() *ActiveSeriesCustomTrackersOverrides { - return overridesReference - }, - }, - expected: overridesReference, - }, - } - - for name, testData := range tests { - t.Run(name, func(t *testing.T) { - assert.Equal(t, testData.expected, testData.provider.Get()) - }) - } -} - -func TestMatchersForUser(t *testing.T) { - defaultMatchers := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ - "foo": `{foo="bar"}`, - "bar": `{baz="bar"}`, - }) - tenantSpecificMatchers := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ - "team_a": `{team="team_a"}`, - "team_b": `{team="team_b"}`, - }) - - activeSeriesCustomTrackersOverrides := &ActiveSeriesCustomTrackersOverrides{ - Default: defaultMatchers, - TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ - "1": tenantSpecificMatchers, - }, - } - - tests := map[string]struct { - userID string - expected *ActiveSeriesMatchers - }{ - "User with no override should return default": { - userID: "5", - expected: NewActiveSeriesMatchers(defaultMatchers), - }, - "User with override should return override": { - userID: "1", - expected: NewActiveSeriesMatchers(tenantSpecificMatchers), - }, - } - for name, testData := range tests { - t.Run(name, func(t *testing.T) { - matchersForUser := activeSeriesCustomTrackersOverrides.MatchersForUser(testData.userID) - assert.True(t, testData.expected.Equals(matchersForUser)) - }) - } -} From 712cbbe05bb962a531f6c2c904b37c249dcbcdbe Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 23 Feb 2022 17:08:18 +0100 Subject: [PATCH 49/91] Updating help template --- cmd/mimir/help.txt.tmpl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index d3add4d334..9298a44530 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -267,6 +267,8 @@ Usage of ./cmd/mimir/mimir: Print basic help. -help-all Print help, also including advanced and experimental parameters. + -ingester.active-series-custom-trackers value + Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag. -ingester.max-global-metadata-per-metric int The maximum number of metadata per metric, across the cluster. 0 to disable. -ingester.max-global-metadata-per-user int From ef3b51bf1ba56fee5d33de6833fafcb91f0680c1 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 23 Feb 2022 17:27:28 +0100 Subject: [PATCH 50/91] Creating ActiveSeriesMatchers only at replace time --- .../active_series_custom_trackers_config.go | 9 +++------ pkg/ingester/ingester.go | 18 +++++++++--------- 2 files changed, 12 insertions(+), 15 deletions(-) diff --git a/pkg/ingester/active_series_custom_trackers_config.go b/pkg/ingester/active_series_custom_trackers_config.go index 7d77b63d26..d028336ee1 100644 --- a/pkg/ingester/active_series_custom_trackers_config.go +++ b/pkg/ingester/active_series_custom_trackers_config.go @@ -153,14 +153,11 @@ type ActiveSeriesCustomTrackersOverrides struct { TenantSpecific map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_specific"` } -func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersForUser(userID string) *ActiveSeriesMatchers { +func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersConfigForUser(userID string) *ActiveSeriesCustomTrackersConfig { if tenantspecific, ok := asmo.TenantSpecific[userID]; ok { - return NewActiveSeriesMatchers(tenantspecific) + return tenantspecific } - if asmo.Default == nil { - return nil - } - return NewActiveSeriesMatchers(asmo.Default) + return asmo.Default } type ActiveSeriesCustomTrackersOverridesProvider struct { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index b0b7216e2a..09cf065b11 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -474,13 +474,13 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers { - var matchers *ActiveSeriesMatchers +func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *ActiveSeriesCustomTrackersConfig { + var matchers *ActiveSeriesCustomTrackersConfig if cfg := i.cfg.ActiveSeriesCustomTrackersOverrides.Get(); cfg != nil { - matchers = cfg.MatchersForUser(userID) + matchers = cfg.MatchersConfigForUser(userID) } if matchers == nil { - matchers = &i.activeSeriesMatcher + matchers = i.activeSeriesMatcher.cfg } return matchers } @@ -498,9 +498,9 @@ func (i *Ingester) updateActiveSeries(now time.Time) { continue } - newMatchers := i.getActiveSeriesMatchers(userID) - if !newMatchers.Equals(userDB.activeSeries.CurrentMatchers()) { - i.replaceMatchers(newMatchers, userDB, now) + newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) + if newMatchersConfig.String() != userDB.activeSeries.asm.cfg.String() { + i.replaceMatchers(NewActiveSeriesMatchers(newMatchersConfig), userDB, now) } userDB.activeSeries.Purge(purgeTime) @@ -1469,11 +1469,11 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - newMatchers := i.getActiveSeriesMatchers(userID) + newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) userDB := &userTSDB{ userID: userID, - activeSeries: NewActiveSeries(newMatchers), + activeSeries: NewActiveSeries(NewActiveSeriesMatchers(newMatchersConfig)), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), From d324631f8af2dec72398d607371451b7424d9494 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 23 Feb 2022 18:03:17 +0100 Subject: [PATCH 51/91] Revert "Creating ActiveSeriesMatchers only at replace time" This reverts commit ef3b51bf1ba56fee5d33de6833fafcb91f0680c1. --- .../active_series_custom_trackers_config.go | 9 ++++++--- pkg/ingester/ingester.go | 18 +++++++++--------- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/ingester/active_series_custom_trackers_config.go b/pkg/ingester/active_series_custom_trackers_config.go index d028336ee1..7d77b63d26 100644 --- a/pkg/ingester/active_series_custom_trackers_config.go +++ b/pkg/ingester/active_series_custom_trackers_config.go @@ -153,11 +153,14 @@ type ActiveSeriesCustomTrackersOverrides struct { TenantSpecific map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_specific"` } -func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersConfigForUser(userID string) *ActiveSeriesCustomTrackersConfig { +func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersForUser(userID string) *ActiveSeriesMatchers { if tenantspecific, ok := asmo.TenantSpecific[userID]; ok { - return tenantspecific + return NewActiveSeriesMatchers(tenantspecific) } - return asmo.Default + if asmo.Default == nil { + return nil + } + return NewActiveSeriesMatchers(asmo.Default) } type ActiveSeriesCustomTrackersOverridesProvider struct { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 09cf065b11..b0b7216e2a 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -474,13 +474,13 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *ActiveSeriesCustomTrackersConfig { - var matchers *ActiveSeriesCustomTrackersConfig +func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers { + var matchers *ActiveSeriesMatchers if cfg := i.cfg.ActiveSeriesCustomTrackersOverrides.Get(); cfg != nil { - matchers = cfg.MatchersConfigForUser(userID) + matchers = cfg.MatchersForUser(userID) } if matchers == nil { - matchers = i.activeSeriesMatcher.cfg + matchers = &i.activeSeriesMatcher } return matchers } @@ -498,9 +498,9 @@ func (i *Ingester) updateActiveSeries(now time.Time) { continue } - newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) - if newMatchersConfig.String() != userDB.activeSeries.asm.cfg.String() { - i.replaceMatchers(NewActiveSeriesMatchers(newMatchersConfig), userDB, now) + newMatchers := i.getActiveSeriesMatchers(userID) + if !newMatchers.Equals(userDB.activeSeries.CurrentMatchers()) { + i.replaceMatchers(newMatchers, userDB, now) } userDB.activeSeries.Purge(purgeTime) @@ -1469,11 +1469,11 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) + newMatchers := i.getActiveSeriesMatchers(userID) userDB := &userTSDB{ userID: userID, - activeSeries: NewActiveSeries(NewActiveSeriesMatchers(newMatchersConfig)), + activeSeries: NewActiveSeries(newMatchers), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), From 07ae65c03b8d89936ac3bcbf1230fb855ae617d2 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 24 Feb 2022 08:08:29 +0100 Subject: [PATCH 52/91] Creating ActiveSeriesMatchers only at replace time This reverts commit d324631f8af2dec72398d607371451b7424d9494. --- pkg/ingester/active_series_custom_tracker.go | 7 --- .../active_series_custom_tracker_test.go | 53 ---------------- .../active_series_custom_trackers_config.go | 9 +-- ...tive_series_custom_trackers_config_test.go | 61 +++++++++++++++++-- pkg/ingester/ingester.go | 18 +++--- pkg/ingester/ingester_test.go | 3 +- 6 files changed, 70 insertions(+), 81 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/active_series_custom_tracker.go index a64f027f19..73113d3253 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/active_series_custom_tracker.go @@ -21,13 +21,6 @@ func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) * return asm } -func (asm *ActiveSeriesMatchers) Equals(other *ActiveSeriesMatchers) bool { - if asm == nil || other == nil { - return asm == other - } - return asm.cfg.String() == other.cfg.String() -} - type ActiveSeriesMatchers struct { cfg *ActiveSeriesCustomTrackersConfig names []string diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index f1e5f3f44f..8604bf356e 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -109,59 +109,6 @@ func TestActiveSeriesCustomTrackersConfigs_MalformedMatcher(t *testing.T) { } } -func TestActiveSeriesMatcher_Equality(t *testing.T) { - matcherSets := [][]string{ - { - `foo:{foo="bar"};baz:{baz="bar"}`, - `baz:{baz="bar"};foo:{foo="bar"}`, - ` foo:{foo="bar"};baz:{baz="bar"} `, - }, - { - `test:{test="true"}`, - }, - { - `foo:{foo="bar"};baz:{baz="bar"};extra:{extra="extra"}`, - }, - } - - for _, matcherSet := range matcherSets { - t.Run("EqualityBetweenSet", func(t *testing.T) { - var activeSeriesMatchers []*ActiveSeriesMatchers - for _, matcherConfig := range matcherSet { - config := &ActiveSeriesCustomTrackersConfig{} - err := config.Set(matcherConfig) - require.NoError(t, err) - asm := NewActiveSeriesMatchers(config) - activeSeriesMatchers = append(activeSeriesMatchers, asm) - } - for i := 0; i < len(activeSeriesMatchers); i++ { - for j := i + 1; j < len(activeSeriesMatchers); j++ { - assert.True(t, activeSeriesMatchers[i].Equals(activeSeriesMatchers[j]), "matcher configs should be equal") - } - } - }) - } - - t.Run("NotEqualsAcrossSets", func(t *testing.T) { - var activeSeriesMatchers []*ActiveSeriesMatchers - for _, matcherConfigs := range matcherSets { - exampleConfig := matcherConfigs[0] - config := &ActiveSeriesCustomTrackersConfig{} - err := config.Set(exampleConfig) - require.NoError(t, err) - asm := NewActiveSeriesMatchers(config) - activeSeriesMatchers = append(activeSeriesMatchers, asm) - } - - for i := 0; i < len(activeSeriesMatchers); i++ { - for j := i + 1; j < len(activeSeriesMatchers); j++ { - assert.False(t, activeSeriesMatchers[i].Equals(activeSeriesMatchers[j]), "matcher configs should NOT be equal") - } - } - }) - -} - func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { correctInput := ` baz: "{baz='bar'}" diff --git a/pkg/ingester/active_series_custom_trackers_config.go b/pkg/ingester/active_series_custom_trackers_config.go index 7d77b63d26..d028336ee1 100644 --- a/pkg/ingester/active_series_custom_trackers_config.go +++ b/pkg/ingester/active_series_custom_trackers_config.go @@ -153,14 +153,11 @@ type ActiveSeriesCustomTrackersOverrides struct { TenantSpecific map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_specific"` } -func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersForUser(userID string) *ActiveSeriesMatchers { +func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersConfigForUser(userID string) *ActiveSeriesCustomTrackersConfig { if tenantspecific, ok := asmo.TenantSpecific[userID]; ok { - return NewActiveSeriesMatchers(tenantspecific) + return tenantspecific } - if asmo.Default == nil { - return nil - } - return NewActiveSeriesMatchers(asmo.Default) + return asmo.Default } type ActiveSeriesCustomTrackersOverridesProvider struct { diff --git a/pkg/ingester/active_series_custom_trackers_config_test.go b/pkg/ingester/active_series_custom_trackers_config_test.go index 19bbc82c4f..94f8494f50 100644 --- a/pkg/ingester/active_series_custom_trackers_config_test.go +++ b/pkg/ingester/active_series_custom_trackers_config_test.go @@ -170,21 +170,72 @@ func TestMatchersForUser(t *testing.T) { tests := map[string]struct { userID string - expected *ActiveSeriesMatchers + expected *ActiveSeriesCustomTrackersConfig }{ "User with no override should return default": { userID: "5", - expected: NewActiveSeriesMatchers(defaultMatchers), + expected: defaultMatchers, }, "User with override should return override": { userID: "1", - expected: NewActiveSeriesMatchers(tenantSpecificMatchers), + expected: tenantSpecificMatchers, }, } for name, testData := range tests { t.Run(name, func(t *testing.T) { - matchersForUser := activeSeriesCustomTrackersOverrides.MatchersForUser(testData.userID) - assert.True(t, testData.expected.Equals(matchersForUser)) + matchersConfigForUser := activeSeriesCustomTrackersOverrides.MatchersConfigForUser(testData.userID) + assert.True(t, testData.expected.String() == matchersConfigForUser.String()) }) } } + +func TestActiveSeriesCustomTrackerConfig_Equality(t *testing.T) { + matcherSets := [][]string{ + { + `foo:{foo="bar"};baz:{baz="bar"}`, + `baz:{baz="bar"};foo:{foo="bar"}`, + ` foo:{foo="bar"};baz:{baz="bar"} `, + }, + { + `test:{test="true"}`, + }, + { + `foo:{foo="bar"};baz:{baz="bar"};extra:{extra="extra"}`, + }, + } + + for _, matcherSet := range matcherSets { + t.Run("EqualityBetweenSet", func(t *testing.T) { + var activeSeriesCustomTrackersConfigs []*ActiveSeriesCustomTrackersConfig + for _, matcherConfig := range matcherSet { + config := &ActiveSeriesCustomTrackersConfig{} + err := config.Set(matcherConfig) + require.NoError(t, err) + activeSeriesCustomTrackersConfigs = append(activeSeriesCustomTrackersConfigs, config) + } + for i := 0; i < len(activeSeriesCustomTrackersConfigs); i++ { + for j := i + 1; j < len(activeSeriesCustomTrackersConfigs); j++ { + assert.Equal(t, activeSeriesCustomTrackersConfigs[i].String(), activeSeriesCustomTrackersConfigs[j].String(), "matcher configs should be equal") + } + } + }) + } + + t.Run("NotEqualsAcrossSets", func(t *testing.T) { + var activeSeriesMatchers []*ActiveSeriesCustomTrackersConfig + for _, matcherConfigs := range matcherSets { + exampleConfig := matcherConfigs[0] + config := &ActiveSeriesCustomTrackersConfig{} + err := config.Set(exampleConfig) + require.NoError(t, err) + activeSeriesMatchers = append(activeSeriesMatchers, config) + } + + for i := 0; i < len(activeSeriesMatchers); i++ { + for j := i + 1; j < len(activeSeriesMatchers); j++ { + assert.NotEqual(t, activeSeriesMatchers[i].String(), activeSeriesMatchers[j].String(), "matcher configs should NOT be equal") + } + } + }) + +} diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index b0b7216e2a..4cc6693df7 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -474,13 +474,13 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) getActiveSeriesMatchers(userID string) *ActiveSeriesMatchers { - var matchers *ActiveSeriesMatchers +func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *ActiveSeriesCustomTrackersConfig { + var matchers *ActiveSeriesCustomTrackersConfig if cfg := i.cfg.ActiveSeriesCustomTrackersOverrides.Get(); cfg != nil { - matchers = cfg.MatchersForUser(userID) + matchers = cfg.MatchersConfigForUser(userID) } if matchers == nil { - matchers = &i.activeSeriesMatcher + matchers = i.activeSeriesMatcher.cfg } return matchers } @@ -498,9 +498,9 @@ func (i *Ingester) updateActiveSeries(now time.Time) { continue } - newMatchers := i.getActiveSeriesMatchers(userID) - if !newMatchers.Equals(userDB.activeSeries.CurrentMatchers()) { - i.replaceMatchers(newMatchers, userDB, now) + newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) + if newMatchersConfig.String() != userDB.activeSeries.asm.cfg.String() { + i.replaceMatchers(NewActiveSeriesMatchers(newMatchersConfig), userDB, now) } userDB.activeSeries.Purge(purgeTime) @@ -1469,11 +1469,11 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - newMatchers := i.getActiveSeriesMatchers(userID) + matchersConfig := i.getActiveSeriesMatchersConfig(userID) userDB := &userTSDB{ userID: userID, - activeSeries: NewActiveSeries(newMatchers), + activeSeries: NewActiveSeries(NewActiveSeriesMatchers(matchersConfig)), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 33d58550f1..fea9532490 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5392,8 +5392,9 @@ func TestIngesterActiveSeries(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, }, - "should not fail with nil provider": { + "should not fail with nil provider and default config": { activeSeriesOverridesProvider: nil, + activeSeriesConfig: ActiveSeriesCustomTrackersConfig{}, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() From 78e325ad775ef2f1afd78690274e3257fd3aeec0 Mon Sep 17 00:00:00 2001 From: Janos Gub Date: Thu, 24 Feb 2022 09:27:57 +0100 Subject: [PATCH 53/91] Update tools/doc-generator/parser.go Co-authored-by: Oleg Zaytsev --- tools/doc-generator/parser.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index 7d7d6f7278..ee142fad5c 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -165,7 +165,7 @@ func parseConfig(block *configBlock, cfg interface{}, flags map[uintptr]*flag.Fl continue } - // Recursively re-iterate if it's a struct and + // Recursively re-iterate if it's a struct and it's not a custom type. if _, custom := getCustomFieldType(field.Type); field.Type.Kind() == reflect.Struct && !custom { // Check whether the sub-block is a root config block rootName, rootDesc, isRoot := isRootBlock(field.Type) From 80b593b76507ed32958317f5cfc893383d947816 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 24 Feb 2022 10:06:02 +0100 Subject: [PATCH 54/91] Changes based on oleg's comments --- .../active_series_custom_tracker_test.go | 38 +----- ...tive_series_custom_trackers_config_test.go | 123 +++++++++++++----- pkg/ingester/active_series_test.go | 6 +- pkg/ingester/ingester.go | 3 - pkg/ingester/ingester_test.go | 14 +- 5 files changed, 105 insertions(+), 79 deletions(-) diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/active_series_custom_tracker_test.go index 8604bf356e..2083007b17 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/active_series_custom_tracker_test.go @@ -6,21 +6,14 @@ import ( "testing" "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" amlabels "github.com/prometheus/alertmanager/pkg/labels" "github.com/prometheus/prometheus/model/labels" "github.com/stretchr/testify/assert" ) -func mustNewActiveSeriesCustomTrackersConfig(t *testing.T, source map[string]string) *ActiveSeriesCustomTrackersConfig { - m, err := newActiveSeriesCustomTrackersConfig(source) - require.NoError(t, err) - return &m -} - func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bar_starts_with_1": `{bar=~"1.*"}`, "does_not_have_foo_label": `{foo=""}`, "has_foo_and_bar_starts_with_1": `{foo!="", bar=~"1.*"}`, @@ -109,35 +102,6 @@ func TestActiveSeriesCustomTrackersConfigs_MalformedMatcher(t *testing.T) { } } -func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { - correctInput := ` - baz: "{baz='bar'}" - foo: "{foo='bar'}" - ` - malformedInput := - ` - baz: "123" - foo: "{foo='bar'}" - ` - t.Run("ShouldDeserializeCorrectInput", func(t *testing.T) { - config := ActiveSeriesCustomTrackersConfig{} - err := yaml.Unmarshal([]byte(correctInput), &config) - assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") - expectedConfig, err := newActiveSeriesCustomTrackersConfig(map[string]string{ - "baz": "{baz='bar'}", - "foo": "{foo='bar'}", - }) - require.NoError(t, err) - assert.Equal(t, expectedConfig.String(), config.String()) - }) - - t.Run("ShouldErrorOnMalformedInput", func(t *testing.T) { - config := ActiveSeriesCustomTrackersConfig{} - err := yaml.Unmarshal([]byte(malformedInput), &config) - assert.Error(t, err, "should not deserialize malformed input") - }) -} - func TestAmlabelMatchersToProm_HappyCase(t *testing.T) { amMatcher, err := amlabels.NewMatcher(amlabels.MatchRegexp, "foo", "bar.*") require.NoError(t, err) diff --git a/pkg/ingester/active_series_custom_trackers_config_test.go b/pkg/ingester/active_series_custom_trackers_config_test.go index 94f8494f50..362a8ae326 100644 --- a/pkg/ingester/active_series_custom_trackers_config_test.go +++ b/pkg/ingester/active_series_custom_trackers_config_test.go @@ -12,6 +12,26 @@ import ( "gopkg.in/yaml.v2" ) +func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *ActiveSeriesCustomTrackersConfig { + m, err := newActiveSeriesCustomTrackersConfig(source) + require.NoError(t, err) + return &m +} + +func mustNewActiveSeriesCustomTrackersConfigFromString(t *testing.T, source string) *ActiveSeriesCustomTrackersConfig { + m := ActiveSeriesCustomTrackersConfig{} + err := m.Set(source) + require.NoError(t, err) + return &m +} + +func mustNewActiveSeriesCustomTrackersConfigDeserializedFromYaml(t *testing.T, yamlString string) *ActiveSeriesCustomTrackersConfig { + m := ActiveSeriesCustomTrackersConfig{} + err := yaml.Unmarshal([]byte(yamlString), &m) + require.NoError(t, err) + return &m +} + func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { for _, tc := range []struct { name string @@ -52,22 +72,22 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "one matcher", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`}), + expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`}), }, { name: "whitespaces are trimmed from name and matcher", flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, - expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`}), + expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`}), }, { name: "two matchers in one flag value", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};baz:{baz="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), + expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), }, { name: "two matchers in two flag values", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), + expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), }, { name: "two matchers with same name in same flag", @@ -106,6 +126,18 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { } func TestRuntimeOverridesUnmarshal(t *testing.T) { + expectedDefaultConfig := mustNewActiveSeriesCustomTrackersConfigFromMap( + t, map[string]string{ + "integrations/apolloserver": "{job='integrations/apollo-server'}", + "integrations/caddy": "{job='integrations/caddy'}", + }, + ) + expectedTenantConfig := mustNewActiveSeriesCustomTrackersConfigFromMap( + t, map[string]string{ + "team_A": "{grafanacloud_team='team_a'}", + "team_B": "{grafanacloud_team='team_b'}", + }, + ) r := ActiveSeriesCustomTrackersOverrides{} input := ` default: @@ -118,6 +150,9 @@ tenant_specific: ` require.NoError(t, yaml.UnmarshalStrict([]byte(input), &r)) + require.Equal(t, expectedDefaultConfig.String(), r.Default.String()) + require.Equal(t, expectedTenantConfig.String(), r.TenantSpecific["1"].String()) + } func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { @@ -152,11 +187,11 @@ func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { } func TestMatchersForUser(t *testing.T) { - defaultMatchers := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + defaultMatchers := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "foo": `{foo="bar"}`, "bar": `{baz="bar"}`, }) - tenantSpecificMatchers := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + tenantSpecificMatchers := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "team_a": `{team="team_a"}`, "team_b": `{team="team_b"}`, }) @@ -190,32 +225,37 @@ func TestMatchersForUser(t *testing.T) { } func TestActiveSeriesCustomTrackerConfig_Equality(t *testing.T) { - matcherSets := [][]string{ + configSets := [][]ActiveSeriesCustomTrackersConfig{ { - `foo:{foo="bar"};baz:{baz="bar"}`, - `baz:{baz="bar"};foo:{foo="bar"}`, - ` foo:{foo="bar"};baz:{baz="bar"} `, + *mustNewActiveSeriesCustomTrackersConfigFromString(t, `foo:{foo='bar'};baz:{baz='bar'}`), + *mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "baz": `{baz='bar'}`, + "foo": `{foo='bar'}`, + }), + *mustNewActiveSeriesCustomTrackersConfigDeserializedFromYaml(t, + ` + baz: "{baz='bar'}" + foo: "{foo='bar'}"`), }, { - `test:{test="true"}`, + *mustNewActiveSeriesCustomTrackersConfigFromString(t, `test:{test='true'}`), + *mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"test": `{test='true'}`}), + *mustNewActiveSeriesCustomTrackersConfigDeserializedFromYaml(t, `test: "{test='true'}"`), }, { - `foo:{foo="bar"};baz:{baz="bar"};extra:{extra="extra"}`, + *mustNewActiveSeriesCustomTrackersConfigDeserializedFromYaml(t, + ` + baz: "{baz='bar'}" + foo: "{foo='bar'}" + extra: "{extra='extra'}"`), }, } - for _, matcherSet := range matcherSets { + for _, configSet := range configSets { t.Run("EqualityBetweenSet", func(t *testing.T) { - var activeSeriesCustomTrackersConfigs []*ActiveSeriesCustomTrackersConfig - for _, matcherConfig := range matcherSet { - config := &ActiveSeriesCustomTrackersConfig{} - err := config.Set(matcherConfig) - require.NoError(t, err) - activeSeriesCustomTrackersConfigs = append(activeSeriesCustomTrackersConfigs, config) - } - for i := 0; i < len(activeSeriesCustomTrackersConfigs); i++ { - for j := i + 1; j < len(activeSeriesCustomTrackersConfigs); j++ { - assert.Equal(t, activeSeriesCustomTrackersConfigs[i].String(), activeSeriesCustomTrackersConfigs[j].String(), "matcher configs should be equal") + for i := 0; i < len(configSet); i++ { + for j := i + 1; j < len(configSet); j++ { + assert.Equal(t, configSet[i].String(), configSet[j].String(), "matcher configs should be equal") } } }) @@ -223,12 +263,8 @@ func TestActiveSeriesCustomTrackerConfig_Equality(t *testing.T) { t.Run("NotEqualsAcrossSets", func(t *testing.T) { var activeSeriesMatchers []*ActiveSeriesCustomTrackersConfig - for _, matcherConfigs := range matcherSets { - exampleConfig := matcherConfigs[0] - config := &ActiveSeriesCustomTrackersConfig{} - err := config.Set(exampleConfig) - require.NoError(t, err) - activeSeriesMatchers = append(activeSeriesMatchers, config) + for _, matcherConfigs := range configSets { + activeSeriesMatchers = append(activeSeriesMatchers, &matcherConfigs[0]) } for i := 0; i < len(activeSeriesMatchers); i++ { @@ -239,3 +275,32 @@ func TestActiveSeriesCustomTrackerConfig_Equality(t *testing.T) { }) } + +func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { + correctInput := ` + baz: "{baz='bar'}" + foo: "{foo='bar'}" + ` + malformedInput := + ` + baz: "123" + foo: "{foo='bar'}" + ` + t.Run("ShouldDeserializeCorrectInput", func(t *testing.T) { + config := ActiveSeriesCustomTrackersConfig{} + err := yaml.Unmarshal([]byte(correctInput), &config) + assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") + expectedConfig, err := newActiveSeriesCustomTrackersConfig(map[string]string{ + "baz": "{baz='bar'}", + "foo": "{foo='bar'}", + }) + require.NoError(t, err) + assert.Equal(t, expectedConfig.String(), config.String()) + }) + + t.Run("ShouldErrorOnMalformedInput", func(t *testing.T) { + config := ActiveSeriesCustomTrackersConfig{} + err := yaml.Unmarshal([]byte(malformedInput), &config) + assert.Error(t, err, "should not deserialize malformed input") + }) +} diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index 697a2e03e7..0041d9d388 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -50,7 +50,7 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { ls2 := []labels.Label{{Name: "a", Value: "2"}} ls3 := []labels.Label{{Name: "a", Value: "3"}} - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{"foo": `{a=~"2|3"}`})) + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~"2|3"}`})) c := NewActiveSeries(asm) allActive, activeMatching := c.Active() @@ -132,7 +132,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { {{Name: "_", Value: "KiqbryhzUpn"}, {Name: "__name__", Value: "logs"}}, } - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{"foo": `{_=~"y.*"}`})) + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{_=~"y.*"}`})) // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { @@ -196,7 +196,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{"foo": `{a=~.*}`})) + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~.*}`})) c := NewActiveSeries(asm) allActive, activeMatching := c.Active() diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 4cc6693df7..0729bfe982 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -250,9 +250,6 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus } asm := NewActiveSeriesMatchers(&cfg.ActiveSeriesCustomTrackersConfig) - if err != nil { - return nil, errors.Wrap(err, "failed to parse active series matchers config") - } return &Ingester{ cfg: cfg, diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index fea9532490..d832672355 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5147,12 +5147,12 @@ func TestIngesterActiveSeries(t *testing.T) { defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { return &ActiveSeriesCustomTrackersOverrides{ - Default: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + Default: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }), TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ - "test_user": mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }), @@ -5161,7 +5161,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, } - activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, }) @@ -5507,7 +5507,7 @@ func TestIngesterActiveSeries(t *testing.T) { activeSeriesOverridesProvider: &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { return &ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ - "test_user": mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }), @@ -5606,12 +5606,12 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ func() *ActiveSeriesCustomTrackersOverrides { return &ActiveSeriesCustomTrackersOverrides{ - Default: mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + Default: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }), TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ - "test_user": mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, }), @@ -5620,7 +5620,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { }, } - activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfig(t, map[string]string{ + activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, }) From 30ae4f66fa71dedda246b4b56f726fb3837d704a Mon Sep 17 00:00:00 2001 From: Janos Date: Fri, 25 Feb 2022 09:11:04 +0100 Subject: [PATCH 55/91] Replacing matchings for active series and including test case for this --- pkg/ingester/active_series.go | 22 ++++++++- pkg/ingester/active_series_test.go | 22 +++++++++ pkg/ingester/ingester_test.go | 77 ++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 2 deletions(-) diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index bddd47c52d..489c0a0e19 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -87,7 +87,7 @@ func (c *ActiveSeries) UpdateSeries(series labels.Labels, now time.Time, labelsC fp := fingerprint(series) stripeID := fp % numActiveSeriesStripes - c.stripes[stripeID].updateSeriesTimestamp(now, series, fp, labelsCopy) + c.stripes[stripeID].updateSeriesTimestamp(now, series, fp, labelsCopy, c.lastUpdate) } var sep = []byte{model.SeparatorByte} @@ -145,7 +145,7 @@ func (s *activeSeriesStripe) getTotalAndUpdateMatching(matching []int) int { return s.active } -func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels.Labels, fingerprint uint64, labelsCopy func(labels.Labels) labels.Labels) { +func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels.Labels, fingerprint uint64, labelsCopy func(labels.Labels) labels.Labels, reloadTime time.Time) { nowNanos := now.UnixNano() e := s.findEntryForSeries(fingerprint, series) @@ -157,6 +157,10 @@ func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels. if !entryTimeSet { if prev := e.Load(); nowNanos > prev { entryTimeSet = e.CAS(prev, nowNanos) + r := reloadTime.UnixNano() + if prev <= r { + s.replaceMatchers(fingerprint, series, nowNanos, labelsCopy) + } } } @@ -216,6 +220,20 @@ func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, seri return e.nanos, true } +func (s *activeSeriesStripe) replaceMatchers(fingerprint uint64, series labels.Labels, nowNanos int64, labelsCopy func(labels.Labels) labels.Labels) { + s.mu.Lock() + defer s.mu.Unlock() + for ix, entry := range s.refs[fingerprint] { + if labels.Equal(entry.lbs, series) { + s.refs[fingerprint][ix].matches = s.asm.Matches(series) + // There is a ref, the series is still active, no need to increase s.active. + // s.activeMatching is not exposed until ActiveSeriesMetricsIdleTimeout passes since reload, it will be recounted at purge time. + // If ActiveSeriesMetricsIdleTimeout passes between two samples, the series is not active, if less, the purge time recalculation will consider the newer sample. + break + } + } +} + //nolint // Linter reports that this method is unused, but it is. func (s *activeSeriesStripe) clear() { s.mu.Lock() diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index 0041d9d388..5d60d2a4d9 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -195,6 +195,8 @@ func TestActiveSeries_PurgeOpt(t *testing.T) { func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} + ls3 := []labels.Label{{Name: "a", Value: "3"}} + ls4 := []labels.Label{{Name: "a", Value: "4"}} asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~.*}`})) @@ -215,6 +217,26 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { assert.Equal(t, 2, allActive, "reloading should not affect general counter") assert.Equal(t, []int{1}, activeMatching, "reloading should clear out matcher counters") assert.True(t, c.lastUpdate.Equal(reloadTime)) + + asmWithLessMatchers := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{})) + reloadTime = time.Now() + c.ReloadSeriesMatchers(asmWithLessMatchers, reloadTime) + c.UpdateSeries(ls3, time.Now(), copyFn) + allActive, activeMatching = c.Active() + assert.Equal(t, 3, allActive, "reloading should not affect general counter") + assert.Equal(t, []int(nil), activeMatching, "reloading should clear out and resize matcher counters") + + asmWithMoreMatchers := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "a": `{a="3"}`, + "b": `{a="4"}`, + })) + reloadTime = time.Now() + c.ReloadSeriesMatchers(asmWithMoreMatchers, reloadTime) + c.UpdateSeries(ls4, time.Now(), copyFn) + allActive, activeMatching = c.Active() + assert.Equal(t, 4, allActive, "reloading should not affect general counter") + assert.Equal(t, []int{0, 1}, activeMatching, "reloading should clear out and resize matcher counters") + } var activeSeriesTestGoroutines = []int{50, 100, 500} diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index d832672355..0b21863163 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5749,6 +5749,83 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, }, + "changing runtime overwrite should result in new metrics": { + activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, + activeSeriesConfig: *activeSeriesDefaultConfig, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + firstPushTime := time.Now() + + for _, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + _, err := ingester.Push(ctx, req(label, firstPushTime)) + require.NoError(t, err) + } + + // Update active series for metrics check. + ingester.updateActiveSeries(firstPushTime) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + ` + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + + // Change runtime configs + ingester.cfg.ActiveSeriesCustomTrackersOverrides = &ActiveSeriesCustomTrackersOverridesProvider{ + func() *ActiveSeriesCustomTrackersOverrides { + return &ActiveSeriesCustomTrackersOverrides{ + TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + "team_c": `{team="b"}`, + "team_d": `{team="b"}`, + }), + }, + } + }, + } + // This will update the runtime config. + configReloadTime := time.Now() + ingester.updateActiveSeries(configReloadTime) + expectedMetrics = ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + ` + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + + // Sleep to emphasize that secondPushTime must be > configReloadTime not to be purged. + time.Sleep(time.Millisecond) + secondPushtime := time.Now() + for _, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + _, err := ingester.Push(ctx, req(label, secondPushtime)) + require.NoError(t, err) + } + // Adding a nanosecond to make updateTime.Before(purgeTime) true + updateTime := configReloadTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout + time.Nanosecond) + ingester.updateActiveSeries(updateTime) + expectedMetrics = ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_c",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_d",user="test_user"} 2 + ` + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + }, + }, } for testName, testData := range tests { From dc4b3e3779d7256a8bf27c4f1325b810f234d8d4 Mon Sep 17 00:00:00 2001 From: Janos Date: Mon, 28 Feb 2022 09:29:19 +0100 Subject: [PATCH 56/91] Rework and implementing oleg and pstibrany's suggestions --- pkg/ingester/active_series.go | 89 ++++++------ pkg/ingester/active_series_test.go | 211 +++++++++++++++++++---------- pkg/ingester/ingester.go | 36 +++-- pkg/ingester/ingester_test.go | 5 +- 4 files changed, 202 insertions(+), 139 deletions(-) diff --git a/pkg/ingester/active_series.go b/pkg/ingester/active_series.go index 489c0a0e19..42f3ace069 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/active_series.go @@ -22,9 +22,12 @@ const ( // ActiveSeries is keeping track of recently active series for a single tenant. type ActiveSeries struct { - asm *ActiveSeriesMatchers - stripes [numActiveSeriesStripes]activeSeriesStripe - lastUpdate time.Time + stripes [numActiveSeriesStripes]activeSeriesStripe + asm *ActiveSeriesMatchers + lastAsmUpdate *atomic.Int64 + // The duration after series become inactive. + timeout time.Duration + mu sync.RWMutex } // activeSeriesStripe holds a subset of the series timestamps for a single tenant. @@ -49,37 +52,36 @@ type activeSeriesEntry struct { matches []bool // Which matchers of ActiveSeriesMatchers does this series match } -func NewActiveSeries(asm *ActiveSeriesMatchers) *ActiveSeries { - c := &ActiveSeries{asm: asm} +func NewActiveSeries(asm *ActiveSeriesMatchers, idleTimeout time.Duration) *ActiveSeries { + c := &ActiveSeries{asm: asm, timeout: idleTimeout, lastAsmUpdate: atomic.NewInt64(0)} // Stripes are pre-allocated so that we only read on them and no lock is required. for i := 0; i < numActiveSeriesStripes; i++ { c.stripes[i] = activeSeriesStripe{ asm: asm, refs: map[uint64][]activeSeriesEntry{}, - activeMatching: makeIntSliceIfNotEmpty(len(asm.MatcherNames()), nil), + activeMatching: resizeAndClear(len(asm.MatcherNames()), nil), } } return c } -func (c *ActiveSeries) CurrentMatchers() *ActiveSeriesMatchers { - return c.asm +func (c *ActiveSeries) CurrentMatcherNames() []string { + c.mu.RLock() + defer c.mu.RUnlock() + return c.asm.MatcherNames() } -func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers, reloadTime time.Time) { - c.asm = asm - for i := 0; i < numActiveSeriesStripes; i++ { - c.stripes[i].mu.Lock() - defer c.stripes[i].mu.Unlock() - } +func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers) { + c.mu.Lock() + defer c.mu.Unlock() for i := 0; i < numActiveSeriesStripes; i++ { - c.stripes[i].asm = asm - c.stripes[i].activeMatching = makeIntSliceIfNotEmpty(len(asm.MatcherNames()), c.stripes[i].activeMatching) + c.stripes[i].reinitialize(asm) } - c.lastUpdate = reloadTime + c.asm = asm + c.lastAsmUpdate.Store(time.Now().UnixNano()) } // Updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. @@ -87,7 +89,7 @@ func (c *ActiveSeries) UpdateSeries(series labels.Labels, now time.Time, labelsC fp := fingerprint(series) stripeID := fp % numActiveSeriesStripes - c.stripes[stripeID].updateSeriesTimestamp(now, series, fp, labelsCopy, c.lastUpdate) + c.stripes[stripeID].updateSeriesTimestamp(now, series, fp, labelsCopy) } var sep = []byte{model.SeparatorByte} @@ -105,9 +107,8 @@ func fingerprint(series labels.Labels) uint64 { return sum.Sum64() } -// Purge removes expired entries from the cache. This function should be called -// periodically to avoid memory leaks. -func (c *ActiveSeries) Purge(keepUntil time.Time) { +// Purge removes expired entries from the cache. This function is called by Active. +func (c *ActiveSeries) purge(keepUntil time.Time) { for s := 0; s < numActiveSeriesStripes; s++ { c.stripes[s].purge(keepUntil) } @@ -122,9 +123,14 @@ func (c *ActiveSeries) clear() { // Active returns the total number of active series, as well as a slice of active series matching each one of the // custom trackers provided (in the same order as custom trackers are defined) -func (c *ActiveSeries) Active() (int, []int) { +// This should be called periodically to avoid memory leaks. +// Active cannot be called concurrently with ReloadSeriesMatchers. +func (c *ActiveSeries) Active(now time.Time) (int, []int) { + c.mu.Lock() + defer c.mu.Unlock() + c.purge(now.Add(-c.timeout)) total := 0 - totalMatching := makeIntSliceIfNotEmpty(len(c.asm.MatcherNames()), nil) + totalMatching := resizeAndClear(len(c.asm.MatcherNames()), nil) for s := 0; s < numActiveSeriesStripes; s++ { total += c.stripes[s].getTotalAndUpdateMatching(totalMatching) } @@ -145,7 +151,7 @@ func (s *activeSeriesStripe) getTotalAndUpdateMatching(matching []int) int { return s.active } -func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels.Labels, fingerprint uint64, labelsCopy func(labels.Labels) labels.Labels, reloadTime time.Time) { +func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels.Labels, fingerprint uint64, labelsCopy func(labels.Labels) labels.Labels) { nowNanos := now.UnixNano() e := s.findEntryForSeries(fingerprint, series) @@ -157,10 +163,6 @@ func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels. if !entryTimeSet { if prev := e.Load(); nowNanos > prev { entryTimeSet = e.CAS(prev, nowNanos) - r := reloadTime.UnixNano() - if prev <= r { - s.replaceMatchers(fingerprint, series, nowNanos, labelsCopy) - } } } @@ -220,20 +222,6 @@ func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, seri return e.nanos, true } -func (s *activeSeriesStripe) replaceMatchers(fingerprint uint64, series labels.Labels, nowNanos int64, labelsCopy func(labels.Labels) labels.Labels) { - s.mu.Lock() - defer s.mu.Unlock() - for ix, entry := range s.refs[fingerprint] { - if labels.Equal(entry.lbs, series) { - s.refs[fingerprint][ix].matches = s.asm.Matches(series) - // There is a ref, the series is still active, no need to increase s.active. - // s.activeMatching is not exposed until ActiveSeriesMetricsIdleTimeout passes since reload, it will be recounted at purge time. - // If ActiveSeriesMetricsIdleTimeout passes between two samples, the series is not active, if less, the purge time recalculation will consider the newer sample. - break - } - } -} - //nolint // Linter reports that this method is unused, but it is. func (s *activeSeriesStripe) clear() { s.mu.Lock() @@ -247,6 +235,18 @@ func (s *activeSeriesStripe) clear() { } } +// Reinitalize is more than clear that it assigns new matchers and corresponding size activeMatching slices. +func (s *activeSeriesStripe) reinitialize(asm *ActiveSeriesMatchers) { + s.mu.Lock() + defer s.mu.Unlock() + + s.oldestEntryTs.Store(0) + s.refs = map[uint64][]activeSeriesEntry{} + s.active = 0 + s.asm = asm + s.activeMatching = resizeAndClear(len(asm.MatcherNames()), s.activeMatching) +} + func (s *activeSeriesStripe) purge(keepUntil time.Time) { keepUntilNanos := keepUntil.UnixNano() if oldest := s.oldestEntryTs.Load(); oldest > 0 && keepUntilNanos <= oldest { @@ -258,7 +258,7 @@ func (s *activeSeriesStripe) purge(keepUntil time.Time) { defer s.mu.Unlock() active := 0 - activeMatching := makeIntSliceIfNotEmpty(len(s.activeMatching), s.activeMatching) + activeMatching := resizeAndClear(len(s.activeMatching), s.activeMatching) oldest := int64(math.MaxInt64) for fp, entries := range s.refs { @@ -324,12 +324,13 @@ func (s *activeSeriesStripe) purge(keepUntil time.Time) { s.activeMatching = activeMatching } -func makeIntSliceIfNotEmpty(l int, prev []int) []int { +func resizeAndClear(l int, prev []int) []int { if cap(prev) < l { if l == 0 { return nil } // The allocation is bigger than the required capacity to save time in cases when the number of matchers are just slightly increasing. + // In cases where the default matchers are slightly changed in size it could save from lot of reallocations, while having low memory impact. return make([]int, l, l*2) } diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/active_series_test.go index 5d60d2a4d9..3b448077a0 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/active_series_test.go @@ -27,21 +27,22 @@ func TestActiveSeries_UpdateSeries_NoMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - c := NewActiveSeries(&ActiveSeriesMatchers{}) - allActive, activeMatching := c.Active() + currentNow := time.Now() + c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) + allActive, activeMatching := c.Active(currentNow) assert.Equal(t, 0, allActive) assert.Nil(t, activeMatching) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, _ = c.Active() + allActive, _ = c.Active(currentNow) assert.Equal(t, 1, allActive) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, _ = c.Active() + allActive, _ = c.Active(currentNow) assert.Equal(t, 1, allActive) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, _ = c.Active() + allActive, _ = c.Active(currentNow) assert.Equal(t, 2, allActive) } @@ -52,28 +53,29 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~"2|3"}`})) - c := NewActiveSeries(asm) - allActive, activeMatching := c.Active() + currentNow := time.Now() + c := NewActiveSeries(asm, 5*time.Minute) + allActive, activeMatching := c.Active(currentNow) assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching = c.Active() + allActive, activeMatching = c.Active(currentNow) assert.Equal(t, 1, allActive) assert.Equal(t, []int{0}, activeMatching) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, activeMatching = c.Active() + allActive, activeMatching = c.Active(currentNow) assert.Equal(t, 2, allActive) assert.Equal(t, []int{1}, activeMatching) c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching = c.Active() + allActive, activeMatching = c.Active(currentNow) assert.Equal(t, 3, allActive) assert.Equal(t, []int{2}, activeMatching) c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching = c.Active() + allActive, activeMatching = c.Active(currentNow) assert.Equal(t, 3, allActive) assert.Equal(t, []int{2}, activeMatching) } @@ -84,12 +86,12 @@ func TestActiveSeries_ShouldCorrectlyHandleFingerprintCollisions(t *testing.T) { ls2 := metric.Set("_", "KiqbryhzUpn").Labels() require.True(t, client.Fingerprint(ls1) == client.Fingerprint(ls2)) - - c := NewActiveSeries(&ActiveSeriesMatchers{}) + currentNow := time.Now() + c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) c.UpdateSeries(ls1, time.Now(), copyFn) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, _ := c.Active() + allActive, _ := c.Active(currentNow) assert.Equal(t, 2, allActive) } @@ -105,18 +107,19 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl: %d", ttl), func(t *testing.T) { - c := NewActiveSeries(&ActiveSeriesMatchers{}) + c := NewActiveSeries(&ActiveSeriesMatchers{}, time.Since(time.Unix(0, 0))) for i := 0; i < len(series); i++ { c.UpdateSeries(series[i], time.Unix(int64(i), 0), copyFn) } - c.Purge(time.Unix(int64(ttl), 0)) + c.purge(time.Unix(int64(ttl), 0)) // call purge twice, just to hit "quick" path. It doesn't really do anything. - c.Purge(time.Unix(int64(ttl), 0)) + c.purge(time.Unix(int64(ttl), 0)) exp := len(series) - (ttl) - allActive, activeMatching := c.Active() + // c.Active is not intented to purge + allActive, activeMatching := c.Active(time.Unix(0, 0)) assert.Equal(t, exp, allActive) assert.Nil(t, activeMatching) }) @@ -137,7 +140,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl=%d", ttl), func(t *testing.T) { - c := NewActiveSeries(asm) + c := NewActiveSeries(asm, time.Since(time.Unix(0, 0))) exp := len(series) - ttl expMatchingSeries := 0 @@ -151,11 +154,12 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { } } - c.Purge(time.Unix(int64(ttl), 0)) + c.purge(time.Unix(int64(ttl), 0)) // call purge twice, just to hit "quick" path. It doesn't really do anything. - c.Purge(time.Unix(int64(ttl), 0)) + c.purge(time.Unix(int64(ttl), 0)) - allActive, activeMatching := c.Active() + // c.Active is not intented to purge + allActive, activeMatching := c.Active(time.Unix(0, 0)) assert.Equal(t, exp, allActive) assert.Equal(t, []int{expMatchingSeries}, activeMatching) }) @@ -167,28 +171,25 @@ func TestActiveSeries_PurgeOpt(t *testing.T) { ls1 := metric.Set("_", "ypfajYg2lsv").Labels() ls2 := metric.Set("_", "KiqbryhzUpn").Labels() - c := NewActiveSeries(&ActiveSeriesMatchers{}) + currentNow := time.Now() + c := NewActiveSeries(&ActiveSeriesMatchers{}, 59*time.Second) - now := time.Now() - c.UpdateSeries(ls1, now.Add(-2*time.Minute), copyFn) - c.UpdateSeries(ls2, now, copyFn) - c.Purge(now) + c.UpdateSeries(ls1, currentNow.Add(-2*time.Minute), copyFn) + c.UpdateSeries(ls2, currentNow, copyFn) - allActive, _ := c.Active() + allActive, _ := c.Active(currentNow) assert.Equal(t, 1, allActive) - c.UpdateSeries(ls1, now.Add(-1*time.Minute), copyFn) - c.UpdateSeries(ls2, now, copyFn) - c.Purge(now) + c.UpdateSeries(ls1, currentNow.Add(-1*time.Minute), copyFn) + c.UpdateSeries(ls2, currentNow, copyFn) - allActive, _ = c.Active() + allActive, _ = c.Active(currentNow) assert.Equal(t, 1, allActive) // This will *not* update the series, since there is already newer timestamp. - c.UpdateSeries(ls2, now.Add(-1*time.Minute), copyFn) - c.Purge(now) + c.UpdateSeries(ls2, currentNow.Add(-1*time.Minute), copyFn) - allActive, _ = c.Active() + allActive, _ = c.Active(currentNow) assert.Equal(t, 1, allActive) } @@ -200,43 +201,110 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~.*}`})) - c := NewActiveSeries(asm) - allActive, activeMatching := c.Active() + currentNow := time.Now() + c := NewActiveSeries(asm, 5*time.Minute) + + allActive, activeMatching := c.Active(time.Now()) assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) - c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching = c.Active() + c.UpdateSeries(ls1, currentNow, copyFn) + allActive, activeMatching = c.Active(time.Now()) assert.Equal(t, 1, allActive) assert.Equal(t, []int{1}, activeMatching) - reloadTime := time.Now() - c.ReloadSeriesMatchers(asm, reloadTime) - c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, activeMatching = c.Active() - assert.Equal(t, 2, allActive, "reloading should not affect general counter") - assert.Equal(t, []int{1}, activeMatching, "reloading should clear out matcher counters") - assert.True(t, c.lastUpdate.Equal(reloadTime)) + c.ReloadSeriesMatchers(asm) + assert.Greater(t, c.lastAsmUpdate.Load(), currentNow.UnixNano()) + assert.Less(t, c.lastAsmUpdate.Load(), time.Now().UnixNano()) + allActive, activeMatching = c.Active(time.Now()) + assert.Equal(t, 0, allActive) + assert.Equal(t, []int{0}, activeMatching) + + c.UpdateSeries(ls1, currentNow, copyFn) + c.UpdateSeries(ls2, currentNow, copyFn) + allActive, activeMatching = c.Active(time.Now()) + assert.Equal(t, 2, allActive) + assert.Equal(t, []int{2}, activeMatching) asmWithLessMatchers := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{})) - reloadTime = time.Now() - c.ReloadSeriesMatchers(asmWithLessMatchers, reloadTime) - c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching = c.Active() - assert.Equal(t, 3, allActive, "reloading should not affect general counter") - assert.Equal(t, []int(nil), activeMatching, "reloading should clear out and resize matcher counters") + c.ReloadSeriesMatchers(asmWithLessMatchers) + + c.UpdateSeries(ls3, currentNow, copyFn) + allActive, activeMatching = c.Active(time.Now()) + assert.Equal(t, 1, allActive) + assert.Equal(t, []int(nil), activeMatching) asmWithMoreMatchers := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "a": `{a="3"}`, "b": `{a="4"}`, })) - reloadTime = time.Now() - c.ReloadSeriesMatchers(asmWithMoreMatchers, reloadTime) - c.UpdateSeries(ls4, time.Now(), copyFn) - allActive, activeMatching = c.Active() - assert.Equal(t, 4, allActive, "reloading should not affect general counter") - assert.Equal(t, []int{0, 1}, activeMatching, "reloading should clear out and resize matcher counters") + c.ReloadSeriesMatchers(asmWithMoreMatchers) + + c.UpdateSeries(ls4, currentNow, copyFn) + allActive, activeMatching = c.Active(time.Now()) + assert.Equal(t, 1, allActive) + assert.Equal(t, []int{0, 1}, activeMatching) +} + +func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { + ls1 := []labels.Label{{Name: "a", Value: "1"}} + + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "foo": `{a=~.+}`, + "bar": `{a=~.+}`, + })) + + c := NewActiveSeries(asm, 5*time.Minute) + allActive, activeMatching := c.Active(time.Now()) + assert.Equal(t, 0, allActive) + assert.Equal(t, []int{0, 0}, activeMatching) + + c.UpdateSeries(ls1, time.Now(), copyFn) + allActive, activeMatching = c.Active(time.Now()) + assert.Equal(t, 1, allActive) + assert.Equal(t, []int{1, 1}, activeMatching) + + asm = NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "foo": `{a=~.+}`, + })) + + c.ReloadSeriesMatchers(asm) + c.purge(time.Time{}) + + allActive, activeMatching = c.Active(time.Now()) + assert.Equal(t, 0, allActive) + assert.Equal(t, []int{0}, activeMatching) +} + +func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { + ls1 := []labels.Label{{Name: "a", Value: "1"}} + + asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "foo": `{a=~.+}`, + "bar": `{a=~.+}`, + })) + + c := NewActiveSeries(asm, 5*time.Minute) + allActive, activeMatching := c.Active(time.Now()) + assert.Equal(t, 0, allActive) + assert.Equal(t, []int{0, 0}, activeMatching) + c.UpdateSeries(ls1, time.Now(), copyFn) + allActive, activeMatching = c.Active(time.Now()) + assert.Equal(t, 1, allActive) + assert.Equal(t, []int{1, 1}, activeMatching) + + asm = NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "foo": `{b=~.+}`, + "bar": `{b=~.+}`, + })) + + c.ReloadSeriesMatchers(asm) + c.purge(time.Time{}) + + allActive, activeMatching = c.Active(time.Now()) + assert.Equal(t, 0, allActive) + assert.Equal(t, []int{0, 0}, activeMatching) } var activeSeriesTestGoroutines = []int{50, 100, 500} @@ -254,7 +322,7 @@ func benchmarkActiveSeriesConcurrencySingleSeries(b *testing.B, goroutines int) {Name: "a", Value: "a"}, } - c := NewActiveSeries(&ActiveSeriesMatchers{}) + c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) wg := &sync.WaitGroup{} start := make(chan struct{}) @@ -281,7 +349,7 @@ func benchmarkActiveSeriesConcurrencySingleSeries(b *testing.B, goroutines int) } func BenchmarkActiveSeries_UpdateSeries(b *testing.B) { - c := NewActiveSeries(&ActiveSeriesMatchers{}) + c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) // Prepare series nameBuf := bytes.Buffer{} @@ -295,11 +363,11 @@ func BenchmarkActiveSeries_UpdateSeries(b *testing.B) { series[s] = labels.Labels{{Name: name, Value: name + strconv.Itoa(s)}} } - now := time.Now().UnixNano() + nowNano := time.Now().UnixNano() b.ResetTimer() for ix := 0; ix < b.N; ix++ { - c.UpdateSeries(series[ix], time.Unix(0, now+int64(ix)), copyFn) + c.UpdateSeries(series[ix], time.Unix(0, nowNano+int64(ix)), copyFn) } } @@ -315,8 +383,8 @@ func benchmarkPurge(b *testing.B, twice bool) { const numSeries = 10000 const numExpiresSeries = numSeries / 25 - now := time.Now() - c := NewActiveSeries(&ActiveSeriesMatchers{}) + currentNow := time.Now() + c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) series := [numSeries]labels.Labels{} for s := 0; s < numSeries; s++ { @@ -329,24 +397,25 @@ func benchmarkPurge(b *testing.B, twice bool) { // Prepare series for ix, s := range series { if ix < numExpiresSeries { - c.UpdateSeries(s, now.Add(-time.Minute), copyFn) + c.UpdateSeries(s, currentNow.Add(-time.Minute), copyFn) } else { - c.UpdateSeries(s, now, copyFn) + c.UpdateSeries(s, currentNow, copyFn) } } - allActive, _ := c.Active() + allActive, _ := c.Active(time.Now()) assert.Equal(b, numSeries, allActive) b.StartTimer() // Purge everything - c.Purge(now) - allActive, _ = c.Active() + now := time.Now() + c.purge(now) + allActive, _ = c.Active(now) assert.Equal(b, numSeries-numExpiresSeries, allActive) if twice { - c.Purge(now) - allActive, _ = c.Active() + c.purge(now) + allActive, _ = c.Active(now) assert.Equal(b, numSeries-numExpiresSeries, allActive) } } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 0729bfe982..b311713e8d 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -482,13 +482,12 @@ func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *ActiveSeriesCus return matchers } -func (i *Ingester) replaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB, now time.Time) { - i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatchers().names) - userDB.activeSeries.ReloadSeriesMatchers(asm, now) +func (i *Ingester) replaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) { + i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatcherNames()) + userDB.activeSeries.ReloadSeriesMatchers(asm) } func (i *Ingester) updateActiveSeries(now time.Time) { - purgeTime := now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout) for _, userID := range i.getTSDBUsers() { userDB := i.getTSDB(userID) if userDB == nil { @@ -497,20 +496,18 @@ func (i *Ingester) updateActiveSeries(now time.Time) { newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) if newMatchersConfig.String() != userDB.activeSeries.asm.cfg.String() { - i.replaceMatchers(NewActiveSeriesMatchers(newMatchersConfig), userDB, now) + i.replaceMatchers(NewActiveSeriesMatchers(newMatchersConfig), userDB) } + if userDB.activeSeries.lastAsmUpdate.Load() < now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout).UnixNano() { + // We are not exposing metrics until enough time passed for stable metrics. + allActive, activeMatching := userDB.activeSeries.Active(now) + if allActive > 0 { + i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(allActive)) + } else { + i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) + } - userDB.activeSeries.Purge(purgeTime) - allActive, activeMatching := userDB.activeSeries.Active() - if allActive > 0 { - i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(allActive)) - } else { - i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) - } - if userDB.activeSeries.lastUpdate.Before(purgeTime) { - // Do not publish metrics until the new matcher setup had time to catch up. - // LastUpdate is Zero when it never get updated. - for idx, name := range userDB.activeSeries.asm.names { + for idx, name := range userDB.activeSeries.CurrentMatcherNames() { // We only set the metrics for matchers that actually exist, to avoid increasing cardinality with zero valued metrics. if activeMatching[idx] > 0 { i.metrics.activeSeriesCustomTrackersPerUser.WithLabelValues(userID, name).Set(float64(activeMatching[idx])) @@ -519,7 +516,6 @@ func (i *Ingester) updateActiveSeries(now time.Time) { } } } - } } @@ -1470,7 +1466,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userDB := &userTSDB{ userID: userID, - activeSeries: NewActiveSeries(NewActiveSeriesMatchers(matchersConfig)), + activeSeries: NewActiveSeries(NewActiveSeriesMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), @@ -1593,7 +1589,7 @@ func (i *Ingester) closeAllTSDB() { i.metrics.memUsers.Dec() i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) - i.metrics.deletePerUserCustomTrackerMetrics(userID, db.activeSeries.asm.names) + i.metrics.deletePerUserCustomTrackerMetrics(userID, db.activeSeries.CurrentMatcherNames()) }(userDB) } @@ -1990,7 +1986,7 @@ func (i *Ingester) closeAndDeleteUserTSDBIfIdle(userID string) tsdbCloseCheckRes i.deleteUserMetadata(userID) i.metrics.deletePerUserMetrics(userID) - i.metrics.deletePerUserCustomTrackerMetrics(userID, userDB.activeSeries.asm.names) + i.metrics.deletePerUserCustomTrackerMetrics(userID, userDB.activeSeries.CurrentMatcherNames()) validation.DeletePerUserValidationMetrics(userID, i.logger) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 0b21863163..856eee0d86 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5801,16 +5801,13 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // Sleep to emphasize that secondPushTime must be > configReloadTime not to be purged. - time.Sleep(time.Millisecond) secondPushtime := time.Now() for _, label := range labelsToPush { ctx := user.InjectOrgID(context.Background(), userID) _, err := ingester.Push(ctx, req(label, secondPushtime)) require.NoError(t, err) } - // Adding a nanosecond to make updateTime.Before(purgeTime) true - updateTime := configReloadTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout + time.Nanosecond) + updateTime := secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout) ingester.updateActiveSeries(updateTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. From 0132477bf5c6fe9bbf03bbcfb66697dc009fa373 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 1 Mar 2022 12:55:39 +0100 Subject: [PATCH 57/91] Moving active series to separate package --- .../{ => activeseries}/active_series.go | 12 ++++- .../active_series_custom_tracker.go | 6 ++- .../active_series_custom_tracker_test.go | 4 +- .../active_series_custom_trackers_config.go | 8 +-- ...tive_series_custom_trackers_config_test.go | 6 +-- .../{ => activeseries}/active_series_test.go | 2 +- pkg/ingester/ingester.go | 31 +++++------ pkg/ingester/ingester_test.go | 51 +++++++++++-------- pkg/ingester/user_tsdb.go | 3 +- pkg/mimir/runtime_config.go | 9 ++-- tools/doc-generator/parser.go | 4 +- 11 files changed, 80 insertions(+), 56 deletions(-) rename pkg/ingester/{ => activeseries}/active_series.go (97%) rename pkg/ingester/{ => activeseries}/active_series_custom_tracker.go (95%) rename pkg/ingester/{ => activeseries}/active_series_custom_tracker_test.go (97%) rename pkg/ingester/{ => activeseries}/active_series_custom_trackers_config.go (96%) rename pkg/ingester/{ => activeseries}/active_series_custom_trackers_config_test.go (98%) rename pkg/ingester/{ => activeseries}/active_series_test.go (99%) diff --git a/pkg/ingester/active_series.go b/pkg/ingester/activeseries/active_series.go similarity index 97% rename from pkg/ingester/active_series.go rename to pkg/ingester/activeseries/active_series.go index 42f3ace069..7a9a7ee7b8 100644 --- a/pkg/ingester/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -3,7 +3,7 @@ // Provenance-includes-license: Apache-2.0 // Provenance-includes-copyright: The Cortex Authors. -package ingester +package activeseries import ( "math" @@ -84,6 +84,16 @@ func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers) { c.lastAsmUpdate.Store(time.Now().UnixNano()) } +func (c *ActiveSeries) LastAsmUpdate() int64 { + return c.lastAsmUpdate.Load() +} + +func (c *ActiveSeries) CurrentConfig() *ActiveSeriesCustomTrackersConfig { + c.mu.RLock() + defer c.mu.RUnlock() + return c.asm.Config() +} + // Updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. func (c *ActiveSeries) UpdateSeries(series labels.Labels, now time.Time, labelsCopy func(labels.Labels) labels.Labels) { fp := fingerprint(series) diff --git a/pkg/ingester/active_series_custom_tracker.go b/pkg/ingester/activeseries/active_series_custom_tracker.go similarity index 95% rename from pkg/ingester/active_series_custom_tracker.go rename to pkg/ingester/activeseries/active_series_custom_tracker.go index 73113d3253..fa96ed239e 100644 --- a/pkg/ingester/active_series_custom_tracker.go +++ b/pkg/ingester/activeseries/active_series_custom_tracker.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-only -package ingester +package activeseries import ( "sort" @@ -31,6 +31,10 @@ func (asm *ActiveSeriesMatchers) MatcherNames() []string { return asm.names } +func (asm *ActiveSeriesMatchers) Config() *ActiveSeriesCustomTrackersConfig { + return asm.cfg +} + func (asm *ActiveSeriesMatchers) Matches(series labels.Labels) []bool { if len(asm.matchers) == 0 { return nil diff --git a/pkg/ingester/active_series_custom_tracker_test.go b/pkg/ingester/activeseries/active_series_custom_tracker_test.go similarity index 97% rename from pkg/ingester/active_series_custom_tracker_test.go rename to pkg/ingester/activeseries/active_series_custom_tracker_test.go index 2083007b17..f6ed55dc51 100644 --- a/pkg/ingester/active_series_custom_tracker_test.go +++ b/pkg/ingester/activeseries/active_series_custom_tracker_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-only -package ingester +package activeseries import ( "testing" @@ -96,7 +96,7 @@ func TestActiveSeriesCustomTrackersConfigs_MalformedMatcher(t *testing.T) { "malformed": matcher, } - _, err := newActiveSeriesCustomTrackersConfig(config) + _, err := NewActiveSeriesCustomTrackersConfig(config) assert.Error(t, err) }) } diff --git a/pkg/ingester/active_series_custom_trackers_config.go b/pkg/ingester/activeseries/active_series_custom_trackers_config.go similarity index 96% rename from pkg/ingester/active_series_custom_trackers_config.go rename to pkg/ingester/activeseries/active_series_custom_trackers_config.go index d028336ee1..e77cc8f064 100644 --- a/pkg/ingester/active_series_custom_trackers_config.go +++ b/pkg/ingester/activeseries/active_series_custom_trackers_config.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-only -package ingester +package activeseries import ( "fmt" @@ -71,7 +71,7 @@ func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { return err } - nc, err := newActiveSeriesCustomTrackersConfig(f) + nc, err := NewActiveSeriesCustomTrackersConfig(f) if err != nil { return err } @@ -125,11 +125,11 @@ func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interfac if err != nil { return err } - *c, err = newActiveSeriesCustomTrackersConfig(stringMap) + *c, err = NewActiveSeriesCustomTrackersConfig(stringMap) return err } -func newActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCustomTrackersConfig, err error) { +func NewActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCustomTrackersConfig, err error) { c.source = m c.config = map[string]labelsMatchers{} for name, matcher := range m { diff --git a/pkg/ingester/active_series_custom_trackers_config_test.go b/pkg/ingester/activeseries/active_series_custom_trackers_config_test.go similarity index 98% rename from pkg/ingester/active_series_custom_trackers_config_test.go rename to pkg/ingester/activeseries/active_series_custom_trackers_config_test.go index 362a8ae326..f9edd50e21 100644 --- a/pkg/ingester/active_series_custom_trackers_config_test.go +++ b/pkg/ingester/activeseries/active_series_custom_trackers_config_test.go @@ -1,6 +1,6 @@ // SPDX-License-Identifier: AGPL-3.0-only -package ingester +package activeseries import ( "flag" @@ -13,7 +13,7 @@ import ( ) func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *ActiveSeriesCustomTrackersConfig { - m, err := newActiveSeriesCustomTrackersConfig(source) + m, err := NewActiveSeriesCustomTrackersConfig(source) require.NoError(t, err) return &m } @@ -290,7 +290,7 @@ func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { config := ActiveSeriesCustomTrackersConfig{} err := yaml.Unmarshal([]byte(correctInput), &config) assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") - expectedConfig, err := newActiveSeriesCustomTrackersConfig(map[string]string{ + expectedConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ "baz": "{baz='bar'}", "foo": "{foo='bar'}", }) diff --git a/pkg/ingester/active_series_test.go b/pkg/ingester/activeseries/active_series_test.go similarity index 99% rename from pkg/ingester/active_series_test.go rename to pkg/ingester/activeseries/active_series_test.go index 3b448077a0..552192adf9 100644 --- a/pkg/ingester/active_series_test.go +++ b/pkg/ingester/activeseries/active_series_test.go @@ -3,7 +3,7 @@ // Provenance-includes-license: Apache-2.0 // Provenance-includes-copyright: The Cortex Authors. -package ingester +package activeseries import ( "bytes" diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index b311713e8d..226c140f40 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -12,6 +12,7 @@ import ( "context" "flag" "fmt" + "github.com/grafana/mimir/pkg/ingester/activeseries" "io" "net/http" "os" @@ -121,11 +122,11 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackersConfig ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` - ActiveSeriesCustomTrackersOverrides *ActiveSeriesCustomTrackersOverridesProvider `yaml:"-"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesCustomTrackersConfig activeseries.ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` + ActiveSeriesCustomTrackersOverrides *activeseries.ActiveSeriesCustomTrackersOverridesProvider `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -198,7 +199,7 @@ type Ingester struct { metrics *ingesterMetrics logger log.Logger - activeSeriesMatcher ActiveSeriesMatchers + activeSeriesMatcher activeseries.ActiveSeriesMatchers lifecycler *ring.Lifecycler limits *validation.Overrides @@ -249,7 +250,7 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus return nil, errors.Wrap(err, "failed to create the bucket client") } - asm := NewActiveSeriesMatchers(&cfg.ActiveSeriesCustomTrackersConfig) + asm := activeseries.NewActiveSeriesMatchers(&cfg.ActiveSeriesCustomTrackersConfig) return &Ingester{ cfg: cfg, @@ -471,18 +472,18 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *ActiveSeriesCustomTrackersConfig { - var matchers *ActiveSeriesCustomTrackersConfig +func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *activeseries.ActiveSeriesCustomTrackersConfig { + var matchers *activeseries.ActiveSeriesCustomTrackersConfig if cfg := i.cfg.ActiveSeriesCustomTrackersOverrides.Get(); cfg != nil { matchers = cfg.MatchersConfigForUser(userID) } if matchers == nil { - matchers = i.activeSeriesMatcher.cfg + matchers = i.activeSeriesMatcher.Config() } return matchers } -func (i *Ingester) replaceMatchers(asm *ActiveSeriesMatchers, userDB *userTSDB) { +func (i *Ingester) replaceMatchers(asm *activeseries.ActiveSeriesMatchers, userDB *userTSDB) { i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatcherNames()) userDB.activeSeries.ReloadSeriesMatchers(asm) } @@ -495,10 +496,10 @@ func (i *Ingester) updateActiveSeries(now time.Time) { } newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) - if newMatchersConfig.String() != userDB.activeSeries.asm.cfg.String() { - i.replaceMatchers(NewActiveSeriesMatchers(newMatchersConfig), userDB) + if newMatchersConfig.String() != userDB.activeSeries.CurrentConfig().String() { + i.replaceMatchers(activeseries.NewActiveSeriesMatchers(newMatchersConfig), userDB) } - if userDB.activeSeries.lastAsmUpdate.Load() < now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout).UnixNano() { + if userDB.activeSeries.LastAsmUpdate() < now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout).UnixNano() { // We are not exposing metrics until enough time passed for stable metrics. allActive, activeMatching := userDB.activeSeries.Active(now) if allActive > 0 { @@ -1466,7 +1467,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userDB := &userTSDB{ userID: userID, - activeSeries: NewActiveSeries(NewActiveSeriesMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout), + activeSeries: activeseries.NewActiveSeries(activeseries.NewActiveSeriesMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 856eee0d86..5c2f240562 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -12,6 +12,7 @@ import ( "bytes" "context" "fmt" + "github.com/grafana/mimir/pkg/ingester/activeseries" "io" "io/ioutil" "math" @@ -63,6 +64,12 @@ import ( "github.com/grafana/mimir/pkg/util/validation" ) +func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *activeseries.ActiveSeriesCustomTrackersConfig { + m, err := activeseries.NewActiveSeriesCustomTrackersConfig(source) + require.NoError(t, err) + return &m +} + func TestIngester_Push(t *testing.T) { metricLabelAdapters := []mimirpb.LabelAdapter{{Name: labels.MetricName, Value: "test"}} metricLabels := mimirpb.FromLabelAdaptersToLabels(metricLabelAdapters) @@ -5144,14 +5151,14 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ - func() *ActiveSeriesCustomTrackersOverrides { - return &ActiveSeriesCustomTrackersOverrides{ + defaultCustomTrackersOverridesProvider := &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ + func() *activeseries.ActiveSeriesCustomTrackersOverrides { + return &activeseries.ActiveSeriesCustomTrackersOverrides{ Default: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }), - TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + TenantSpecific: map[string]*activeseries.ActiveSeriesCustomTrackersConfig{ "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, @@ -5170,8 +5177,8 @@ func TestIngesterActiveSeries(t *testing.T) { reqs []*mimirpb.WriteRequest expectedMetrics string disableActiveSeries bool - activeSeriesOverridesProvider *ActiveSeriesCustomTrackersOverridesProvider - activeSeriesConfig ActiveSeriesCustomTrackersConfig + activeSeriesOverridesProvider *activeseries.ActiveSeriesCustomTrackersOverridesProvider + activeSeriesConfig activeseries.ActiveSeriesCustomTrackersConfig }{ "successful push, should count active series": { activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, @@ -5357,8 +5364,8 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with empty runtime config": { - activeSeriesOverridesProvider: &ActiveSeriesCustomTrackersOverridesProvider{ - func() *ActiveSeriesCustomTrackersOverrides { + activeSeriesOverridesProvider: &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ + func() *activeseries.ActiveSeriesCustomTrackersOverrides { return nil }, }, @@ -5394,7 +5401,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should not fail with nil provider and default config": { activeSeriesOverridesProvider: nil, - activeSeriesConfig: ActiveSeriesCustomTrackersConfig{}, + activeSeriesConfig: activeseries.ActiveSeriesCustomTrackersConfig{}, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5504,9 +5511,9 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should revert to flag based default if only tenant-specific overwrite is present": { - activeSeriesOverridesProvider: &ActiveSeriesCustomTrackersOverridesProvider{ - func() *ActiveSeriesCustomTrackersOverrides { - return &ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + activeSeriesOverridesProvider: &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ + func() *activeseries.ActiveSeriesCustomTrackersOverrides { + return &activeseries.ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*activeseries.ActiveSeriesCustomTrackersConfig{ "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, @@ -5603,14 +5610,14 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { } userID := "test_user" - defaultCustomTrackersOverridesProvider := &ActiveSeriesCustomTrackersOverridesProvider{ - func() *ActiveSeriesCustomTrackersOverrides { - return &ActiveSeriesCustomTrackersOverrides{ + defaultCustomTrackersOverridesProvider := &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ + func() *activeseries.ActiveSeriesCustomTrackersOverrides { + return &activeseries.ActiveSeriesCustomTrackersOverrides{ Default: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, }), - TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + TenantSpecific: map[string]*activeseries.ActiveSeriesCustomTrackersConfig{ "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, @@ -5629,8 +5636,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest expectedMetrics string - activeSeriesOverridesProvider *ActiveSeriesCustomTrackersOverridesProvider - activeSeriesConfig ActiveSeriesCustomTrackersConfig + activeSeriesOverridesProvider *activeseries.ActiveSeriesCustomTrackersOverridesProvider + activeSeriesConfig activeseries.ActiveSeriesCustomTrackersConfig }{ "overwrite flag based config with runtime overwrite": { activeSeriesOverridesProvider: nil, @@ -5777,10 +5784,10 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) // Change runtime configs - ingester.cfg.ActiveSeriesCustomTrackersOverrides = &ActiveSeriesCustomTrackersOverridesProvider{ - func() *ActiveSeriesCustomTrackersOverrides { - return &ActiveSeriesCustomTrackersOverrides{ - TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ + ingester.cfg.ActiveSeriesCustomTrackersOverrides = &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ + func() *activeseries.ActiveSeriesCustomTrackersOverrides { + return &activeseries.ActiveSeriesCustomTrackersOverrides{ + TenantSpecific: map[string]*activeseries.ActiveSeriesCustomTrackersConfig{ "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "team_a": `{team="a"}`, "team_b": `{team="b"}`, diff --git a/pkg/ingester/user_tsdb.go b/pkg/ingester/user_tsdb.go index a8ca1feeca..b55ee4e2f7 100644 --- a/pkg/ingester/user_tsdb.go +++ b/pkg/ingester/user_tsdb.go @@ -7,6 +7,7 @@ package ingester import ( "context" + "github.com/grafana/mimir/pkg/ingester/activeseries" "os" "sync" "time" @@ -57,7 +58,7 @@ func (r tsdbCloseCheckResult) shouldClose() bool { type userTSDB struct { db *tsdb.DB userID string - activeSeries *ActiveSeries + activeSeries *activeseries.ActiveSeries seriesInMetric *metricCounter limiter *Limiter diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index 18a0633ad0..ed85faf2fd 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -7,6 +7,7 @@ package mimir import ( "errors" + "github.com/grafana/mimir/pkg/ingester/activeseries" "io" "net/http" @@ -35,7 +36,7 @@ type runtimeConfigValues struct { IngesterLimits *ingester.InstanceLimits `yaml:"ingester_limits"` - ActiveSeriesCustomTrackersOverrides *ingester.ActiveSeriesCustomTrackersOverrides `yaml:"active_series_custom_trackers_overrides"` + ActiveSeriesCustomTrackersOverrides *activeseries.ActiveSeriesCustomTrackersOverrides `yaml:"active_series_custom_trackers_overrides"` } // runtimeConfigTenantLimits provides per-tenant limit overrides based on a runtimeconfig.Manager @@ -147,13 +148,13 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeActiveSeriesCustomTrackersOverrides(manager *runtimeconfig.Manager) *ingester.ActiveSeriesCustomTrackersOverridesProvider { +func runtimeActiveSeriesCustomTrackersOverrides(manager *runtimeconfig.Manager) *activeseries.ActiveSeriesCustomTrackersOverridesProvider { if manager == nil { return nil } - return &ingester.ActiveSeriesCustomTrackersOverridesProvider{ - Getter: func() *ingester.ActiveSeriesCustomTrackersOverrides { + return &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ + Getter: func() *activeseries.ActiveSeriesCustomTrackersOverrides { val := manager.GetConfig() if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { return cfg.ActiveSeriesCustomTrackersOverrides diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index ee142fad5c..1b20c208cc 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -8,6 +8,7 @@ package main import ( "flag" "fmt" + "github.com/grafana/mimir/pkg/ingester/activeseries" "net/url" "reflect" "regexp" @@ -22,7 +23,6 @@ import ( "github.com/prometheus/prometheus/model/relabel" "github.com/weaveworks/common/logging" - "github.com/grafana/mimir/pkg/ingester" "github.com/grafana/mimir/pkg/util/fieldcategory" ) @@ -348,7 +348,7 @@ func getCustomFieldType(t reflect.Type) (string, bool) { return "string", true case reflect.TypeOf([]*relabel.Config{}).String(): return "relabel_config...", true - case reflect.TypeOf(ingester.ActiveSeriesCustomTrackersConfig{}).String(): + case reflect.TypeOf(activeseries.ActiveSeriesCustomTrackersConfig{}).String(): return "map of tracker name (string) to matcher (string)", true default: return "", false From 6c332a9d86f50dca3905b29afb43cf9bff117a9e Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 1 Mar 2022 15:14:21 +0100 Subject: [PATCH 58/91] Moving tenant specific configuration to limits, updating tests --- .../active_series_custom_trackers_config.go | 24 - ...tive_series_custom_trackers_config_test.go | 103 +--- pkg/ingester/ingester.go | 24 +- pkg/ingester/ingester_test.go | 457 +++++++----------- pkg/mimir/modules.go | 2 +- pkg/mimir/runtime_config.go | 18 +- pkg/util/validation/limits.go | 7 + 7 files changed, 219 insertions(+), 416 deletions(-) diff --git a/pkg/ingester/activeseries/active_series_custom_trackers_config.go b/pkg/ingester/activeseries/active_series_custom_trackers_config.go index e77cc8f064..b6a46defc3 100644 --- a/pkg/ingester/activeseries/active_series_custom_trackers_config.go +++ b/pkg/ingester/activeseries/active_series_custom_trackers_config.go @@ -146,27 +146,3 @@ func NewActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCus c.string = activeSeriesCustomTrackersConfigString(c.source) return c, nil } - -// ActiveSeriesCustomTrackersOverrides holds the definition of custom tracking rules. -type ActiveSeriesCustomTrackersOverrides struct { - Default *ActiveSeriesCustomTrackersConfig `yaml:"default"` - TenantSpecific map[string]*ActiveSeriesCustomTrackersConfig `yaml:"tenant_specific"` -} - -func (asmo *ActiveSeriesCustomTrackersOverrides) MatchersConfigForUser(userID string) *ActiveSeriesCustomTrackersConfig { - if tenantspecific, ok := asmo.TenantSpecific[userID]; ok { - return tenantspecific - } - return asmo.Default -} - -type ActiveSeriesCustomTrackersOverridesProvider struct { - Getter func() *ActiveSeriesCustomTrackersOverrides -} - -func (p *ActiveSeriesCustomTrackersOverridesProvider) Get() *ActiveSeriesCustomTrackersOverrides { - if p == nil || p.Getter == nil { - return nil - } - return p.Getter() -} diff --git a/pkg/ingester/activeseries/active_series_custom_trackers_config_test.go b/pkg/ingester/activeseries/active_series_custom_trackers_config_test.go index f9edd50e21..c8e7f9b929 100644 --- a/pkg/ingester/activeseries/active_series_custom_trackers_config_test.go +++ b/pkg/ingester/activeseries/active_series_custom_trackers_config_test.go @@ -75,8 +75,8 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`}), }, { - name: "whitespaces are trimmed from name and matcher", - flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, + name: "whitespaces are trimmed from name and matcher", + flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`}), }, { @@ -125,105 +125,6 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { } } -func TestRuntimeOverridesUnmarshal(t *testing.T) { - expectedDefaultConfig := mustNewActiveSeriesCustomTrackersConfigFromMap( - t, map[string]string{ - "integrations/apolloserver": "{job='integrations/apollo-server'}", - "integrations/caddy": "{job='integrations/caddy'}", - }, - ) - expectedTenantConfig := mustNewActiveSeriesCustomTrackersConfigFromMap( - t, map[string]string{ - "team_A": "{grafanacloud_team='team_a'}", - "team_B": "{grafanacloud_team='team_b'}", - }, - ) - r := ActiveSeriesCustomTrackersOverrides{} - input := ` -default: - integrations/apolloserver: "{job='integrations/apollo-server'}" - integrations/caddy: "{job='integrations/caddy'}" -tenant_specific: - 1: - team_A: "{grafanacloud_team='team_a'}" - team_B: "{grafanacloud_team='team_b'}" -` - - require.NoError(t, yaml.UnmarshalStrict([]byte(input), &r)) - require.Equal(t, expectedDefaultConfig.String(), r.Default.String()) - require.Equal(t, expectedTenantConfig.String(), r.TenantSpecific["1"].String()) - -} - -func TestActiveSeriesCustomTrackersOverridesProvider(t *testing.T) { - overridesReference := &ActiveSeriesCustomTrackersOverrides{} - tests := map[string]struct { - provider *ActiveSeriesCustomTrackersOverridesProvider - expected *ActiveSeriesCustomTrackersOverrides - }{ - "nil provider returns nil": { - provider: nil, - expected: nil, - }, - "nil getter returns nil": { - provider: &ActiveSeriesCustomTrackersOverridesProvider{}, - expected: nil, - }, - "getter is called": { - provider: &ActiveSeriesCustomTrackersOverridesProvider{ - Getter: func() *ActiveSeriesCustomTrackersOverrides { - return overridesReference - }, - }, - expected: overridesReference, - }, - } - - for name, testData := range tests { - t.Run(name, func(t *testing.T) { - assert.Equal(t, testData.expected, testData.provider.Get()) - }) - } -} - -func TestMatchersForUser(t *testing.T) { - defaultMatchers := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "foo": `{foo="bar"}`, - "bar": `{baz="bar"}`, - }) - tenantSpecificMatchers := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "team_a": `{team="team_a"}`, - "team_b": `{team="team_b"}`, - }) - - activeSeriesCustomTrackersOverrides := &ActiveSeriesCustomTrackersOverrides{ - Default: defaultMatchers, - TenantSpecific: map[string]*ActiveSeriesCustomTrackersConfig{ - "1": tenantSpecificMatchers, - }, - } - - tests := map[string]struct { - userID string - expected *ActiveSeriesCustomTrackersConfig - }{ - "User with no override should return default": { - userID: "5", - expected: defaultMatchers, - }, - "User with override should return override": { - userID: "1", - expected: tenantSpecificMatchers, - }, - } - for name, testData := range tests { - t.Run(name, func(t *testing.T) { - matchersConfigForUser := activeSeriesCustomTrackersOverrides.MatchersConfigForUser(testData.userID) - assert.True(t, testData.expected.String() == matchersConfigForUser.String()) - }) - } -} - func TestActiveSeriesCustomTrackerConfig_Equality(t *testing.T) { configSets := [][]ActiveSeriesCustomTrackersConfig{ { diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 226c140f40..f5f65c780f 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -122,11 +122,11 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackersConfig activeseries.ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` - ActiveSeriesCustomTrackersOverrides *activeseries.ActiveSeriesCustomTrackersOverridesProvider `yaml:"-"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesCustomTrackersConfig activeseries.ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` + ActiveSeriesCustomTrackersOverridesFn func() *activeseries.ActiveSeriesCustomTrackersConfig `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -473,14 +473,16 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *activeseries.ActiveSeriesCustomTrackersConfig { - var matchers *activeseries.ActiveSeriesCustomTrackersConfig - if cfg := i.cfg.ActiveSeriesCustomTrackersOverrides.Get(); cfg != nil { - matchers = cfg.MatchersConfigForUser(userID) + matchersConfig := i.limits.ActiveSeriesCustomTrackersConfig(userID) + if matchersConfig == nil { + if i.cfg.ActiveSeriesCustomTrackersOverridesFn != nil { + matchersConfig = i.cfg.ActiveSeriesCustomTrackersOverridesFn() + } } - if matchers == nil { - matchers = i.activeSeriesMatcher.Config() + if matchersConfig == nil { + matchersConfig = i.activeSeriesMatcher.Config() } - return matchers + return matchersConfig } func (i *Ingester) replaceMatchers(asm *activeseries.ActiveSeriesMatchers, userDB *userTSDB) { diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 5c2f240562..67b2a1ff1c 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -2851,6 +2851,14 @@ func prepareIngesterWithBlocksStorage(t testing.TB, ingesterCfg Config, register } func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, limits validation.Limits, dataDir string, registerer prometheus.Registerer) (*Ingester, error) { + overrides, err := validation.NewOverrides(limits, nil) + if err != nil { + return nil, err + } + return prepareIngesterWithBlockStorageAndOverrides(t, ingesterCfg, overrides, dataDir, registerer) +} + +func prepareIngesterWithBlockStorageAndOverrides(t testing.TB, ingesterCfg Config, overrides *validation.Overrides, dataDir string, registerer prometheus.Registerer) (*Ingester, error) { // Create a data dir if none has been provided. if dataDir == "" { var err error @@ -2874,11 +2882,6 @@ func prepareIngesterWithBlocksStorageAndLimits(t testing.TB, ingesterCfg Config, clientCfg := defaultClientTestConfig() - overrides, err := validation.NewOverrides(limits, nil) - if err != nil { - return nil, err - } - ingesterCfg.BlocksStorageConfig.TSDB.Dir = dataDir ingesterCfg.BlocksStorageConfig.Bucket.Backend = "filesystem" ingesterCfg.BlocksStorageConfig.Bucket.Filesystem.Directory = bucketDir @@ -5126,6 +5129,19 @@ func benchmarkData(nSeries int) (allLabels []labels.Labels, allSamples []mimirpb return } +type TenantLimitsMock struct { + mock.Mock + validation.TenantLimits +} + +func (t *TenantLimitsMock) ByUserID(userID string) *validation.Limits { + returnArgs := t.Called(userID) + if returnArgs.Get(0) == nil { + return nil + } + return returnArgs.Get(0).(*validation.Limits) +} + func TestIngesterActiveSeries(t *testing.T) { labelsToPush := []labels.Labels{ labels.FromStrings(labels.MetricName, "test_metric", "bool", "false", "team", "a"), @@ -5151,52 +5167,44 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultCustomTrackersOverridesProvider := &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ - func() *activeseries.ActiveSeriesCustomTrackersOverrides { - return &activeseries.ActiveSeriesCustomTrackersOverrides{ - Default: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, - }), - TenantSpecific: map[string]*activeseries.ActiveSeriesCustomTrackersConfig{ - "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - }), - }, - } - }, + defaultActiveSeriesCustomTrackersFn := func() *activeseries.ActiveSeriesCustomTrackersConfig { + return mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "bool_is_true": `{bool="true"}`, + "bool_is_false": `{bool="false"}`, + }) } activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, }) + + activeSeriesTenantConfig := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }) + + activeSeriesTenantOverride := new(TenantLimitsMock) + activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: activeSeriesTenantConfig}) + activeSeriesTenantOverride.On("ByUserID", userID2).Return(nil) + tests := map[string]struct { - test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) - reqs []*mimirpb.WriteRequest - expectedMetrics string - disableActiveSeries bool - activeSeriesOverridesProvider *activeseries.ActiveSeriesCustomTrackersOverridesProvider - activeSeriesConfig activeseries.ActiveSeriesCustomTrackersConfig + test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) + reqs []*mimirpb.WriteRequest + expectedMetrics string + disableActiveSeries bool + defaultActiveSeriesCustomTrackers func() *activeseries.ActiveSeriesCustomTrackersConfig + activeSeriesConfig activeseries.ActiveSeriesCustomTrackersConfig + tenantLimits *TenantLimitsMock }{ "successful push, should count active series": { - activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, + defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, + tenantLimits: activeSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(now) @@ -5219,22 +5227,13 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should cleanup metrics when tsdb closed": { - activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, + defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, + tenantLimits: activeSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(now) @@ -5261,22 +5260,13 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should track custom matchers, removing when zero": { - activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, + defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, + tenantLimits: activeSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(firstPushTime) @@ -5305,12 +5295,7 @@ func TestIngesterActiveSeries(t *testing.T) { // Sleep another millisecond to make sure that secondPushTime is strictly less than the append time of the second push. time.Sleep(time.Millisecond) - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) // Update active series for metrics check in the future. // We update them in the exact moment in time where append time of the first push is already considered idle, @@ -5336,23 +5321,13 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "successful push, active series disabled": { - activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, - disableActiveSeries: true, + defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, + disableActiveSeries: true, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(now) @@ -5364,26 +5339,12 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with empty runtime config": { - activeSeriesOverridesProvider: &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ - func() *activeseries.ActiveSeriesCustomTrackersOverrides { - return nil - }, - }, + defaultActiveSeriesCustomTrackers: func() *activeseries.ActiveSeriesCustomTrackersConfig { return nil }, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(now) @@ -5400,23 +5361,13 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with nil provider and default config": { - activeSeriesOverridesProvider: nil, - activeSeriesConfig: activeseries.ActiveSeriesCustomTrackersConfig{}, + defaultActiveSeriesCustomTrackers: nil, + activeSeriesConfig: activeseries.ActiveSeriesCustomTrackersConfig{}, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(now) @@ -5433,23 +5384,13 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should use flag based custom tracker if no runtime config specified": { - activeSeriesOverridesProvider: nil, - activeSeriesConfig: *activeSeriesDefaultConfig, + defaultActiveSeriesCustomTrackers: nil, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(now) @@ -5472,23 +5413,14 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should use runtime matcher config if both specified": { - activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, - activeSeriesConfig: *activeSeriesDefaultConfig, + tenantLimits: activeSeriesTenantOverride, + defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(now) @@ -5506,54 +5438,6 @@ func TestIngesterActiveSeries(t *testing.T) { cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 ` - // Check tracked Prometheus metrics - require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - }, - }, - "should revert to flag based default if only tenant-specific overwrite is present": { - activeSeriesOverridesProvider: &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ - func() *activeseries.ActiveSeriesCustomTrackersOverrides { - return &activeseries.ActiveSeriesCustomTrackersOverrides{TenantSpecific: map[string]*activeseries.ActiveSeriesCustomTrackersConfig{ - "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - }), - }} - }, - }, - activeSeriesConfig: *activeSeriesDefaultConfig, - test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - now := time.Now() - - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - for i, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID2) - offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now().Add(offset))) - require.NoError(t, err) - } - - // Update active series for metrics check. - ingester.updateActiveSeries(now) - - expectedMetrics := ` - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 - # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. - # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 - ` - // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, @@ -5568,10 +5452,21 @@ func TestIngesterActiveSeries(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.IngesterRing.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - cfg.ActiveSeriesCustomTrackersOverrides = testData.activeSeriesOverridesProvider + cfg.ActiveSeriesCustomTrackersOverridesFn = testData.defaultActiveSeriesCustomTrackers cfg.ActiveSeriesCustomTrackersConfig = testData.activeSeriesConfig + limits := defaultLimitsTestConfig() + var overrides *validation.Overrides + var err error + // Without this, TenantLimitsMock(nil) != nil when using getOverridesForUser in limits.go + if testData.tenantLimits != nil { + overrides, err = validation.NewOverrides(limits, testData.tenantLimits) + require.NoError(t, err) + } else { + overrides, err = validation.NewOverrides(limits, nil) + require.NoError(t, err) + } - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) + ing, err := prepareIngesterWithBlockStorageAndOverrides(t, cfg, overrides, "", registry) require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck @@ -5609,22 +5504,13 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { "cortex_ingester_active_series_custom_tracker", } userID := "test_user" + userID2 := "other_test_user" - defaultCustomTrackersOverridesProvider := &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ - func() *activeseries.ActiveSeriesCustomTrackersOverrides { - return &activeseries.ActiveSeriesCustomTrackersOverrides{ - Default: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, - }), - TenantSpecific: map[string]*activeseries.ActiveSeriesCustomTrackersConfig{ - "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - }), - }, - } - }, + defaultActiveSeriesCustomTrackersFn := func() *activeseries.ActiveSeriesCustomTrackersConfig { + return mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "bool_is_true": `{bool="true"}`, + "bool_is_false": `{bool="false"}`, + }) } activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ @@ -5632,24 +5518,32 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { "bool_is_false_flagbased": `{bool="false"}`, }) + activeSeriesTenantConfig := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + }) + + defaultActiveSeriesTenantOverride := new(TenantLimitsMock) + defaultActiveSeriesTenantOverride.On("ByUserID", userID2).Return(nil) + defaultActiveSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: activeSeriesTenantConfig}) + tests := map[string]struct { - test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) - reqs []*mimirpb.WriteRequest - expectedMetrics string - activeSeriesOverridesProvider *activeseries.ActiveSeriesCustomTrackersOverridesProvider - activeSeriesConfig activeseries.ActiveSeriesCustomTrackersConfig + test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) + reqs []*mimirpb.WriteRequest + expectedMetrics string + defaultActiveSeriesCustomTrackers func() *activeseries.ActiveSeriesCustomTrackersConfig + activeSeriesConfig activeseries.ActiveSeriesCustomTrackersConfig + tenantLimits *TenantLimitsMock }{ "overwrite flag based config with runtime overwrite": { - activeSeriesOverridesProvider: nil, - activeSeriesConfig: *activeSeriesDefaultConfig, + defaultActiveSeriesCustomTrackers: nil, + tenantLimits: nil, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() - for _, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req(label, time.Now())) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(firstPushTime) @@ -5657,9 +5551,12 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="test_user"} 2 cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="test_user"} 2 ` @@ -5667,11 +5564,18 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) // Add new runtime configs - ingester.cfg.ActiveSeriesCustomTrackersOverrides = defaultCustomTrackersOverridesProvider + ingester.cfg.ActiveSeriesCustomTrackersOverridesFn = defaultActiveSeriesCustomTrackersFn + activeSeriesTenantOverride := new(TenantLimitsMock) + activeSeriesTenantOverride.On("ByUserID", userID2).Return(nil) + activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: activeSeriesTenantConfig}) + override, err := validation.NewOverrides(defaultLimitsTestConfig(), activeSeriesTenantOverride) + require.NoError(t, err) + ingester.limits = override ingester.updateActiveSeries(firstPushTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 cortex_ingester_active_series{user="test_user"} 4 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5679,18 +5583,18 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { secondPushtime := time.Now() // Sleep here to ensure that the second batch of push happens after purgeTime to have entries for custom trackers time.Sleep(time.Millisecond) - for _, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req(label, time.Now())) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 ` @@ -5698,17 +5602,14 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { }, }, "remove runtime overwrite and revert to flag based config": { - activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, - activeSeriesConfig: *activeSeriesDefaultConfig, + defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, + activeSeriesConfig: *activeSeriesDefaultConfig, + tenantLimits: defaultActiveSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() - for _, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - //offset := time.Duration(len(labelsToPush) - i) - _, err := ingester.Push(ctx, req(label, time.Now())) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. ingester.updateActiveSeries(firstPushTime) @@ -5716,21 +5617,28 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 ` // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) // Remove runtime configs - ingester.cfg.ActiveSeriesCustomTrackersOverrides = nil + ingester.cfg.ActiveSeriesCustomTrackersOverridesFn = nil + var err error + ingester.limits, err = validation.NewOverrides(defaultLimitsTestConfig(), nil) + require.NoError(t, err) ingester.updateActiveSeries(firstPushTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 cortex_ingester_active_series{user="test_user"} 4 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5738,35 +5646,31 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { secondPushtime := time.Now() // Sleep here to ensure that the second batch of push happens after purgeTime to have entries for custom trackers time.Sleep(time.Millisecond) - for _, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req(label, time.Now())) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="test_user"} 2 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, }, "changing runtime overwrite should result in new metrics": { - activeSeriesOverridesProvider: defaultCustomTrackersOverridesProvider, - activeSeriesConfig: *activeSeriesDefaultConfig, + defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, + activeSeriesConfig: *activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() - for _, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req(label, firstPushTime)) - require.NoError(t, err) - } + pushWithUser(t, ingester, labelsToPush, userID, req) // Update active series for metrics check. ingester.updateActiveSeries(firstPushTime) @@ -5777,27 +5681,23 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="test_user"} 2 ` // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) // Change runtime configs - ingester.cfg.ActiveSeriesCustomTrackersOverrides = &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ - func() *activeseries.ActiveSeriesCustomTrackersOverrides { - return &activeseries.ActiveSeriesCustomTrackersOverrides{ - TenantSpecific: map[string]*activeseries.ActiveSeriesCustomTrackersConfig{ - "test_user": mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "team_a": `{team="a"}`, - "team_b": `{team="b"}`, - "team_c": `{team="b"}`, - "team_d": `{team="b"}`, - }), - }, - } - }, - } + activeSeriesTenantOverride := new(TenantLimitsMock) + activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + "team_a": `{team="a"}`, + "team_b": `{team="b"}`, + "team_c": `{team="b"}`, + "team_d": `{team="b"}`, + })}) + override, err := validation.NewOverrides(defaultLimitsTestConfig(), activeSeriesTenantOverride) + require.NoError(t, err) + ingester.limits = override // This will update the runtime config. configReloadTime := time.Now() ingester.updateActiveSeries(configReloadTime) @@ -5840,11 +5740,22 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.IngesterRing.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = true - cfg.ActiveSeriesCustomTrackersOverrides = testData.activeSeriesOverridesProvider + cfg.ActiveSeriesCustomTrackersOverridesFn = testData.defaultActiveSeriesCustomTrackers cfg.ActiveSeriesCustomTrackersConfig = testData.activeSeriesConfig - ing, err := prepareIngesterWithBlocksStorageAndLimits(t, cfg, defaultLimitsTestConfig(), "", registry) - require.NoError(t, err) + limits := defaultLimitsTestConfig() + var overrides *validation.Overrides + var err error + // Without this, TenantLimitsMock(nil) != nil when using getOverridesForUser in limits.go + if testData.tenantLimits != nil { + overrides, err = validation.NewOverrides(limits, testData.tenantLimits) + require.NoError(t, err) + } else { + overrides, err = validation.NewOverrides(limits, nil) + require.NoError(t, err) + } + + ing, err := prepareIngesterWithBlockStorageAndOverrides(t, cfg, overrides, "", registry) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck @@ -5858,6 +5769,14 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { } } +func pushWithUser(t *testing.T, ingester *Ingester, labelsToPush []labels.Labels, userID string, req func(lbls labels.Labels, t time.Time) *mimirpb.WriteRequest) { + for _, label := range labelsToPush { + ctx := user.InjectOrgID(context.Background(), userID) + _, err := ingester.Push(ctx, req(label, time.Now())) + require.NoError(t, err) + } +} + func TestGetIgnoreSeriesLimitForMetricNamesMap(t *testing.T) { cfg := Config{} diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 6004fc9ae4..481180470a 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -436,7 +436,7 @@ func (t *Mimir) initIngesterService() (serv services.Service, err error) { t.Cfg.Ingester.IngesterRing.ListenPort = t.Cfg.Server.GRPCListenPort t.Cfg.Ingester.StreamTypeFn = ingesterChunkStreaming(t.RuntimeConfig) t.Cfg.Ingester.InstanceLimitsFn = ingesterInstanceLimits(t.RuntimeConfig) - t.Cfg.Ingester.ActiveSeriesCustomTrackersOverrides = runtimeActiveSeriesCustomTrackersOverrides(t.RuntimeConfig) + t.Cfg.Ingester.ActiveSeriesCustomTrackersOverridesFn = runtimeActiveSeriesCustomTrackersDefaultOverridesFn(t.RuntimeConfig) t.tsdbIngesterConfig() t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger) diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index ed85faf2fd..29f3836a69 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -36,7 +36,7 @@ type runtimeConfigValues struct { IngesterLimits *ingester.InstanceLimits `yaml:"ingester_limits"` - ActiveSeriesCustomTrackersOverrides *activeseries.ActiveSeriesCustomTrackersOverrides `yaml:"active_series_custom_trackers_overrides"` + ActiveSeriesCustomTrackersDefaultOverrides *activeseries.ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers_overrides"` } // runtimeConfigTenantLimits provides per-tenant limit overrides based on a runtimeconfig.Manager @@ -148,19 +148,17 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeActiveSeriesCustomTrackersOverrides(manager *runtimeconfig.Manager) *activeseries.ActiveSeriesCustomTrackersOverridesProvider { +func runtimeActiveSeriesCustomTrackersDefaultOverridesFn(manager *runtimeconfig.Manager) func() *activeseries.ActiveSeriesCustomTrackersConfig { if manager == nil { return nil } - return &activeseries.ActiveSeriesCustomTrackersOverridesProvider{ - Getter: func() *activeseries.ActiveSeriesCustomTrackersOverrides { - val := manager.GetConfig() - if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { - return cfg.ActiveSeriesCustomTrackersOverrides - } - return nil - }, + return func() *activeseries.ActiveSeriesCustomTrackersConfig { + val := manager.GetConfig() + if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { + return cfg.ActiveSeriesCustomTrackersDefaultOverrides + } + return nil } } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 7982e3138e..a882323134 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -9,6 +9,7 @@ import ( "bytes" "encoding/json" "flag" + "github.com/grafana/mimir/pkg/ingester/activeseries" "math" "strings" "time" @@ -55,6 +56,8 @@ type Limits struct { MaxGlobalMetadataPerMetric int `yaml:"max_global_metadata_per_metric" json:"max_global_metadata_per_metric"` // Exemplars MaxGlobalExemplarsPerUser int `yaml:"max_global_exemplars_per_user" json:"max_global_exemplars_per_user" category:"experimental"` + // Active series custom trackers + ActiveSeriesCustomTrackersConfig *activeseries.ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers_config"` // Querier enforced limits. MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"` @@ -421,6 +424,10 @@ func (o *Overrides) MaxGlobalExemplarsPerUser(userID string) int { return o.getOverridesForUser(userID).MaxGlobalExemplarsPerUser } +func (o *Overrides) ActiveSeriesCustomTrackersConfig(userID string) *activeseries.ActiveSeriesCustomTrackersConfig { + return o.getOverridesForUser(userID).ActiveSeriesCustomTrackersConfig +} + // IngestionTenantShardSize returns the ingesters shard size for a given user. func (o *Overrides) IngestionTenantShardSize(userID string) int { return o.getOverridesForUser(userID).IngestionTenantShardSize From 982007454d4baadb8887cfc47bb281b26fd31b34 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 2 Mar 2022 08:54:20 +0100 Subject: [PATCH 59/91] Lint fixes, file renaming --- pkg/ingester/activeseries/active_series.go | 18 +++--- .../activeseries/active_series_test.go | 36 +++++------ ...rs_config.go => custom_trackers_config.go} | 16 ++--- ...test.go => custom_trackers_config_test.go} | 62 +++++++++---------- ...e_series_custom_tracker.go => matchers.go} | 38 ++++++------ ...ustom_tracker_test.go => matchers_test.go} | 2 +- pkg/ingester/ingester.go | 24 +++---- pkg/ingester/ingester_test.go | 21 ++++--- pkg/ingester/user_tsdb.go | 2 +- pkg/mimir/runtime_config.go | 8 +-- pkg/util/validation/limits.go | 7 ++- tools/doc-generator/parser.go | 4 +- 12 files changed, 120 insertions(+), 118 deletions(-) rename pkg/ingester/activeseries/{active_series_custom_trackers_config.go => custom_trackers_config.go} (85%) rename pkg/ingester/activeseries/{active_series_custom_trackers_config_test.go => custom_trackers_config_test.go} (71%) rename pkg/ingester/activeseries/{active_series_custom_tracker.go => matchers.go} (61%) rename pkg/ingester/activeseries/{active_series_custom_tracker_test.go => matchers_test.go} (97%) diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index 7a9a7ee7b8..e89fe00587 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -23,7 +23,7 @@ const ( // ActiveSeries is keeping track of recently active series for a single tenant. type ActiveSeries struct { stripes [numActiveSeriesStripes]activeSeriesStripe - asm *ActiveSeriesMatchers + asm *Matchers lastAsmUpdate *atomic.Int64 // The duration after series become inactive. timeout time.Duration @@ -32,7 +32,7 @@ type ActiveSeries struct { // activeSeriesStripe holds a subset of the series timestamps for a single tenant. type activeSeriesStripe struct { - asm *ActiveSeriesMatchers + asm *Matchers // Unix nanoseconds. Only used by purge. Zero = unknown. // Updated in purge and when old timestamp is used when updating series (in this case, oldestEntryTs is updated @@ -42,17 +42,17 @@ type activeSeriesStripe struct { mu sync.RWMutex refs map[uint64][]activeSeriesEntry active int // Number of active entries in this stripe. Only decreased during purge or clear. - activeMatching []int // Number of active entries in this stripe matching each matcher of the configured ActiveSeriesMatchers. + activeMatching []int // Number of active entries in this stripe matching each matcher of the configured Matchers. } // activeSeriesEntry holds a timestamp for single series. type activeSeriesEntry struct { lbs labels.Labels nanos *atomic.Int64 // Unix timestamp in nanoseconds. Needs to be a pointer because we don't store pointers to entries in the stripe. - matches []bool // Which matchers of ActiveSeriesMatchers does this series match + matches []bool // Which matchers of Matchers does this series match } -func NewActiveSeries(asm *ActiveSeriesMatchers, idleTimeout time.Duration) *ActiveSeries { +func NewActiveSeries(asm *Matchers, idleTimeout time.Duration) *ActiveSeries { c := &ActiveSeries{asm: asm, timeout: idleTimeout, lastAsmUpdate: atomic.NewInt64(0)} // Stripes are pre-allocated so that we only read on them and no lock is required. @@ -73,7 +73,7 @@ func (c *ActiveSeries) CurrentMatcherNames() []string { return c.asm.MatcherNames() } -func (c *ActiveSeries) ReloadSeriesMatchers(asm *ActiveSeriesMatchers) { +func (c *ActiveSeries) ReloadSeriesMatchers(asm *Matchers) { c.mu.Lock() defer c.mu.Unlock() @@ -88,7 +88,7 @@ func (c *ActiveSeries) LastAsmUpdate() int64 { return c.lastAsmUpdate.Load() } -func (c *ActiveSeries) CurrentConfig() *ActiveSeriesCustomTrackersConfig { +func (c *ActiveSeries) CurrentConfig() *CustomTrackersConfig { c.mu.RLock() defer c.mu.RUnlock() return c.asm.Config() @@ -245,8 +245,8 @@ func (s *activeSeriesStripe) clear() { } } -// Reinitalize is more than clear that it assigns new matchers and corresponding size activeMatching slices. -func (s *activeSeriesStripe) reinitialize(asm *ActiveSeriesMatchers) { +// Reinitialize is more than clear that it assigns new matchers and corresponding size activeMatching slices. +func (s *activeSeriesStripe) reinitialize(asm *Matchers) { s.mu.Lock() defer s.mu.Unlock() diff --git a/pkg/ingester/activeseries/active_series_test.go b/pkg/ingester/activeseries/active_series_test.go index 552192adf9..4fd098f075 100644 --- a/pkg/ingester/activeseries/active_series_test.go +++ b/pkg/ingester/activeseries/active_series_test.go @@ -28,7 +28,7 @@ func TestActiveSeries_UpdateSeries_NoMatchers(t *testing.T) { ls2 := []labels.Label{{Name: "a", Value: "2"}} currentNow := time.Now() - c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) + c := NewActiveSeries(&Matchers{}, 5*time.Minute) allActive, activeMatching := c.Active(currentNow) assert.Equal(t, 0, allActive) assert.Nil(t, activeMatching) @@ -51,7 +51,7 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { ls2 := []labels.Label{{Name: "a", Value: "2"}} ls3 := []labels.Label{{Name: "a", Value: "3"}} - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~"2|3"}`})) + asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~"2|3"}`})) currentNow := time.Now() c := NewActiveSeries(asm, 5*time.Minute) @@ -87,7 +87,7 @@ func TestActiveSeries_ShouldCorrectlyHandleFingerprintCollisions(t *testing.T) { require.True(t, client.Fingerprint(ls1) == client.Fingerprint(ls2)) currentNow := time.Now() - c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) + c := NewActiveSeries(&Matchers{}, 5*time.Minute) c.UpdateSeries(ls1, time.Now(), copyFn) c.UpdateSeries(ls2, time.Now(), copyFn) @@ -107,7 +107,7 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl: %d", ttl), func(t *testing.T) { - c := NewActiveSeries(&ActiveSeriesMatchers{}, time.Since(time.Unix(0, 0))) + c := NewActiveSeries(&Matchers{}, time.Since(time.Unix(0, 0))) for i := 0; i < len(series); i++ { c.UpdateSeries(series[i], time.Unix(int64(i), 0), copyFn) @@ -118,7 +118,7 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { c.purge(time.Unix(int64(ttl), 0)) exp := len(series) - (ttl) - // c.Active is not intented to purge + // c.Active is not intended to purge allActive, activeMatching := c.Active(time.Unix(0, 0)) assert.Equal(t, exp, allActive) assert.Nil(t, activeMatching) @@ -135,7 +135,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { {{Name: "_", Value: "KiqbryhzUpn"}, {Name: "__name__", Value: "logs"}}, } - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{_=~"y.*"}`})) + asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{_=~"y.*"}`})) // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { @@ -158,7 +158,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { // call purge twice, just to hit "quick" path. It doesn't really do anything. c.purge(time.Unix(int64(ttl), 0)) - // c.Active is not intented to purge + // c.Active is not intended to purge allActive, activeMatching := c.Active(time.Unix(0, 0)) assert.Equal(t, exp, allActive) assert.Equal(t, []int{expMatchingSeries}, activeMatching) @@ -172,7 +172,7 @@ func TestActiveSeries_PurgeOpt(t *testing.T) { ls2 := metric.Set("_", "KiqbryhzUpn").Labels() currentNow := time.Now() - c := NewActiveSeries(&ActiveSeriesMatchers{}, 59*time.Second) + c := NewActiveSeries(&Matchers{}, 59*time.Second) c.UpdateSeries(ls1, currentNow.Add(-2*time.Minute), copyFn) c.UpdateSeries(ls2, currentNow, copyFn) @@ -199,7 +199,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { ls3 := []labels.Label{{Name: "a", Value: "3"}} ls4 := []labels.Label{{Name: "a", Value: "4"}} - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~.*}`})) + asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~.*}`})) currentNow := time.Now() c := NewActiveSeries(asm, 5*time.Minute) @@ -226,7 +226,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { assert.Equal(t, 2, allActive) assert.Equal(t, []int{2}, activeMatching) - asmWithLessMatchers := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{})) + asmWithLessMatchers := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{})) c.ReloadSeriesMatchers(asmWithLessMatchers) c.UpdateSeries(ls3, currentNow, copyFn) @@ -234,7 +234,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { assert.Equal(t, 1, allActive) assert.Equal(t, []int(nil), activeMatching) - asmWithMoreMatchers := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + asmWithMoreMatchers := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "a": `{a="3"}`, "b": `{a="4"}`, })) @@ -249,7 +249,7 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "foo": `{a=~.+}`, "bar": `{a=~.+}`, })) @@ -264,7 +264,7 @@ func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { assert.Equal(t, 1, allActive) assert.Equal(t, []int{1, 1}, activeMatching) - asm = NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + asm = NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "foo": `{a=~.+}`, })) @@ -279,7 +279,7 @@ func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "foo": `{a=~.+}`, "bar": `{a=~.+}`, })) @@ -294,7 +294,7 @@ func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { assert.Equal(t, 1, allActive) assert.Equal(t, []int{1, 1}, activeMatching) - asm = NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + asm = NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "foo": `{b=~.+}`, "bar": `{b=~.+}`, })) @@ -322,7 +322,7 @@ func benchmarkActiveSeriesConcurrencySingleSeries(b *testing.B, goroutines int) {Name: "a", Value: "a"}, } - c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) + c := NewActiveSeries(&Matchers{}, 5*time.Minute) wg := &sync.WaitGroup{} start := make(chan struct{}) @@ -349,7 +349,7 @@ func benchmarkActiveSeriesConcurrencySingleSeries(b *testing.B, goroutines int) } func BenchmarkActiveSeries_UpdateSeries(b *testing.B) { - c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) + c := NewActiveSeries(&Matchers{}, 5*time.Minute) // Prepare series nameBuf := bytes.Buffer{} @@ -384,7 +384,7 @@ func benchmarkPurge(b *testing.B, twice bool) { const numExpiresSeries = numSeries / 25 currentNow := time.Now() - c := NewActiveSeries(&ActiveSeriesMatchers{}, 5*time.Minute) + c := NewActiveSeries(&Matchers{}, 5*time.Minute) series := [numSeries]labels.Labels{} for s := 0; s < numSeries; s++ { diff --git a/pkg/ingester/activeseries/active_series_custom_trackers_config.go b/pkg/ingester/activeseries/custom_trackers_config.go similarity index 85% rename from pkg/ingester/activeseries/active_series_custom_trackers_config.go rename to pkg/ingester/activeseries/custom_trackers_config.go index b6a46defc3..e654ecf62d 100644 --- a/pkg/ingester/activeseries/active_series_custom_trackers_config.go +++ b/pkg/ingester/activeseries/custom_trackers_config.go @@ -10,16 +10,16 @@ import ( amlabels "github.com/prometheus/alertmanager/pkg/labels" ) -// ActiveSeriesCustomTrackersConfig configures active series custom trackers. +// CustomTrackersConfig configures active series custom trackers. // It can be set using a flag, or parsed from yaml. -type ActiveSeriesCustomTrackersConfig struct { +type CustomTrackersConfig struct { source map[string]string config map[string]labelsMatchers string string } // ExampleDoc provides an example doc for this config, especially valuable since it's custom-unmarshaled. -func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { +func (c *CustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, map[string]string{ @@ -30,7 +30,7 @@ func (c *ActiveSeriesCustomTrackersConfig) ExampleDoc() (comment string, yaml in // String is a canonical representation of the config, it is compatible with flag definition. // String is also needed to implement flag.Value. -func (c *ActiveSeriesCustomTrackersConfig) String() string { +func (c *CustomTrackersConfig) String() string { return c.string } @@ -61,7 +61,7 @@ func activeSeriesCustomTrackersConfigString(cfg map[string]string) string { } // Set implements flag.Value, and is used to set the config value from a flag value provided as string. -func (c *ActiveSeriesCustomTrackersConfig) Set(s string) error { +func (c *CustomTrackersConfig) Set(s string) error { if strings.TrimSpace(s) == "" { return nil } @@ -118,8 +118,8 @@ func activeSeriesCustomTrackerFlagValueToMap(s string) (map[string]string, error } // UnmarshalYAML implements the yaml.Unmarshaler interface. -// ActiveSeriesCustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. -func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { +// CustomTrackersConfig are marshaled in yaml as a map[string]string, with matcher names as keys and strings as matchers definitions. +func (c *CustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { stringMap := map[string]string{} err := unmarshal(&stringMap) if err != nil { @@ -129,7 +129,7 @@ func (c *ActiveSeriesCustomTrackersConfig) UnmarshalYAML(unmarshal func(interfac return err } -func NewActiveSeriesCustomTrackersConfig(m map[string]string) (c ActiveSeriesCustomTrackersConfig, err error) { +func NewActiveSeriesCustomTrackersConfig(m map[string]string) (c CustomTrackersConfig, err error) { c.source = m c.config = map[string]labelsMatchers{} for name, matcher := range m { diff --git a/pkg/ingester/activeseries/active_series_custom_trackers_config_test.go b/pkg/ingester/activeseries/custom_trackers_config_test.go similarity index 71% rename from pkg/ingester/activeseries/active_series_custom_trackers_config_test.go rename to pkg/ingester/activeseries/custom_trackers_config_test.go index c8e7f9b929..008083d6ad 100644 --- a/pkg/ingester/activeseries/active_series_custom_trackers_config_test.go +++ b/pkg/ingester/activeseries/custom_trackers_config_test.go @@ -12,37 +12,37 @@ import ( "gopkg.in/yaml.v2" ) -func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *ActiveSeriesCustomTrackersConfig { +func mustNewCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *CustomTrackersConfig { m, err := NewActiveSeriesCustomTrackersConfig(source) require.NoError(t, err) return &m } -func mustNewActiveSeriesCustomTrackersConfigFromString(t *testing.T, source string) *ActiveSeriesCustomTrackersConfig { - m := ActiveSeriesCustomTrackersConfig{} +func mustNewCustomTrackersConfigFromString(t *testing.T, source string) *CustomTrackersConfig { + m := CustomTrackersConfig{} err := m.Set(source) require.NoError(t, err) return &m } -func mustNewActiveSeriesCustomTrackersConfigDeserializedFromYaml(t *testing.T, yamlString string) *ActiveSeriesCustomTrackersConfig { - m := ActiveSeriesCustomTrackersConfig{} +func mustNewCustomTrackersConfigDeserializedFromYaml(t *testing.T, yamlString string) *CustomTrackersConfig { + m := CustomTrackersConfig{} err := yaml.Unmarshal([]byte(yamlString), &m) require.NoError(t, err) return &m } -func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { +func TestCustomTrackersConfigs(t *testing.T) { for _, tc := range []struct { name string flags []string - expected *ActiveSeriesCustomTrackersConfig + expected *CustomTrackersConfig error error }{ { name: "empty flag value produces empty config", flags: []string{`-ingester.active-series-custom-trackers=`}, - expected: &ActiveSeriesCustomTrackersConfig{}, + expected: &CustomTrackersConfig{}, }, { name: "empty matcher fails", @@ -72,22 +72,22 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { { name: "one matcher", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`}), + expected: mustNewCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`}), }, { - name: "whitespaces are trimmed from name and matcher", - flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, - expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`}), + name: "whitespaces are trimmed from name and matcher", + flags: []string{`-ingester.active-series-custom-trackers= foo : {foo="bar"}` + "\n "}, + expected: mustNewCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`}), }, { name: "two matchers in one flag value", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"};baz:{baz="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), + expected: mustNewCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), }, { name: "two matchers in two flag values", flags: []string{`-ingester.active-series-custom-trackers=foo:{foo="bar"}`, `-ingester.active-series-custom-trackers=baz:{baz="bar"}`}, - expected: mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), + expected: mustNewCustomTrackersConfigFromMap(t, map[string]string{`foo`: `{foo="bar"}`, `baz`: `{baz="bar"}`}), }, { name: "two matchers with same name in same flag", @@ -103,7 +103,7 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { t.Run(tc.name, func(t *testing.T) { flagSet := flag.NewFlagSet("test", flag.ContinueOnError) - var config ActiveSeriesCustomTrackersConfig + var config CustomTrackersConfig flagSet.Var(&config, "ingester.active-series-custom-trackers", "...usage docs...") err := flagSet.Parse(tc.flags) @@ -114,9 +114,9 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { require.Equal(t, tc.expected, &config) - // Check that ActiveSeriesCustomTrackersConfig.String() value is a valid flag value. + // Check that CustomTrackersConfig.String() value is a valid flag value. flagSetAgain := flag.NewFlagSet("test-string", flag.ContinueOnError) - var configAgain ActiveSeriesCustomTrackersConfig + var configAgain CustomTrackersConfig flagSetAgain.Var(&configAgain, "ingester.active-series-custom-trackers", "...usage docs...") require.NoError(t, flagSetAgain.Parse([]string{"-ingester.active-series-custom-trackers=" + config.String()})) @@ -125,26 +125,26 @@ func TestActiveSeriesCustomTrackersConfigs(t *testing.T) { } } -func TestActiveSeriesCustomTrackerConfig_Equality(t *testing.T) { - configSets := [][]ActiveSeriesCustomTrackersConfig{ +func TestCustomTrackerConfig_Equality(t *testing.T) { + configSets := [][]CustomTrackersConfig{ { - *mustNewActiveSeriesCustomTrackersConfigFromString(t, `foo:{foo='bar'};baz:{baz='bar'}`), - *mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + *mustNewCustomTrackersConfigFromString(t, `foo:{foo='bar'};baz:{baz='bar'}`), + *mustNewCustomTrackersConfigFromMap(t, map[string]string{ "baz": `{baz='bar'}`, "foo": `{foo='bar'}`, }), - *mustNewActiveSeriesCustomTrackersConfigDeserializedFromYaml(t, + *mustNewCustomTrackersConfigDeserializedFromYaml(t, ` baz: "{baz='bar'}" foo: "{foo='bar'}"`), }, { - *mustNewActiveSeriesCustomTrackersConfigFromString(t, `test:{test='true'}`), - *mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{"test": `{test='true'}`}), - *mustNewActiveSeriesCustomTrackersConfigDeserializedFromYaml(t, `test: "{test='true'}"`), + *mustNewCustomTrackersConfigFromString(t, `test:{test='true'}`), + *mustNewCustomTrackersConfigFromMap(t, map[string]string{"test": `{test='true'}`}), + *mustNewCustomTrackersConfigDeserializedFromYaml(t, `test: "{test='true'}"`), }, { - *mustNewActiveSeriesCustomTrackersConfigDeserializedFromYaml(t, + *mustNewCustomTrackersConfigDeserializedFromYaml(t, ` baz: "{baz='bar'}" foo: "{foo='bar'}" @@ -163,7 +163,7 @@ func TestActiveSeriesCustomTrackerConfig_Equality(t *testing.T) { } t.Run("NotEqualsAcrossSets", func(t *testing.T) { - var activeSeriesMatchers []*ActiveSeriesCustomTrackersConfig + var activeSeriesMatchers []*CustomTrackersConfig for _, matcherConfigs := range configSets { activeSeriesMatchers = append(activeSeriesMatchers, &matcherConfigs[0]) } @@ -177,7 +177,7 @@ func TestActiveSeriesCustomTrackerConfig_Equality(t *testing.T) { } -func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { +func TestTrackersConfigs_Deserialization(t *testing.T) { correctInput := ` baz: "{baz='bar'}" foo: "{foo='bar'}" @@ -188,9 +188,9 @@ func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { foo: "{foo='bar'}" ` t.Run("ShouldDeserializeCorrectInput", func(t *testing.T) { - config := ActiveSeriesCustomTrackersConfig{} + config := CustomTrackersConfig{} err := yaml.Unmarshal([]byte(correctInput), &config) - assert.NoError(t, err, "failed do deserialize ActiveSeriesMatchers") + assert.NoError(t, err, "failed do deserialize Matchers") expectedConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ "baz": "{baz='bar'}", "foo": "{foo='bar'}", @@ -200,7 +200,7 @@ func TestActiveSeriesCustomTrackersConfigs_Deserialization(t *testing.T) { }) t.Run("ShouldErrorOnMalformedInput", func(t *testing.T) { - config := ActiveSeriesCustomTrackersConfig{} + config := CustomTrackersConfig{} err := yaml.Unmarshal([]byte(malformedInput), &config) assert.Error(t, err, "should not deserialize malformed input") }) diff --git a/pkg/ingester/activeseries/active_series_custom_tracker.go b/pkg/ingester/activeseries/matchers.go similarity index 61% rename from pkg/ingester/activeseries/active_series_custom_tracker.go rename to pkg/ingester/activeseries/matchers.go index fa96ed239e..2f45d98ca8 100644 --- a/pkg/ingester/activeseries/active_series_custom_tracker.go +++ b/pkg/ingester/activeseries/matchers.go @@ -9,8 +9,8 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) *ActiveSeriesMatchers { - asm := &ActiveSeriesMatchers{cfg: matchersConfig} +func NewMatchers(matchersConfig *CustomTrackersConfig) *Matchers { + asm := &Matchers{cfg: matchersConfig} for name, matchers := range (*matchersConfig).config { asm.matchers = append(asm.matchers, matchers) asm.names = append(asm.names, name) @@ -21,26 +21,26 @@ func NewActiveSeriesMatchers(matchersConfig *ActiveSeriesCustomTrackersConfig) * return asm } -type ActiveSeriesMatchers struct { - cfg *ActiveSeriesCustomTrackersConfig +type Matchers struct { + cfg *CustomTrackersConfig names []string matchers []labelsMatchers } -func (asm *ActiveSeriesMatchers) MatcherNames() []string { - return asm.names +func (m *Matchers) MatcherNames() []string { + return m.names } -func (asm *ActiveSeriesMatchers) Config() *ActiveSeriesCustomTrackersConfig { - return asm.cfg +func (m *Matchers) Config() *CustomTrackersConfig { + return m.cfg } -func (asm *ActiveSeriesMatchers) Matches(series labels.Labels) []bool { - if len(asm.matchers) == 0 { +func (m *Matchers) Matches(series labels.Labels) []bool { + if len(m.matchers) == 0 { return nil } - matches := make([]bool, len(asm.matchers)) - for i, sm := range asm.matchers { + matches := make([]bool, len(m.matchers)) + for i, sm := range m.matchers { matches[i] = sm.Matches(series) } return matches @@ -60,17 +60,17 @@ func (ms labelsMatchers) Matches(lset labels.Labels) bool { return true } -func (asm *ActiveSeriesMatchers) Len() int { - return len(asm.names) +func (m *Matchers) Len() int { + return len(m.names) } -func (asm *ActiveSeriesMatchers) Less(i, j int) bool { - return asm.names[i] < asm.names[j] +func (m *Matchers) Less(i, j int) bool { + return m.names[i] < m.names[j] } -func (asm *ActiveSeriesMatchers) Swap(i, j int) { - asm.names[i], asm.names[j] = asm.names[j], asm.names[i] - asm.matchers[i], asm.matchers[j] = asm.matchers[j], asm.matchers[i] +func (m *Matchers) Swap(i, j int) { + m.names[i], m.names[j] = m.names[j], m.names[i] + m.matchers[i], m.matchers[j] = m.matchers[j], m.matchers[i] } func amlabelMatcherToProm(m *amlabels.Matcher) *labels.Matcher { diff --git a/pkg/ingester/activeseries/active_series_custom_tracker_test.go b/pkg/ingester/activeseries/matchers_test.go similarity index 97% rename from pkg/ingester/activeseries/active_series_custom_tracker_test.go rename to pkg/ingester/activeseries/matchers_test.go index f6ed55dc51..e3dfd4f7d6 100644 --- a/pkg/ingester/activeseries/active_series_custom_tracker_test.go +++ b/pkg/ingester/activeseries/matchers_test.go @@ -13,7 +13,7 @@ import ( ) func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { - asm := NewActiveSeriesMatchers(mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ + asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "bar_starts_with_1": `{bar=~"1.*"}`, "does_not_have_foo_label": `{foo=""}`, "has_foo_and_bar_starts_with_1": `{foo!="", bar=~"1.*"}`, diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index f5f65c780f..a1e2f721fe 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -12,7 +12,6 @@ import ( "context" "flag" "fmt" - "github.com/grafana/mimir/pkg/ingester/activeseries" "io" "net/http" "os" @@ -48,6 +47,7 @@ import ( "golang.org/x/sync/errgroup" "google.golang.org/grpc/codes" + "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/ingester/client" "github.com/grafana/mimir/pkg/mimirpb" "github.com/grafana/mimir/pkg/storage/bucket" @@ -122,11 +122,11 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackersConfig activeseries.ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` - ActiveSeriesCustomTrackersOverridesFn func() *activeseries.ActiveSeriesCustomTrackersConfig `yaml:"-"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` + ActiveSeriesCustomTrackersOverridesFn func() *activeseries.CustomTrackersConfig `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -199,7 +199,7 @@ type Ingester struct { metrics *ingesterMetrics logger log.Logger - activeSeriesMatcher activeseries.ActiveSeriesMatchers + activeSeriesMatcher activeseries.Matchers lifecycler *ring.Lifecycler limits *validation.Overrides @@ -250,7 +250,7 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus return nil, errors.Wrap(err, "failed to create the bucket client") } - asm := activeseries.NewActiveSeriesMatchers(&cfg.ActiveSeriesCustomTrackersConfig) + asm := activeseries.NewMatchers(&cfg.ActiveSeriesCustomTrackersConfig) return &Ingester{ cfg: cfg, @@ -472,7 +472,7 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *activeseries.ActiveSeriesCustomTrackersConfig { +func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *activeseries.CustomTrackersConfig { matchersConfig := i.limits.ActiveSeriesCustomTrackersConfig(userID) if matchersConfig == nil { if i.cfg.ActiveSeriesCustomTrackersOverridesFn != nil { @@ -485,7 +485,7 @@ func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *activeseries.Ac return matchersConfig } -func (i *Ingester) replaceMatchers(asm *activeseries.ActiveSeriesMatchers, userDB *userTSDB) { +func (i *Ingester) replaceMatchers(asm *activeseries.Matchers, userDB *userTSDB) { i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatcherNames()) userDB.activeSeries.ReloadSeriesMatchers(asm) } @@ -499,7 +499,7 @@ func (i *Ingester) updateActiveSeries(now time.Time) { newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) if newMatchersConfig.String() != userDB.activeSeries.CurrentConfig().String() { - i.replaceMatchers(activeseries.NewActiveSeriesMatchers(newMatchersConfig), userDB) + i.replaceMatchers(activeseries.NewMatchers(newMatchersConfig), userDB) } if userDB.activeSeries.LastAsmUpdate() < now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout).UnixNano() { // We are not exposing metrics until enough time passed for stable metrics. @@ -1469,7 +1469,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userDB := &userTSDB{ userID: userID, - activeSeries: activeseries.NewActiveSeries(activeseries.NewActiveSeriesMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout), + activeSeries: activeseries.NewActiveSeries(activeseries.NewMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 67b2a1ff1c..9c63672dca 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -12,7 +12,6 @@ import ( "bytes" "context" "fmt" - "github.com/grafana/mimir/pkg/ingester/activeseries" "io" "io/ioutil" "math" @@ -53,6 +52,7 @@ import ( "golang.org/x/sync/errgroup" "google.golang.org/grpc" + "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/ingester/client" "github.com/grafana/mimir/pkg/mimirpb" "github.com/grafana/mimir/pkg/storage/chunk" @@ -64,7 +64,7 @@ import ( "github.com/grafana/mimir/pkg/util/validation" ) -func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *activeseries.ActiveSeriesCustomTrackersConfig { +func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *activeseries.CustomTrackersConfig { m, err := activeseries.NewActiveSeriesCustomTrackersConfig(source) require.NoError(t, err) return &m @@ -5167,7 +5167,7 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultActiveSeriesCustomTrackersFn := func() *activeseries.ActiveSeriesCustomTrackersConfig { + defaultActiveSeriesCustomTrackersFn := func() *activeseries.CustomTrackersConfig { return mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, @@ -5193,8 +5193,8 @@ func TestIngesterActiveSeries(t *testing.T) { reqs []*mimirpb.WriteRequest expectedMetrics string disableActiveSeries bool - defaultActiveSeriesCustomTrackers func() *activeseries.ActiveSeriesCustomTrackersConfig - activeSeriesConfig activeseries.ActiveSeriesCustomTrackersConfig + defaultActiveSeriesCustomTrackers func() *activeseries.CustomTrackersConfig + activeSeriesConfig activeseries.CustomTrackersConfig tenantLimits *TenantLimitsMock }{ "successful push, should count active series": { @@ -5339,7 +5339,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with empty runtime config": { - defaultActiveSeriesCustomTrackers: func() *activeseries.ActiveSeriesCustomTrackersConfig { return nil }, + defaultActiveSeriesCustomTrackers: func() *activeseries.CustomTrackersConfig { return nil }, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5362,7 +5362,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should not fail with nil provider and default config": { defaultActiveSeriesCustomTrackers: nil, - activeSeriesConfig: activeseries.ActiveSeriesCustomTrackersConfig{}, + activeSeriesConfig: activeseries.CustomTrackersConfig{}, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5506,7 +5506,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultActiveSeriesCustomTrackersFn := func() *activeseries.ActiveSeriesCustomTrackersConfig { + defaultActiveSeriesCustomTrackersFn := func() *activeseries.CustomTrackersConfig { return mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, @@ -5531,8 +5531,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest expectedMetrics string - defaultActiveSeriesCustomTrackers func() *activeseries.ActiveSeriesCustomTrackersConfig - activeSeriesConfig activeseries.ActiveSeriesCustomTrackersConfig + defaultActiveSeriesCustomTrackers func() *activeseries.CustomTrackersConfig + activeSeriesConfig activeseries.CustomTrackersConfig tenantLimits *TenantLimitsMock }{ "overwrite flag based config with runtime overwrite": { @@ -5756,6 +5756,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { } ing, err := prepareIngesterWithBlockStorageAndOverrides(t, cfg, overrides, "", registry) + require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), ing)) defer services.StopAndAwaitTerminated(context.Background(), ing) //nolint:errcheck diff --git a/pkg/ingester/user_tsdb.go b/pkg/ingester/user_tsdb.go index b55ee4e2f7..defcc02b59 100644 --- a/pkg/ingester/user_tsdb.go +++ b/pkg/ingester/user_tsdb.go @@ -7,7 +7,6 @@ package ingester import ( "context" - "github.com/grafana/mimir/pkg/ingester/activeseries" "os" "sync" "time" @@ -20,6 +19,7 @@ import ( "github.com/thanos-io/thanos/pkg/shipper" "go.uber.org/atomic" + "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/util/extract" util_math "github.com/grafana/mimir/pkg/util/math" ) diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index 29f3836a69..675fb3dac6 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -7,7 +7,6 @@ package mimir import ( "errors" - "github.com/grafana/mimir/pkg/ingester/activeseries" "io" "net/http" @@ -16,6 +15,7 @@ import ( "gopkg.in/yaml.v2" "github.com/grafana/mimir/pkg/ingester" + "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/util" "github.com/grafana/mimir/pkg/util/validation" ) @@ -36,7 +36,7 @@ type runtimeConfigValues struct { IngesterLimits *ingester.InstanceLimits `yaml:"ingester_limits"` - ActiveSeriesCustomTrackersDefaultOverrides *activeseries.ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers_overrides"` + ActiveSeriesCustomTrackersDefaultOverrides *activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_overrides"` } // runtimeConfigTenantLimits provides per-tenant limit overrides based on a runtimeconfig.Manager @@ -148,12 +148,12 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeActiveSeriesCustomTrackersDefaultOverridesFn(manager *runtimeconfig.Manager) func() *activeseries.ActiveSeriesCustomTrackersConfig { +func runtimeActiveSeriesCustomTrackersDefaultOverridesFn(manager *runtimeconfig.Manager) func() *activeseries.CustomTrackersConfig { if manager == nil { return nil } - return func() *activeseries.ActiveSeriesCustomTrackersConfig { + return func() *activeseries.CustomTrackersConfig { val := manager.GetConfig() if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { return cfg.ActiveSeriesCustomTrackersDefaultOverrides diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index a882323134..94aa58c350 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -9,7 +9,6 @@ import ( "bytes" "encoding/json" "flag" - "github.com/grafana/mimir/pkg/ingester/activeseries" "math" "strings" "time" @@ -18,6 +17,8 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/relabel" "golang.org/x/time/rate" + + "github.com/grafana/mimir/pkg/ingester/activeseries" ) // LimitError are errors that do not comply with the limits specified. @@ -57,7 +58,7 @@ type Limits struct { // Exemplars MaxGlobalExemplarsPerUser int `yaml:"max_global_exemplars_per_user" json:"max_global_exemplars_per_user" category:"experimental"` // Active series custom trackers - ActiveSeriesCustomTrackersConfig *activeseries.ActiveSeriesCustomTrackersConfig `yaml:"active_series_custom_trackers_config"` + ActiveSeriesCustomTrackersConfig *activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_config"` // Querier enforced limits. MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"` @@ -424,7 +425,7 @@ func (o *Overrides) MaxGlobalExemplarsPerUser(userID string) int { return o.getOverridesForUser(userID).MaxGlobalExemplarsPerUser } -func (o *Overrides) ActiveSeriesCustomTrackersConfig(userID string) *activeseries.ActiveSeriesCustomTrackersConfig { +func (o *Overrides) ActiveSeriesCustomTrackersConfig(userID string) *activeseries.CustomTrackersConfig { return o.getOverridesForUser(userID).ActiveSeriesCustomTrackersConfig } diff --git a/tools/doc-generator/parser.go b/tools/doc-generator/parser.go index 1b20c208cc..55a20e4e0a 100644 --- a/tools/doc-generator/parser.go +++ b/tools/doc-generator/parser.go @@ -8,7 +8,6 @@ package main import ( "flag" "fmt" - "github.com/grafana/mimir/pkg/ingester/activeseries" "net/url" "reflect" "regexp" @@ -23,6 +22,7 @@ import ( "github.com/prometheus/prometheus/model/relabel" "github.com/weaveworks/common/logging" + "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/util/fieldcategory" ) @@ -348,7 +348,7 @@ func getCustomFieldType(t reflect.Type) (string, bool) { return "string", true case reflect.TypeOf([]*relabel.Config{}).String(): return "relabel_config...", true - case reflect.TypeOf(activeseries.ActiveSeriesCustomTrackersConfig{}).String(): + case reflect.TypeOf(activeseries.CustomTrackersConfig{}).String(): return "map of tracker name (string) to matcher (string)", true default: return "", false From 3035b4ffbdf8fda6500773515d0c88577411539f Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Wed, 2 Mar 2022 11:10:34 +0100 Subject: [PATCH 60/91] Use plain struct for config, not pointer Signed-off-by: Oleg Zaytsev --- .../reference-configuration-parameters.md | 9 +++++ pkg/ingester/activeseries/active_series.go | 2 +- .../activeseries/custom_trackers_config.go | 8 +++-- .../custom_trackers_config_test.go | 34 +++++++++---------- pkg/ingester/activeseries/matchers.go | 8 ++--- pkg/ingester/ingester.go | 29 +++++++--------- pkg/ingester/ingester_test.go | 24 ++++++------- pkg/mimir/runtime_config.go | 8 ++--- pkg/util/validation/limits.go | 4 +-- 9 files changed, 67 insertions(+), 59 deletions(-) diff --git a/docs/sources/configuration/reference-configuration-parameters.md b/docs/sources/configuration/reference-configuration-parameters.md index b00d0bc515..c8643afc22 100644 --- a/docs/sources/configuration/reference-configuration-parameters.md +++ b/docs/sources/configuration/reference-configuration-parameters.md @@ -2523,6 +2523,15 @@ The `limits` block configures default and per-tenant limits imposed by component # CLI flag: -ingester.max-global-exemplars-per-user [max_global_exemplars_per_user: | default = 0] +# Example: +# The following configuration will count the active series coming from dev and +# prod namespaces for each tenant and label them as {name="dev"} and +# {name="prod"} in the cortex_ingester_active_series_custom_tracker metric. +# active_series_custom_trackers_config: +# dev: '{namespace=~"dev-.*"}' +# prod: '{namespace=~"prod-.*"}' +[active_series_custom_trackers_config: | default = ] + # Maximum number of chunks that can be fetched in a single query from ingesters # and long-term storage. This limit is enforced in the querier, ruler and # store-gateway. 0 to disable. diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index 57a5cbadc6..3f6c42e08a 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -86,7 +86,7 @@ func (c *ActiveSeries) LastAsmUpdate() int64 { return c.lastAsmUpdate.Load() } -func (c *ActiveSeries) CurrentConfig() *CustomTrackersConfig { +func (c *ActiveSeries) CurrentConfig() CustomTrackersConfig { c.mu.RLock() defer c.mu.RUnlock() return c.asm.Config() diff --git a/pkg/ingester/activeseries/custom_trackers_config.go b/pkg/ingester/activeseries/custom_trackers_config.go index e654ecf62d..de7f30af85 100644 --- a/pkg/ingester/activeseries/custom_trackers_config.go +++ b/pkg/ingester/activeseries/custom_trackers_config.go @@ -19,7 +19,7 @@ type CustomTrackersConfig struct { } // ExampleDoc provides an example doc for this config, especially valuable since it's custom-unmarshaled. -func (c *CustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { +func (c CustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { return `The following configuration will count the active series coming from dev and prod namespaces for each tenant` + ` and label them as {name="dev"} and {name="prod"} in the cortex_ingester_active_series_custom_tracker metric.`, map[string]string{ @@ -28,9 +28,13 @@ func (c *CustomTrackersConfig) ExampleDoc() (comment string, yaml interface{}) { } } +func (c CustomTrackersConfig) Empty() bool { + return c.string == "" +} + // String is a canonical representation of the config, it is compatible with flag definition. // String is also needed to implement flag.Value. -func (c *CustomTrackersConfig) String() string { +func (c CustomTrackersConfig) String() string { return c.string } diff --git a/pkg/ingester/activeseries/custom_trackers_config_test.go b/pkg/ingester/activeseries/custom_trackers_config_test.go index 008083d6ad..5171467a4a 100644 --- a/pkg/ingester/activeseries/custom_trackers_config_test.go +++ b/pkg/ingester/activeseries/custom_trackers_config_test.go @@ -12,37 +12,37 @@ import ( "gopkg.in/yaml.v2" ) -func mustNewCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *CustomTrackersConfig { +func mustNewCustomTrackersConfigFromMap(t *testing.T, source map[string]string) CustomTrackersConfig { m, err := NewActiveSeriesCustomTrackersConfig(source) require.NoError(t, err) - return &m + return m } -func mustNewCustomTrackersConfigFromString(t *testing.T, source string) *CustomTrackersConfig { +func mustNewCustomTrackersConfigFromString(t *testing.T, source string) CustomTrackersConfig { m := CustomTrackersConfig{} err := m.Set(source) require.NoError(t, err) - return &m + return m } -func mustNewCustomTrackersConfigDeserializedFromYaml(t *testing.T, yamlString string) *CustomTrackersConfig { +func mustNewCustomTrackersConfigDeserializedFromYaml(t *testing.T, yamlString string) CustomTrackersConfig { m := CustomTrackersConfig{} err := yaml.Unmarshal([]byte(yamlString), &m) require.NoError(t, err) - return &m + return m } func TestCustomTrackersConfigs(t *testing.T) { for _, tc := range []struct { name string flags []string - expected *CustomTrackersConfig + expected CustomTrackersConfig error error }{ { name: "empty flag value produces empty config", flags: []string{`-ingester.active-series-custom-trackers=`}, - expected: &CustomTrackersConfig{}, + expected: CustomTrackersConfig{}, }, { name: "empty matcher fails", @@ -112,7 +112,7 @@ func TestCustomTrackersConfigs(t *testing.T) { return } - require.Equal(t, tc.expected, &config) + require.Equal(t, tc.expected, config) // Check that CustomTrackersConfig.String() value is a valid flag value. flagSetAgain := flag.NewFlagSet("test-string", flag.ContinueOnError) @@ -120,7 +120,7 @@ func TestCustomTrackersConfigs(t *testing.T) { flagSetAgain.Var(&configAgain, "ingester.active-series-custom-trackers", "...usage docs...") require.NoError(t, flagSetAgain.Parse([]string{"-ingester.active-series-custom-trackers=" + config.String()})) - require.Equal(t, tc.expected, &configAgain) + require.Equal(t, tc.expected, configAgain) }) } } @@ -128,23 +128,23 @@ func TestCustomTrackersConfigs(t *testing.T) { func TestCustomTrackerConfig_Equality(t *testing.T) { configSets := [][]CustomTrackersConfig{ { - *mustNewCustomTrackersConfigFromString(t, `foo:{foo='bar'};baz:{baz='bar'}`), - *mustNewCustomTrackersConfigFromMap(t, map[string]string{ + mustNewCustomTrackersConfigFromString(t, `foo:{foo='bar'};baz:{baz='bar'}`), + mustNewCustomTrackersConfigFromMap(t, map[string]string{ "baz": `{baz='bar'}`, "foo": `{foo='bar'}`, }), - *mustNewCustomTrackersConfigDeserializedFromYaml(t, + mustNewCustomTrackersConfigDeserializedFromYaml(t, ` baz: "{baz='bar'}" foo: "{foo='bar'}"`), }, { - *mustNewCustomTrackersConfigFromString(t, `test:{test='true'}`), - *mustNewCustomTrackersConfigFromMap(t, map[string]string{"test": `{test='true'}`}), - *mustNewCustomTrackersConfigDeserializedFromYaml(t, `test: "{test='true'}"`), + mustNewCustomTrackersConfigFromString(t, `test:{test='true'}`), + mustNewCustomTrackersConfigFromMap(t, map[string]string{"test": `{test='true'}`}), + mustNewCustomTrackersConfigDeserializedFromYaml(t, `test: "{test='true'}"`), }, { - *mustNewCustomTrackersConfigDeserializedFromYaml(t, + mustNewCustomTrackersConfigDeserializedFromYaml(t, ` baz: "{baz='bar'}" foo: "{foo='bar'}" diff --git a/pkg/ingester/activeseries/matchers.go b/pkg/ingester/activeseries/matchers.go index 2f45d98ca8..a276f077eb 100644 --- a/pkg/ingester/activeseries/matchers.go +++ b/pkg/ingester/activeseries/matchers.go @@ -9,9 +9,9 @@ import ( "github.com/prometheus/prometheus/model/labels" ) -func NewMatchers(matchersConfig *CustomTrackersConfig) *Matchers { +func NewMatchers(matchersConfig CustomTrackersConfig) *Matchers { asm := &Matchers{cfg: matchersConfig} - for name, matchers := range (*matchersConfig).config { + for name, matchers := range matchersConfig.config { asm.matchers = append(asm.matchers, matchers) asm.names = append(asm.names, name) } @@ -22,7 +22,7 @@ func NewMatchers(matchersConfig *CustomTrackersConfig) *Matchers { } type Matchers struct { - cfg *CustomTrackersConfig + cfg CustomTrackersConfig names []string matchers []labelsMatchers } @@ -31,7 +31,7 @@ func (m *Matchers) MatcherNames() []string { return m.names } -func (m *Matchers) Config() *CustomTrackersConfig { +func (m *Matchers) Config() CustomTrackersConfig { return m.cfg } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index deafc78450..9aa8f7b897 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -122,11 +122,11 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` - ActiveSeriesCustomTrackersOverridesFn func() *activeseries.CustomTrackersConfig `yaml:"-"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` + ActiveSeriesCustomTrackersOverridesFn func() activeseries.CustomTrackersConfig `yaml:"-"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -199,8 +199,6 @@ type Ingester struct { metrics *ingesterMetrics logger log.Logger - activeSeriesMatcher activeseries.Matchers - lifecycler *ring.Lifecycler limits *validation.Overrides limiter *Limiter @@ -250,13 +248,10 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus return nil, errors.Wrap(err, "failed to create the bucket client") } - asm := activeseries.NewMatchers(&cfg.ActiveSeriesCustomTrackersConfig) - return &Ingester{ - cfg: cfg, - limits: limits, - logger: logger, - activeSeriesMatcher: *asm, + cfg: cfg, + limits: limits, + logger: logger, tsdbs: make(map[string]*userTSDB), usersMetadata: make(map[string]*userMetricsMetadata), @@ -472,15 +467,15 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) getActiveSeriesMatchersConfig(userID string) *activeseries.CustomTrackersConfig { +func (i *Ingester) getActiveSeriesMatchersConfig(userID string) activeseries.CustomTrackersConfig { matchersConfig := i.limits.ActiveSeriesCustomTrackersConfig(userID) - if matchersConfig == nil { + if matchersConfig.Empty() { if i.cfg.ActiveSeriesCustomTrackersOverridesFn != nil { matchersConfig = i.cfg.ActiveSeriesCustomTrackersOverridesFn() } } - if matchersConfig == nil { - matchersConfig = i.activeSeriesMatcher.Config() + if matchersConfig.Empty() { + matchersConfig = i.cfg.ActiveSeriesCustomTrackersConfig } return matchersConfig } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 9c63672dca..b3f81ef86d 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -64,10 +64,10 @@ import ( "github.com/grafana/mimir/pkg/util/validation" ) -func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) *activeseries.CustomTrackersConfig { +func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) activeseries.CustomTrackersConfig { m, err := activeseries.NewActiveSeriesCustomTrackersConfig(source) require.NoError(t, err) - return &m + return m } func TestIngester_Push(t *testing.T) { @@ -5167,7 +5167,7 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultActiveSeriesCustomTrackersFn := func() *activeseries.CustomTrackersConfig { + defaultActiveSeriesCustomTrackersFn := func() activeseries.CustomTrackersConfig { return mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, @@ -5193,7 +5193,7 @@ func TestIngesterActiveSeries(t *testing.T) { reqs []*mimirpb.WriteRequest expectedMetrics string disableActiveSeries bool - defaultActiveSeriesCustomTrackers func() *activeseries.CustomTrackersConfig + defaultActiveSeriesCustomTrackers func() activeseries.CustomTrackersConfig activeSeriesConfig activeseries.CustomTrackersConfig tenantLimits *TenantLimitsMock }{ @@ -5339,7 +5339,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should not fail with empty runtime config": { - defaultActiveSeriesCustomTrackers: func() *activeseries.CustomTrackersConfig { return nil }, + defaultActiveSeriesCustomTrackers: func() activeseries.CustomTrackersConfig { return activeseries.CustomTrackersConfig{} }, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5385,7 +5385,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should use flag based custom tracker if no runtime config specified": { defaultActiveSeriesCustomTrackers: nil, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5415,7 +5415,7 @@ func TestIngesterActiveSeries(t *testing.T) { "should use runtime matcher config if both specified": { tenantLimits: activeSeriesTenantOverride, defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5506,7 +5506,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultActiveSeriesCustomTrackersFn := func() *activeseries.CustomTrackersConfig { + defaultActiveSeriesCustomTrackersFn := func() activeseries.CustomTrackersConfig { return mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true": `{bool="true"}`, "bool_is_false": `{bool="false"}`, @@ -5531,14 +5531,14 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) reqs []*mimirpb.WriteRequest expectedMetrics string - defaultActiveSeriesCustomTrackers func() *activeseries.CustomTrackersConfig + defaultActiveSeriesCustomTrackers func() activeseries.CustomTrackersConfig activeSeriesConfig activeseries.CustomTrackersConfig tenantLimits *TenantLimitsMock }{ "overwrite flag based config with runtime overwrite": { defaultActiveSeriesCustomTrackers: nil, tenantLimits: nil, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5603,7 +5603,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { }, "remove runtime overwrite and revert to flag based config": { defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, tenantLimits: defaultActiveSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5666,7 +5666,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { }, "changing runtime overwrite should result in new metrics": { defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - activeSeriesConfig: *activeSeriesDefaultConfig, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index 675fb3dac6..a400ae63f0 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -36,7 +36,7 @@ type runtimeConfigValues struct { IngesterLimits *ingester.InstanceLimits `yaml:"ingester_limits"` - ActiveSeriesCustomTrackersDefaultOverrides *activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_overrides"` + ActiveSeriesCustomTrackersDefaultOverrides activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_overrides"` } // runtimeConfigTenantLimits provides per-tenant limit overrides based on a runtimeconfig.Manager @@ -148,17 +148,17 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeActiveSeriesCustomTrackersDefaultOverridesFn(manager *runtimeconfig.Manager) func() *activeseries.CustomTrackersConfig { +func runtimeActiveSeriesCustomTrackersDefaultOverridesFn(manager *runtimeconfig.Manager) func() activeseries.CustomTrackersConfig { if manager == nil { return nil } - return func() *activeseries.CustomTrackersConfig { + return func() activeseries.CustomTrackersConfig { val := manager.GetConfig() if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { return cfg.ActiveSeriesCustomTrackersDefaultOverrides } - return nil + return activeseries.CustomTrackersConfig{} } } diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 301eb98103..7f2913ddc2 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -58,7 +58,7 @@ type Limits struct { // Exemplars MaxGlobalExemplarsPerUser int `yaml:"max_global_exemplars_per_user" json:"max_global_exemplars_per_user" category:"experimental"` // Active series custom trackers - ActiveSeriesCustomTrackersConfig *activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_config"` + ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_config"` // Querier enforced limits. MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"` @@ -425,7 +425,7 @@ func (o *Overrides) MaxGlobalExemplarsPerUser(userID string) int { return o.getOverridesForUser(userID).MaxGlobalExemplarsPerUser } -func (o *Overrides) ActiveSeriesCustomTrackersConfig(userID string) *activeseries.CustomTrackersConfig { +func (o *Overrides) ActiveSeriesCustomTrackersConfig(userID string) activeseries.CustomTrackersConfig { return o.getOverridesForUser(userID).ActiveSeriesCustomTrackersConfig } From 44f9fdc16b0ad538f56403499a1cadb03ae51db6 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 3 Mar 2022 08:01:24 +0100 Subject: [PATCH 61/91] CustomTrackersConfig annotation fix --- pkg/util/validation/limits.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 7f2913ddc2..8fdf9057c4 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -58,7 +58,7 @@ type Limits struct { // Exemplars MaxGlobalExemplarsPerUser int `yaml:"max_global_exemplars_per_user" json:"max_global_exemplars_per_user" category:"experimental"` // Active series custom trackers - ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_config"` + ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_config" json:"active_series_custom_trackers_config" category:"advanced"` // Querier enforced limits. MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"` From bfe3f6db3e8a4ea8b55b4b978f9103f8de5f0f63 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 3 Mar 2022 08:12:01 +0100 Subject: [PATCH 62/91] Adding documentation on limits --- .../configuration/reference-configuration-parameters.md | 4 ++++ pkg/util/validation/limits.go | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/sources/configuration/reference-configuration-parameters.md b/docs/sources/configuration/reference-configuration-parameters.md index c8643afc22..cc076fcdb1 100644 --- a/docs/sources/configuration/reference-configuration-parameters.md +++ b/docs/sources/configuration/reference-configuration-parameters.md @@ -2523,6 +2523,10 @@ The `limits` block configures default and per-tenant limits imposed by component # CLI flag: -ingester.max-global-exemplars-per-user [max_global_exemplars_per_user: | default = 0] +# (advanced) Additional custom trackers for active metrics. If there are active +# series matching a provided matcher (map value), the count will be exposed in +# the custom trackers metric labeled using the tracker name (map key). Zero +# valued counts are not exposed (and removed when they go back to zero). # Example: # The following configuration will count the active series coming from dev and # prod namespaces for each tenant and label them as {name="dev"} and diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 8fdf9057c4..4d0d0b46e1 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -58,7 +58,7 @@ type Limits struct { // Exemplars MaxGlobalExemplarsPerUser int `yaml:"max_global_exemplars_per_user" json:"max_global_exemplars_per_user" category:"experimental"` // Active series custom trackers - ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_config" json:"active_series_custom_trackers_config" category:"advanced"` + ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_config" json:"active_series_custom_trackers_config" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` // Querier enforced limits. MaxChunksPerQuery int `yaml:"max_fetched_chunks_per_query" json:"max_fetched_chunks_per_query"` From e289c56e6a1375e94bc8687d94d5d2c6d736c092 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 3 Mar 2022 16:03:51 +0100 Subject: [PATCH 63/91] Moving active serreis flag to limits, removing runtime default, fixing up tests --- .../reference-configuration-parameters.md | 15 +- pkg/ingester/ingester.go | 21 +- pkg/ingester/ingester_test.go | 242 ++++-------------- pkg/mimir/modules.go | 1 - pkg/mimir/runtime_config.go | 17 -- pkg/util/validation/limits.go | 1 + 6 files changed, 62 insertions(+), 235 deletions(-) diff --git a/docs/sources/configuration/reference-configuration-parameters.md b/docs/sources/configuration/reference-configuration-parameters.md index cc076fcdb1..fe7d8120ec 100644 --- a/docs/sources/configuration/reference-configuration-parameters.md +++ b/docs/sources/configuration/reference-configuration-parameters.md @@ -794,20 +794,6 @@ ring: # CLI flag: -ingester.active-series-metrics-idle-timeout [active_series_metrics_idle_timeout: | default = 10m] -# (advanced) Additional custom trackers for active metrics. If there are active -# series matching a provided matcher (map value), the count will be exposed in -# the custom trackers metric labeled using the tracker name (map key). Zero -# valued counts are not exposed (and removed when they go back to zero). -# Example: -# The following configuration will count the active series coming from dev and -# prod namespaces for each tenant and label them as {name="dev"} and -# {name="prod"} in the cortex_ingester_active_series_custom_tracker metric. -# active_series_custom_trackers: -# dev: '{namespace=~"dev-.*"}' -# prod: '{namespace=~"prod-.*"}' -# CLI flag: -ingester.active-series-custom-trackers -[active_series_custom_trackers: | default = ] - # (experimental) Period with which to update per-tenant max exemplar limit. # CLI flag: -ingester.exemplars-update-period [exemplars_update_period: | default = 15s] @@ -2534,6 +2520,7 @@ The `limits` block configures default and per-tenant limits imposed by component # active_series_custom_trackers_config: # dev: '{namespace=~"dev-.*"}' # prod: '{namespace=~"prod-.*"}' +# CLI flag: -ingester.active-series-custom-trackers [active_series_custom_trackers_config: | default = ] # Maximum number of chunks that can be fetched in a single query from ingesters diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 9aa8f7b897..842a854a59 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -122,11 +122,9 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` - ActiveSeriesCustomTrackersConfig activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` - ActiveSeriesCustomTrackersOverridesFn func() activeseries.CustomTrackersConfig `yaml:"-"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` @@ -154,7 +152,6 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { f.BoolVar(&cfg.ActiveSeriesMetricsEnabled, "ingester.active-series-metrics-enabled", true, "Enable tracking of active series and export them as metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsUpdatePeriod, "ingester.active-series-metrics-update-period", 1*time.Minute, "How often to update active series metrics.") f.DurationVar(&cfg.ActiveSeriesMetricsIdleTimeout, "ingester.active-series-metrics-idle-timeout", 10*time.Minute, "After what time a series is considered to be inactive.") - f.Var(&cfg.ActiveSeriesCustomTrackersConfig, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag.") f.BoolVar(&cfg.StreamChunksWhenUsingBlocks, "ingester.stream-chunks-when-using-blocks", true, "Stream chunks from ingesters to queriers.") f.DurationVar(&cfg.ExemplarsUpdatePeriod, "ingester.exemplars-update-period", 15*time.Second, "Period with which to update per-tenant max exemplar limit.") @@ -468,19 +465,11 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } func (i *Ingester) getActiveSeriesMatchersConfig(userID string) activeseries.CustomTrackersConfig { - matchersConfig := i.limits.ActiveSeriesCustomTrackersConfig(userID) - if matchersConfig.Empty() { - if i.cfg.ActiveSeriesCustomTrackersOverridesFn != nil { - matchersConfig = i.cfg.ActiveSeriesCustomTrackersOverridesFn() - } - } - if matchersConfig.Empty() { - matchersConfig = i.cfg.ActiveSeriesCustomTrackersConfig - } - return matchersConfig + return i.limits.ActiveSeriesCustomTrackersConfig(userID) } func (i *Ingester) replaceMatchers(asm *activeseries.Matchers, userDB *userTSDB) { + i.metrics.activeSeriesPerUser.DeleteLabelValues(userDB.userID) i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatcherNames()) userDB.activeSeries.ReloadSeriesMatchers(asm) } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index b3f81ef86d..fa66d18d5c 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -4869,8 +4869,8 @@ func TestIngester_Push_SeriesWithBlankLabel(t *testing.T) { require.NoError(t, err) res, _, err := runTestQuery(ctx, t, ing, labels.MatchEqual, labels.MetricName, "testmetric") - require.NoError(t, err) + require.NoError(t, err) expected := model.Matrix{ { Metric: model.Metric{labels.MetricName: "testmetric"}, @@ -5167,13 +5167,6 @@ func TestIngesterActiveSeries(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultActiveSeriesCustomTrackersFn := func() activeseries.CustomTrackersConfig { - return mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, - }) - } - activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, @@ -5189,17 +5182,12 @@ func TestIngesterActiveSeries(t *testing.T) { activeSeriesTenantOverride.On("ByUserID", userID2).Return(nil) tests := map[string]struct { - test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) - reqs []*mimirpb.WriteRequest - expectedMetrics string - disableActiveSeries bool - defaultActiveSeriesCustomTrackers func() activeseries.CustomTrackersConfig - activeSeriesConfig activeseries.CustomTrackersConfig - tenantLimits *TenantLimitsMock + test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) + reqs []*mimirpb.WriteRequest + expectedMetrics string + disableActiveSeries bool }{ "successful push, should count active series": { - defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - tenantLimits: activeSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5218,8 +5206,8 @@ func TestIngesterActiveSeries(t *testing.T) { # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 ` // Check tracked Prometheus metrics @@ -5227,8 +5215,6 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should cleanup metrics when tsdb closed": { - defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - tenantLimits: activeSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5247,8 +5233,8 @@ func TestIngesterActiveSeries(t *testing.T) { # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 ` // Check tracked Prometheus metrics @@ -5260,8 +5246,6 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "should track custom matchers, removing when zero": { - defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - tenantLimits: activeSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5280,8 +5264,8 @@ func TestIngesterActiveSeries(t *testing.T) { # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 ` // Check tracked Prometheus metrics @@ -5321,8 +5305,7 @@ func TestIngesterActiveSeries(t *testing.T) { }, }, "successful push, active series disabled": { - defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - disableActiveSeries: true, + disableActiveSeries: true, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { now := time.Now() @@ -5334,110 +5317,6 @@ func TestIngesterActiveSeries(t *testing.T) { expectedMetrics := `` - // Check tracked Prometheus metrics - require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - }, - }, - "should not fail with empty runtime config": { - defaultActiveSeriesCustomTrackers: func() activeseries.CustomTrackersConfig { return activeseries.CustomTrackersConfig{} }, - test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - now := time.Now() - - pushWithUser(t, ingester, labelsToPush, userID, req) - pushWithUser(t, ingester, labelsToPush, userID2, req) - - // Update active series for metrics check. - ingester.updateActiveSeries(now) - - expectedMetrics := ` - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 - ` - - // Check tracked Prometheus metrics - require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - }, - }, - "should not fail with nil provider and default config": { - defaultActiveSeriesCustomTrackers: nil, - activeSeriesConfig: activeseries.CustomTrackersConfig{}, - test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - now := time.Now() - - pushWithUser(t, ingester, labelsToPush, userID, req) - pushWithUser(t, ingester, labelsToPush, userID2, req) - - // Update active series for metrics check. - ingester.updateActiveSeries(now) - - expectedMetrics := ` - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 - ` - - // Check tracked Prometheus metrics - require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - }, - }, - "should use flag based custom tracker if no runtime config specified": { - defaultActiveSeriesCustomTrackers: nil, - activeSeriesConfig: activeSeriesDefaultConfig, - test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - now := time.Now() - - pushWithUser(t, ingester, labelsToPush, userID, req) - pushWithUser(t, ingester, labelsToPush, userID2, req) - - // Update active series for metrics check. - ingester.updateActiveSeries(now) - - expectedMetrics := ` - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 - # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. - # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="test_user"} 2 - ` - - // Check tracked Prometheus metrics - require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - }, - }, - "should use runtime matcher config if both specified": { - tenantLimits: activeSeriesTenantOverride, - defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - activeSeriesConfig: activeSeriesDefaultConfig, - test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - now := time.Now() - - pushWithUser(t, ingester, labelsToPush, userID, req) - pushWithUser(t, ingester, labelsToPush, userID2, req) - - // Update active series for metrics check. - ingester.updateActiveSeries(now) - - expectedMetrics := ` - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 - # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. - # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 - ` - // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, @@ -5452,19 +5331,11 @@ func TestIngesterActiveSeries(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.IngesterRing.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = !testData.disableActiveSeries - cfg.ActiveSeriesCustomTrackersOverridesFn = testData.defaultActiveSeriesCustomTrackers - cfg.ActiveSeriesCustomTrackersConfig = testData.activeSeriesConfig + limits := defaultLimitsTestConfig() - var overrides *validation.Overrides - var err error - // Without this, TenantLimitsMock(nil) != nil when using getOverridesForUser in limits.go - if testData.tenantLimits != nil { - overrides, err = validation.NewOverrides(limits, testData.tenantLimits) - require.NoError(t, err) - } else { - overrides, err = validation.NewOverrides(limits, nil) - require.NoError(t, err) - } + limits.ActiveSeriesCustomTrackersConfig = activeSeriesDefaultConfig + overrides, err := validation.NewOverrides(limits, activeSeriesTenantOverride) + require.NoError(t, err) ing, err := prepareIngesterWithBlockStorageAndOverrides(t, cfg, overrides, "", registry) require.NoError(t, err) @@ -5506,13 +5377,6 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { userID := "test_user" userID2 := "other_test_user" - defaultActiveSeriesCustomTrackersFn := func() activeseries.CustomTrackersConfig { - return mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ - "bool_is_true": `{bool="true"}`, - "bool_is_false": `{bool="false"}`, - }) - } - activeSeriesDefaultConfig := mustNewActiveSeriesCustomTrackersConfigFromMap(t, map[string]string{ "bool_is_true_flagbased": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, @@ -5528,17 +5392,15 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { defaultActiveSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: activeSeriesTenantConfig}) tests := map[string]struct { - test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) - reqs []*mimirpb.WriteRequest - expectedMetrics string - defaultActiveSeriesCustomTrackers func() activeseries.CustomTrackersConfig - activeSeriesConfig activeseries.CustomTrackersConfig - tenantLimits *TenantLimitsMock + test func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) + reqs []*mimirpb.WriteRequest + expectedMetrics string + activeSeriesConfig activeseries.CustomTrackersConfig + tenantLimits *TenantLimitsMock }{ - "overwrite flag based config with runtime overwrite": { - defaultActiveSeriesCustomTrackers: nil, - tenantLimits: nil, - activeSeriesConfig: activeSeriesDefaultConfig, + "override flag based config with runtime overwrite": { + tenantLimits: nil, + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5564,19 +5426,24 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) // Add new runtime configs - ingester.cfg.ActiveSeriesCustomTrackersOverridesFn = defaultActiveSeriesCustomTrackersFn activeSeriesTenantOverride := new(TenantLimitsMock) activeSeriesTenantOverride.On("ByUserID", userID2).Return(nil) activeSeriesTenantOverride.On("ByUserID", userID).Return(&validation.Limits{ActiveSeriesCustomTrackersConfig: activeSeriesTenantConfig}) - override, err := validation.NewOverrides(defaultLimitsTestConfig(), activeSeriesTenantOverride) + limits := defaultLimitsTestConfig() + limits.ActiveSeriesCustomTrackersConfig = activeSeriesDefaultConfig + override, err := validation.NewOverrides(limits, activeSeriesTenantOverride) require.NoError(t, err) ingester.limits = override + // First update reloads the config ingester.updateActiveSeries(firstPushTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5585,6 +5452,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { time.Sleep(time.Millisecond) pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) + // Adding idleTimeout to expose the metrics ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5593,8 +5461,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 ` @@ -5602,9 +5470,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { }, }, "remove runtime overwrite and revert to flag based config": { - defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - activeSeriesConfig: activeSeriesDefaultConfig, - tenantLimits: defaultActiveSeriesTenantOverride, + activeSeriesConfig: activeSeriesDefaultConfig, + tenantLimits: defaultActiveSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5621,8 +5488,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="other_test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 ` @@ -5630,16 +5497,20 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) // Remove runtime configs - ingester.cfg.ActiveSeriesCustomTrackersOverridesFn = nil - var err error - ingester.limits, err = validation.NewOverrides(defaultLimitsTestConfig(), nil) + limits := defaultLimitsTestConfig() + limits.ActiveSeriesCustomTrackersConfig = activeSeriesDefaultConfig + override, err := validation.NewOverrides(limits, nil) require.NoError(t, err) + ingester.limits = override ingester.updateActiveSeries(firstPushTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5664,9 +5535,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, }, - "changing runtime overwrite should result in new metrics": { - defaultActiveSeriesCustomTrackers: defaultActiveSeriesCustomTrackersFn, - activeSeriesConfig: activeSeriesDefaultConfig, + "changing runtime override should result in new metrics": { + activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { firstPushTime := time.Now() @@ -5681,8 +5551,8 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge - cortex_ingester_active_series_custom_tracker{name="bool_is_false",user="test_user"} 2 - cortex_ingester_active_series_custom_tracker{name="bool_is_true",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="test_user"} 2 ` // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5695,16 +5565,15 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { "team_c": `{team="b"}`, "team_d": `{team="b"}`, })}) - override, err := validation.NewOverrides(defaultLimitsTestConfig(), activeSeriesTenantOverride) + limits := defaultLimitsTestConfig() + limits.ActiveSeriesCustomTrackersConfig = activeSeriesDefaultConfig + override, err := validation.NewOverrides(limits, activeSeriesTenantOverride) require.NoError(t, err) ingester.limits = override // This will update the runtime config. configReloadTime := time.Now() ingester.updateActiveSeries(configReloadTime) expectedMetrics = ` - # HELP cortex_ingester_active_series Number of currently active series per user. - # TYPE cortex_ingester_active_series gauge - cortex_ingester_active_series{user="test_user"} 4 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5740,10 +5609,9 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { cfg := defaultIngesterTestConfig(t) cfg.IngesterRing.JoinAfter = 0 cfg.ActiveSeriesMetricsEnabled = true - cfg.ActiveSeriesCustomTrackersOverridesFn = testData.defaultActiveSeriesCustomTrackers - cfg.ActiveSeriesCustomTrackersConfig = testData.activeSeriesConfig limits := defaultLimitsTestConfig() + limits.ActiveSeriesCustomTrackersConfig = testData.activeSeriesConfig var overrides *validation.Overrides var err error // Without this, TenantLimitsMock(nil) != nil when using getOverridesForUser in limits.go diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 40edaae0e6..67b2061871 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -437,7 +437,6 @@ func (t *Mimir) initIngesterService() (serv services.Service, err error) { t.Cfg.Ingester.IngesterRing.ListenPort = t.Cfg.Server.GRPCListenPort t.Cfg.Ingester.StreamTypeFn = ingesterChunkStreaming(t.RuntimeConfig) t.Cfg.Ingester.InstanceLimitsFn = ingesterInstanceLimits(t.RuntimeConfig) - t.Cfg.Ingester.ActiveSeriesCustomTrackersOverridesFn = runtimeActiveSeriesCustomTrackersDefaultOverridesFn(t.RuntimeConfig) t.tsdbIngesterConfig() t.Ingester, err = ingester.New(t.Cfg.Ingester, t.Cfg.IngesterClient, t.Overrides, prometheus.DefaultRegisterer, util_log.Logger) diff --git a/pkg/mimir/runtime_config.go b/pkg/mimir/runtime_config.go index a400ae63f0..769de95bce 100644 --- a/pkg/mimir/runtime_config.go +++ b/pkg/mimir/runtime_config.go @@ -15,7 +15,6 @@ import ( "gopkg.in/yaml.v2" "github.com/grafana/mimir/pkg/ingester" - "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/util" "github.com/grafana/mimir/pkg/util/validation" ) @@ -35,8 +34,6 @@ type runtimeConfigValues struct { IngesterChunkStreaming *bool `yaml:"ingester_stream_chunks_when_using_blocks"` IngesterLimits *ingester.InstanceLimits `yaml:"ingester_limits"` - - ActiveSeriesCustomTrackersDefaultOverrides activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers_overrides"` } // runtimeConfigTenantLimits provides per-tenant limit overrides based on a runtimeconfig.Manager @@ -148,20 +145,6 @@ func ingesterInstanceLimits(manager *runtimeconfig.Manager) func() *ingester.Ins } } -func runtimeActiveSeriesCustomTrackersDefaultOverridesFn(manager *runtimeconfig.Manager) func() activeseries.CustomTrackersConfig { - if manager == nil { - return nil - } - - return func() activeseries.CustomTrackersConfig { - val := manager.GetConfig() - if cfg, ok := val.(*runtimeConfigValues); ok && cfg != nil { - return cfg.ActiveSeriesCustomTrackersDefaultOverrides - } - return activeseries.CustomTrackersConfig{} - } -} - func runtimeConfigHandler(runtimeCfgManager *runtimeconfig.Manager, defaultLimits validation.Limits) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { cfg, ok := runtimeCfgManager.GetConfig().(*runtimeConfigValues) diff --git a/pkg/util/validation/limits.go b/pkg/util/validation/limits.go index 4d0d0b46e1..a95969ed8d 100644 --- a/pkg/util/validation/limits.go +++ b/pkg/util/validation/limits.go @@ -137,6 +137,7 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.IntVar(&l.MaxGlobalMetricsWithMetadataPerUser, "ingester.max-global-metadata-per-user", 0, "The maximum number of active metrics with metadata per tenant, across the cluster. 0 to disable.") f.IntVar(&l.MaxGlobalMetadataPerMetric, "ingester.max-global-metadata-per-metric", 0, "The maximum number of metadata per metric, across the cluster. 0 to disable.") f.IntVar(&l.MaxGlobalExemplarsPerUser, "ingester.max-global-exemplars-per-user", 0, "The maximum number of exemplars in memory, across the cluster. 0 to disable exemplars ingestion.") + f.Var(&l.ActiveSeriesCustomTrackersConfig, "ingester.active-series-custom-trackers", "Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo=\"bar\"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag.") f.IntVar(&l.MaxChunksPerQuery, "querier.max-fetched-chunks-per-query", 2e6, "Maximum number of chunks that can be fetched in a single query from ingesters and long-term storage. This limit is enforced in the querier, ruler and store-gateway. 0 to disable.") f.IntVar(&l.MaxFetchedSeriesPerQuery, "querier.max-fetched-series-per-query", 0, "The maximum number of unique series for which a query can fetch samples from each ingesters and storage. This limit is enforced in the querier and ruler. 0 to disable") From 62fd2ef482faa672ce0cd80514e315ce652f8d46 Mon Sep 17 00:00:00 2001 From: Janos Date: Fri, 4 Mar 2022 09:20:53 +0100 Subject: [PATCH 64/91] Adding custom trackers loading metrics and tests for it --- pkg/ingester/ingester.go | 9 +++--- pkg/ingester/ingester_test.go | 60 +++++++++++++++++++++++++++++++++-- pkg/ingester/metrics.go | 11 ++++++- 3 files changed, 73 insertions(+), 7 deletions(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 842a854a59..7e5ac07c8d 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -469,7 +469,6 @@ func (i *Ingester) getActiveSeriesMatchersConfig(userID string) activeseries.Cus } func (i *Ingester) replaceMatchers(asm *activeseries.Matchers, userDB *userTSDB) { - i.metrics.activeSeriesPerUser.DeleteLabelValues(userDB.userID) i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatcherNames()) userDB.activeSeries.ReloadSeriesMatchers(asm) } @@ -485,8 +484,11 @@ func (i *Ingester) updateActiveSeries(now time.Time) { if newMatchersConfig.String() != userDB.activeSeries.CurrentConfig().String() { i.replaceMatchers(activeseries.NewMatchers(newMatchersConfig), userDB) } - if userDB.activeSeries.LastAsmUpdate() < now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout).UnixNano() { - // We are not exposing metrics until enough time passed for stable metrics. + if userDB.activeSeries.LastAsmUpdate() >= now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout).UnixNano() { + // Active series config has been reloaded, exposing loading metric until MetricsIdleTimeout passes + i.metrics.activeSeriesLoading.WithLabelValues(userID).Set(1) + } else { + i.metrics.activeSeriesLoading.DeleteLabelValues(userID) allActive, activeMatching := userDB.activeSeries.Active(now) if allActive > 0 { i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(allActive)) @@ -1575,7 +1577,6 @@ func (i *Ingester) closeAllTSDB() { i.tsdbsMtx.Unlock() i.metrics.memUsers.Dec() - i.metrics.activeSeriesPerUser.DeleteLabelValues(userID) i.metrics.deletePerUserCustomTrackerMetrics(userID, db.activeSeries.CurrentMatcherNames()) }(userDB) } diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index fa66d18d5c..db3f62c391 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5201,7 +5201,7 @@ func TestIngesterActiveSeries(t *testing.T) { # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 @@ -5228,7 +5228,7 @@ func TestIngesterActiveSeries(t *testing.T) { # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge cortex_ingester_active_series{user="other_test_user"} 4 - cortex_ingester_active_series{user="test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 @@ -5371,6 +5371,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { } metricNames := []string{ + "cortex_ingester_active_series_loading", "cortex_ingester_active_series", "cortex_ingester_active_series_custom_tracker", } @@ -5444,6 +5445,9 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. + # TYPE cortex_ingester_active_series_loading gauge + cortex_ingester_active_series_loading{user="test_user"} 1 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5511,6 +5515,9 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 + # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. + # TYPE cortex_ingester_active_series_loading gauge + cortex_ingester_active_series_loading{user="test_user"} 1 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5574,6 +5581,9 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { configReloadTime := time.Now() ingester.updateActiveSeries(configReloadTime) expectedMetrics = ` + # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. + # TYPE cortex_ingester_active_series_loading gauge + cortex_ingester_active_series_loading{user="test_user"} 1 ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) @@ -5599,6 +5609,52 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) }, }, + "should cleanup loading metric at close": { + activeSeriesConfig: activeSeriesDefaultConfig, + tenantLimits: defaultActiveSeriesTenantOverride, + test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { + firstPushTime := time.Now() + + pushWithUser(t, ingester, labelsToPush, userID, req) + pushWithUser(t, ingester, labelsToPush, userID2, req) + + // Update active series for metrics check. + ingester.updateActiveSeries(firstPushTime) + + expectedMetrics := ` + # HELP cortex_ingester_active_series Number of currently active series per user. + # TYPE cortex_ingester_active_series gauge + cortex_ingester_active_series{user="other_test_user"} 4 + cortex_ingester_active_series{user="test_user"} 4 + # HELP cortex_ingester_active_series_custom_tracker Number of currently active series matching a pre-configured label matchers per user. + # TYPE cortex_ingester_active_series_custom_tracker gauge + cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_a",user="test_user"} 2 + cortex_ingester_active_series_custom_tracker{name="team_b",user="test_user"} 2 + ` + // Check tracked Prometheus metrics + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + + // Remove all configs + limits := defaultLimitsTestConfig() + override, err := validation.NewOverrides(limits, nil) + require.NoError(t, err) + ingester.limits = override + ingester.updateActiveSeries(firstPushTime) + expectedMetrics = ` + # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. + # TYPE cortex_ingester_active_series_loading gauge + cortex_ingester_active_series_loading{user="test_user"} 1 + cortex_ingester_active_series_loading{user="other_test_user"} 1 + ` + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + ingester.closeAllTSDB() + expectedMetrics = ` + ` + require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) + }, + }, } for testName, testData := range tests { diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index b75378e1a0..af23e2e10a 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -30,6 +30,7 @@ type ingesterMetrics struct { memMetadataCreatedTotal *prometheus.CounterVec memMetadataRemovedTotal *prometheus.CounterVec + activeSeriesLoading *prometheus.GaugeVec activeSeriesPerUser *prometheus.GaugeVec activeSeriesCustomTrackersPerUser *prometheus.GaugeVec @@ -207,6 +208,12 @@ func newIngesterMetrics( return 0 }), + // Not registered automatically, but only if activeSeriesEnabled is true. + activeSeriesLoading: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "cortex_ingester_active_series_loading", + Help: "Indicating that active series configuration has been reloaded, and waiting to become stable.", + }, []string{"user"}), + // Not registered automatically, but only if activeSeriesEnabled is true. activeSeriesPerUser: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_ingester_active_series", @@ -248,6 +255,7 @@ func newIngesterMetrics( } if activeSeriesEnabled && r != nil { + r.MustRegister(m.activeSeriesLoading) r.MustRegister(m.activeSeriesPerUser) r.MustRegister(m.activeSeriesCustomTrackersPerUser) } @@ -258,10 +266,11 @@ func newIngesterMetrics( func (m *ingesterMetrics) deletePerUserMetrics(userID string) { m.memMetadataCreatedTotal.DeleteLabelValues(userID) m.memMetadataRemovedTotal.DeleteLabelValues(userID) - m.activeSeriesPerUser.DeleteLabelValues(userID) } func (m *ingesterMetrics) deletePerUserCustomTrackerMetrics(userID string, customTrackerMetrics []string) { + m.activeSeriesLoading.DeleteLabelValues(userID) + m.activeSeriesPerUser.DeleteLabelValues(userID) for _, name := range customTrackerMetrics { m.activeSeriesCustomTrackersPerUser.DeleteLabelValues(userID, name) } From 23a8996b8d9fced7461d9f61c3e2dbc633c087a7 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Mar 2022 09:05:23 +0100 Subject: [PATCH 65/91] Adjustments based on Oleg's comments, and introducing single source of time in ingester. --- pkg/ingester/activeseries/active_series.go | 50 +++--- .../activeseries/active_series_test.go | 148 ++++++++++-------- .../activeseries/custom_trackers_config.go | 16 +- .../custom_trackers_config_test.go | 4 +- pkg/ingester/activeseries/matchers_test.go | 2 +- pkg/ingester/ingester.go | 28 ++-- pkg/ingester/ingester_test.go | 133 +++++++++------- pkg/ingester/metrics.go | 2 +- 8 files changed, 208 insertions(+), 175 deletions(-) diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index 3f6c42e08a..cc73083ad4 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -20,17 +20,19 @@ const ( // ActiveSeries is keeping track of recently active series for a single tenant. type ActiveSeries struct { - stripes [numActiveSeriesStripes]activeSeriesStripe - asm *Matchers - lastAsmUpdate *atomic.Int64 + mu sync.RWMutex + stripes [numActiveSeriesStripes]activeSeriesStripe + matchers *Matchers + + lastMatchersUpdate time.Time // The duration after series become inactive. timeout time.Duration - mu sync.RWMutex + now func() time.Time } // activeSeriesStripe holds a subset of the series timestamps for a single tenant. type activeSeriesStripe struct { - asm *Matchers + matchers *Matchers // Unix nanoseconds. Only used by purge. Zero = unknown. // Updated in purge and when old timestamp is used when updating series (in this case, oldestEntryTs is updated @@ -50,13 +52,13 @@ type activeSeriesEntry struct { matches []bool // Which matchers of Matchers does this series match } -func NewActiveSeries(asm *Matchers, idleTimeout time.Duration) *ActiveSeries { - c := &ActiveSeries{asm: asm, timeout: idleTimeout, lastAsmUpdate: atomic.NewInt64(0)} +func NewActiveSeries(asm *Matchers, idleTimeout time.Duration, now func() time.Time) *ActiveSeries { + c := &ActiveSeries{matchers: asm, timeout: idleTimeout, now: now} // Stripes are pre-allocated so that we only read on them and no lock is required. for i := 0; i < numActiveSeriesStripes; i++ { c.stripes[i] = activeSeriesStripe{ - asm: asm, + matchers: asm, refs: map[uint64][]activeSeriesEntry{}, activeMatching: resizeAndClear(len(asm.MatcherNames()), nil), } @@ -68,28 +70,24 @@ func NewActiveSeries(asm *Matchers, idleTimeout time.Duration) *ActiveSeries { func (c *ActiveSeries) CurrentMatcherNames() []string { c.mu.RLock() defer c.mu.RUnlock() - return c.asm.MatcherNames() + return c.matchers.MatcherNames() } -func (c *ActiveSeries) ReloadSeriesMatchers(asm *Matchers) { +func (c *ActiveSeries) ReloadMatchers(asm *Matchers) { c.mu.Lock() defer c.mu.Unlock() for i := 0; i < numActiveSeriesStripes; i++ { c.stripes[i].reinitialize(asm) } - c.asm = asm - c.lastAsmUpdate.Store(time.Now().UnixNano()) -} - -func (c *ActiveSeries) LastAsmUpdate() int64 { - return c.lastAsmUpdate.Load() + c.matchers = asm + c.lastMatchersUpdate = c.now() } func (c *ActiveSeries) CurrentConfig() CustomTrackersConfig { c.mu.RLock() defer c.mu.RUnlock() - return c.asm.Config() + return c.matchers.Config() } // Updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. @@ -115,19 +113,21 @@ func (c *ActiveSeries) clear() { } // Active returns the total number of active series, as well as a slice of active series matching each one of the -// custom trackers provided (in the same order as custom trackers are defined) +// custom trackers provided (in the same order as custom trackers are defined). The third return value shows +// if enough time has passed since last reload to consider the result. // This should be called periodically to avoid memory leaks. -// Active cannot be called concurrently with ReloadSeriesMatchers. -func (c *ActiveSeries) Active(now time.Time) (int, []int) { +// Active cannot be called concurrently with ReloadMatchers. +func (c *ActiveSeries) Active() (int, []int, bool) { c.mu.Lock() defer c.mu.Unlock() - c.purge(now.Add(-c.timeout)) + purgeTime := c.now().Add(-c.timeout) + c.purge(purgeTime) total := 0 - totalMatching := resizeAndClear(len(c.asm.MatcherNames()), nil) + totalMatching := resizeAndClear(len(c.matchers.MatcherNames()), nil) for s := 0; s < numActiveSeriesStripes; s++ { total += c.stripes[s].getTotalAndUpdateMatching(totalMatching) } - return total, totalMatching + return total, totalMatching, purgeTime.After(c.lastMatchersUpdate) } // getTotalAndUpdateMatching will return the total active series in the stripe and also update the slice provided @@ -196,7 +196,7 @@ func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, seri } } - matches := s.asm.Matches(series) + matches := s.matchers.Matches(series) s.active++ for i, ok := range matches { @@ -237,7 +237,7 @@ func (s *activeSeriesStripe) reinitialize(asm *Matchers) { s.oldestEntryTs.Store(0) s.refs = map[uint64][]activeSeriesEntry{} s.active = 0 - s.asm = asm + s.matchers = asm s.activeMatching = resizeAndClear(len(asm.MatcherNames()), s.activeMatching) } diff --git a/pkg/ingester/activeseries/active_series_test.go b/pkg/ingester/activeseries/active_series_test.go index 34711a8fae..408f01f4dd 100644 --- a/pkg/ingester/activeseries/active_series_test.go +++ b/pkg/ingester/activeseries/active_series_test.go @@ -26,22 +26,23 @@ func TestActiveSeries_UpdateSeries_NoMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - currentNow := time.Now() - c := NewActiveSeries(&Matchers{}, 5*time.Minute) - allActive, activeMatching := c.Active(currentNow) + currentNow := time.Now + c := NewActiveSeries(&Matchers{}, 5*time.Minute, currentNow) + allActive, activeMatching, valid := c.Active() assert.Equal(t, 0, allActive) assert.Nil(t, activeMatching) + assert.True(t, valid) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, _ = c.Active(currentNow) + allActive, _, _ = c.Active() assert.Equal(t, 1, allActive) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, _ = c.Active(currentNow) + allActive, _, _ = c.Active() assert.Equal(t, 1, allActive) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, _ = c.Active(currentNow) + allActive, _, _ = c.Active() assert.Equal(t, 2, allActive) } @@ -52,29 +53,29 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~"2|3"}`})) - currentNow := time.Now() - c := NewActiveSeries(asm, 5*time.Minute) - allActive, activeMatching := c.Active(currentNow) + currentNow := time.Now + c := NewActiveSeries(asm, 5*time.Minute, currentNow) + allActive, activeMatching, _ := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching = c.Active(currentNow) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{0}, activeMatching) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, activeMatching = c.Active(currentNow) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 2, allActive) assert.Equal(t, []int{1}, activeMatching) c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching = c.Active(currentNow) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 3, allActive) assert.Equal(t, []int{2}, activeMatching) c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching = c.Active(currentNow) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 3, allActive) assert.Equal(t, []int{2}, activeMatching) } @@ -85,12 +86,12 @@ func TestActiveSeries_ShouldCorrectlyHandleFingerprintCollisions(t *testing.T) { ls2 := metric.Set("_", "KiqbryhzUpn").Labels() require.True(t, client.Fingerprint(ls1) == client.Fingerprint(ls2)) - currentNow := time.Now() - c := NewActiveSeries(&Matchers{}, 5*time.Minute) + currentNow := time.Now + c := NewActiveSeries(&Matchers{}, 5*time.Minute, currentNow) c.UpdateSeries(ls1, time.Now(), copyFn) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, _ := c.Active(currentNow) + allActive, _, _ := c.Active() assert.Equal(t, 2, allActive) } @@ -106,7 +107,10 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl: %d", ttl), func(t *testing.T) { - c := NewActiveSeries(&Matchers{}, time.Since(time.Unix(0, 0))) + currentNow := func() time.Time { + return time.Unix(int64(ttl), 0) + } + c := NewActiveSeries(&Matchers{}, 5*time.Minute, currentNow) for i := 0; i < len(series); i++ { c.UpdateSeries(series[i], time.Unix(int64(i), 0), copyFn) @@ -118,7 +122,7 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { exp := len(series) - (ttl) // c.Active is not intended to purge - allActive, activeMatching := c.Active(time.Unix(0, 0)) + allActive, activeMatching, _ := c.Active() assert.Equal(t, exp, allActive) assert.Nil(t, activeMatching) }) @@ -139,7 +143,10 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl=%d", ttl), func(t *testing.T) { - c := NewActiveSeries(asm, time.Since(time.Unix(0, 0))) + currentNow := func() time.Time { + return time.Unix(int64(ttl), 0) + } + c := NewActiveSeries(asm, 5*time.Minute, currentNow) exp := len(series) - ttl expMatchingSeries := 0 @@ -158,7 +165,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { c.purge(time.Unix(int64(ttl), 0)) // c.Active is not intended to purge - allActive, activeMatching := c.Active(time.Unix(0, 0)) + allActive, activeMatching, _ := c.Active() assert.Equal(t, exp, allActive) assert.Equal(t, []int{expMatchingSeries}, activeMatching) }) @@ -170,25 +177,28 @@ func TestActiveSeries_PurgeOpt(t *testing.T) { ls1 := metric.Set("_", "ypfajYg2lsv").Labels() ls2 := metric.Set("_", "KiqbryhzUpn").Labels() - currentNow := time.Now() - c := NewActiveSeries(&Matchers{}, 59*time.Second) + currentTime := time.Now() + currentNow := func() time.Time { + return currentTime + } + c := NewActiveSeries(&Matchers{}, 59*time.Second, currentNow) - c.UpdateSeries(ls1, currentNow.Add(-2*time.Minute), copyFn) - c.UpdateSeries(ls2, currentNow, copyFn) + c.UpdateSeries(ls1, currentTime.Add(-2*time.Minute), copyFn) + c.UpdateSeries(ls2, currentTime, copyFn) - allActive, _ := c.Active(currentNow) + allActive, _, _ := c.Active() assert.Equal(t, 1, allActive) - c.UpdateSeries(ls1, currentNow.Add(-1*time.Minute), copyFn) - c.UpdateSeries(ls2, currentNow, copyFn) + c.UpdateSeries(ls1, currentTime.Add(-1*time.Minute), copyFn) + c.UpdateSeries(ls2, currentTime, copyFn) - allActive, _ = c.Active(currentNow) + allActive, _, _ = c.Active() assert.Equal(t, 1, allActive) // This will *not* update the series, since there is already newer timestamp. - c.UpdateSeries(ls2, currentNow.Add(-1*time.Minute), copyFn) + c.UpdateSeries(ls2, currentTime.Add(-1*time.Minute), copyFn) - allActive, _ = c.Active(currentNow) + allActive, _, _ = c.Active() assert.Equal(t, 1, allActive) } @@ -200,49 +210,59 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~.*}`})) - currentNow := time.Now() - c := NewActiveSeries(asm, 5*time.Minute) + currentTime := time.Now() + currentNow := func() time.Time { + return currentTime + } + c := NewActiveSeries(asm, 5*time.Minute, currentNow) - allActive, activeMatching := c.Active(time.Now()) + allActive, activeMatching, _ := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) - c.UpdateSeries(ls1, currentNow, copyFn) - allActive, activeMatching = c.Active(time.Now()) + c.UpdateSeries(ls1, currentNow(), copyFn) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{1}, activeMatching) - c.ReloadSeriesMatchers(asm) - assert.Greater(t, c.lastAsmUpdate.Load(), currentNow.UnixNano()) - assert.Less(t, c.lastAsmUpdate.Load(), time.Now().UnixNano()) - allActive, activeMatching = c.Active(time.Now()) + c.ReloadMatchers(asm) + allActive, activeMatching, valid := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) + assert.False(t, valid) - c.UpdateSeries(ls1, currentNow, copyFn) - c.UpdateSeries(ls2, currentNow, copyFn) - allActive, activeMatching = c.Active(time.Now()) + c.UpdateSeries(ls1, currentNow(), copyFn) + c.UpdateSeries(ls2, currentNow(), copyFn) + allActive, activeMatching, valid = c.Active() assert.Equal(t, 2, allActive) assert.Equal(t, []int{2}, activeMatching) + assert.False(t, valid) asmWithLessMatchers := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{})) - c.ReloadSeriesMatchers(asmWithLessMatchers) + c.ReloadMatchers(asmWithLessMatchers) - c.UpdateSeries(ls3, currentNow, copyFn) - allActive, activeMatching = c.Active(time.Now()) + c.UpdateSeries(ls3, currentNow(), copyFn) + allActive, activeMatching, valid = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int(nil), activeMatching) + assert.False(t, valid) asmWithMoreMatchers := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "a": `{a="3"}`, "b": `{a="4"}`, })) - c.ReloadSeriesMatchers(asmWithMoreMatchers) + c.ReloadMatchers(asmWithMoreMatchers) - c.UpdateSeries(ls4, currentNow, copyFn) - allActive, activeMatching = c.Active(time.Now()) + c.UpdateSeries(ls4, currentNow(), copyFn) + allActive, activeMatching, valid = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{0, 1}, activeMatching) + assert.False(t, valid) + + currentTime = currentTime.Add(5 * time.Minute).Add(1 * time.Second) + + _, _, valid = c.Active() + assert.True(t, valid) } func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { @@ -253,13 +273,13 @@ func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { "bar": `{a=~.+}`, })) - c := NewActiveSeries(asm, 5*time.Minute) - allActive, activeMatching := c.Active(time.Now()) + c := NewActiveSeries(asm, 5*time.Minute, time.Now) + allActive, activeMatching, _ := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching = c.Active(time.Now()) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{1, 1}, activeMatching) @@ -267,10 +287,10 @@ func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { "foo": `{a=~.+}`, })) - c.ReloadSeriesMatchers(asm) + c.ReloadMatchers(asm) c.purge(time.Time{}) - allActive, activeMatching = c.Active(time.Now()) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) } @@ -283,13 +303,13 @@ func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { "bar": `{a=~.+}`, })) - c := NewActiveSeries(asm, 5*time.Minute) - allActive, activeMatching := c.Active(time.Now()) + c := NewActiveSeries(asm, 5*time.Minute, time.Now) + allActive, activeMatching, _ := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching = c.Active(time.Now()) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{1, 1}, activeMatching) @@ -298,10 +318,10 @@ func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { "bar": `{b=~.+}`, })) - c.ReloadSeriesMatchers(asm) + c.ReloadMatchers(asm) c.purge(time.Time{}) - allActive, activeMatching = c.Active(time.Now()) + allActive, activeMatching, _ = c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) } @@ -321,7 +341,7 @@ func benchmarkActiveSeriesConcurrencySingleSeries(b *testing.B, goroutines int) {Name: "a", Value: "a"}, } - c := NewActiveSeries(&Matchers{}, 5*time.Minute) + c := NewActiveSeries(&Matchers{}, 5*time.Minute, time.Now) wg := &sync.WaitGroup{} start := make(chan struct{}) @@ -385,7 +405,7 @@ func BenchmarkActiveSeries_UpdateSeries(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - c := NewActiveSeries(&Matchers{}, 5*time.Minute) + c := NewActiveSeries(&Matchers{}, 5*time.Minute, time.Now) for round := 0; round <= tt.nRounds; round++ { for ix := 0; ix < tt.nSeries; ix++ { c.UpdateSeries(series[ix], time.Unix(0, now), copyFn) @@ -410,7 +430,7 @@ func benchmarkPurge(b *testing.B, twice bool) { const numExpiresSeries = numSeries / 25 currentNow := time.Now() - c := NewActiveSeries(&Matchers{}, 5*time.Minute) + c := NewActiveSeries(&Matchers{}, 5*time.Minute, time.Now) series := [numSeries]labels.Labels{} for s := 0; s < numSeries; s++ { @@ -429,19 +449,19 @@ func benchmarkPurge(b *testing.B, twice bool) { } } - allActive, _ := c.Active(time.Now()) + allActive, _, _ := c.Active() assert.Equal(b, numSeries, allActive) b.StartTimer() // Purge everything now := time.Now() c.purge(now) - allActive, _ = c.Active(now) + allActive, _, _ = c.Active() assert.Equal(b, numSeries-numExpiresSeries, allActive) if twice { c.purge(now) - allActive, _ = c.Active(now) + allActive, _, _ = c.Active() assert.Equal(b, numSeries-numExpiresSeries, allActive) } } diff --git a/pkg/ingester/activeseries/custom_trackers_config.go b/pkg/ingester/activeseries/custom_trackers_config.go index de7f30af85..a00821e8d4 100644 --- a/pkg/ingester/activeseries/custom_trackers_config.go +++ b/pkg/ingester/activeseries/custom_trackers_config.go @@ -38,7 +38,7 @@ func (c CustomTrackersConfig) String() string { return c.string } -func activeSeriesCustomTrackersConfigString(cfg map[string]string) string { +func customTrackersConfigString(cfg map[string]string) string { if len(cfg) == 0 { return "" } @@ -70,12 +70,12 @@ func (c *CustomTrackersConfig) Set(s string) error { return nil } - f, err := activeSeriesCustomTrackerFlagValueToMap(s) + f, err := customTrackerFlagValueToMap(s) if err != nil { return err } - nc, err := NewActiveSeriesCustomTrackersConfig(f) + nc, err := NewCustomTrackersConfig(f) if err != nil { return err } @@ -97,11 +97,11 @@ func (c *CustomTrackersConfig) Set(s string) error { } // Recalculate the string after merging. - c.string = activeSeriesCustomTrackersConfigString(c.source) + c.string = customTrackersConfigString(c.source) return nil } -func activeSeriesCustomTrackerFlagValueToMap(s string) (map[string]string, error) { +func customTrackerFlagValueToMap(s string) (map[string]string, error) { source := map[string]string{} pairs := strings.Split(s, ";") for i, p := range pairs { @@ -129,11 +129,11 @@ func (c *CustomTrackersConfig) UnmarshalYAML(unmarshal func(interface{}) error) if err != nil { return err } - *c, err = NewActiveSeriesCustomTrackersConfig(stringMap) + *c, err = NewCustomTrackersConfig(stringMap) return err } -func NewActiveSeriesCustomTrackersConfig(m map[string]string) (c CustomTrackersConfig, err error) { +func NewCustomTrackersConfig(m map[string]string) (c CustomTrackersConfig, err error) { c.source = m c.config = map[string]labelsMatchers{} for name, matcher := range m { @@ -147,6 +147,6 @@ func NewActiveSeriesCustomTrackersConfig(m map[string]string) (c CustomTrackersC } c.config[name] = matchers } - c.string = activeSeriesCustomTrackersConfigString(c.source) + c.string = customTrackersConfigString(c.source) return c, nil } diff --git a/pkg/ingester/activeseries/custom_trackers_config_test.go b/pkg/ingester/activeseries/custom_trackers_config_test.go index 5171467a4a..4c91fa5af9 100644 --- a/pkg/ingester/activeseries/custom_trackers_config_test.go +++ b/pkg/ingester/activeseries/custom_trackers_config_test.go @@ -13,7 +13,7 @@ import ( ) func mustNewCustomTrackersConfigFromMap(t *testing.T, source map[string]string) CustomTrackersConfig { - m, err := NewActiveSeriesCustomTrackersConfig(source) + m, err := NewCustomTrackersConfig(source) require.NoError(t, err) return m } @@ -191,7 +191,7 @@ func TestTrackersConfigs_Deserialization(t *testing.T) { config := CustomTrackersConfig{} err := yaml.Unmarshal([]byte(correctInput), &config) assert.NoError(t, err, "failed do deserialize Matchers") - expectedConfig, err := NewActiveSeriesCustomTrackersConfig(map[string]string{ + expectedConfig, err := NewCustomTrackersConfig(map[string]string{ "baz": "{baz='bar'}", "foo": "{foo='bar'}", }) diff --git a/pkg/ingester/activeseries/matchers_test.go b/pkg/ingester/activeseries/matchers_test.go index e3dfd4f7d6..6100ce83bc 100644 --- a/pkg/ingester/activeseries/matchers_test.go +++ b/pkg/ingester/activeseries/matchers_test.go @@ -96,7 +96,7 @@ func TestActiveSeriesCustomTrackersConfigs_MalformedMatcher(t *testing.T) { "malformed": matcher, } - _, err := NewActiveSeriesCustomTrackersConfig(config) + _, err := NewCustomTrackersConfig(config) assert.Error(t, err) }) } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 7e5ac07c8d..58146b689b 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -233,6 +233,9 @@ type Ingester struct { // Rate of pushed samples. Used to limit global samples push rate. ingestionRate *util_math.EwmaRate inflightPushRequests atomic.Int64 + + // Single source of time for making ingester testable. + now func() time.Time } func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { @@ -257,6 +260,7 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus forceCompactTrigger: make(chan requestWithUsersAndCallback), shipTrigger: make(chan requestWithUsersAndCallback), seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), + now: time.Now, }, nil } @@ -454,7 +458,7 @@ func (i *Ingester) updateLoop(ctx context.Context) error { i.applyExemplarsSettings() case <-activeSeriesTickerChan: - i.updateActiveSeries(time.Now()) + i.updateActiveSeries() case <-ctx.Done(): return nil @@ -464,32 +468,28 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) getActiveSeriesMatchersConfig(userID string) activeseries.CustomTrackersConfig { - return i.limits.ActiveSeriesCustomTrackersConfig(userID) -} - func (i *Ingester) replaceMatchers(asm *activeseries.Matchers, userDB *userTSDB) { i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatcherNames()) - userDB.activeSeries.ReloadSeriesMatchers(asm) + userDB.activeSeries.ReloadMatchers(asm) } -func (i *Ingester) updateActiveSeries(now time.Time) { +func (i *Ingester) updateActiveSeries() { for _, userID := range i.getTSDBUsers() { userDB := i.getTSDB(userID) if userDB == nil { continue } - newMatchersConfig := i.getActiveSeriesMatchersConfig(userID) + newMatchersConfig := i.limits.ActiveSeriesCustomTrackersConfig(userID) if newMatchersConfig.String() != userDB.activeSeries.CurrentConfig().String() { i.replaceMatchers(activeseries.NewMatchers(newMatchersConfig), userDB) } - if userDB.activeSeries.LastAsmUpdate() >= now.Add(-i.cfg.ActiveSeriesMetricsIdleTimeout).UnixNano() { - // Active series config has been reloaded, exposing loading metric until MetricsIdleTimeout passes + allActive, activeMatching, valid := userDB.activeSeries.Active() + if !valid { + // Active series config has been reloaded, exposing loading metric until MetricsIdleTimeout passes. i.metrics.activeSeriesLoading.WithLabelValues(userID).Set(1) } else { i.metrics.activeSeriesLoading.DeleteLabelValues(userID) - allActive, activeMatching := userDB.activeSeries.Active(now) if allActive > 0 { i.metrics.activeSeriesPerUser.WithLabelValues(userID).Set(float64(allActive)) } else { @@ -609,7 +609,7 @@ func (i *Ingester) PushWithCleanup(ctx context.Context, req *mimirpb.WriteReques failedSamplesCount = 0 succeededExemplarsCount = 0 failedExemplarsCount = 0 - startAppend = time.Now() + startAppend = i.now() sampleOutOfBoundsCount = 0 sampleOutOfOrderCount = 0 newValueForTimestampCount = 0 @@ -1451,11 +1451,11 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userLogger := util_log.WithUserID(userID, i.logger) blockRanges := i.cfg.BlocksStorageConfig.TSDB.BlockRanges.ToMilliseconds() - matchersConfig := i.getActiveSeriesMatchersConfig(userID) + matchersConfig := i.limits.ActiveSeriesCustomTrackersConfig(userID) userDB := &userTSDB{ userID: userID, - activeSeries: activeseries.NewActiveSeries(activeseries.NewMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout), + activeSeries: activeseries.NewActiveSeries(activeseries.NewMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout, i.now), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index db3f62c391..875370c186 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -65,7 +65,7 @@ import ( ) func mustNewActiveSeriesCustomTrackersConfigFromMap(t *testing.T, source map[string]string) activeseries.CustomTrackersConfig { - m, err := activeseries.NewActiveSeriesCustomTrackersConfig(source) + m, err := activeseries.NewCustomTrackersConfig(source) require.NoError(t, err) return m } @@ -696,7 +696,7 @@ func TestIngester_Push(t *testing.T) { // Update active series for metrics check. if !testData.disableActiveSeries { - i.updateActiveSeries(time.Now()) + i.updateActiveSeries() } // Append additional metrics to assert on. @@ -765,7 +765,7 @@ func TestIngester_Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *testi } // Update active series for metrics check. - i.updateActiveSeries(time.Now()) + i.updateActiveSeries() // Check tracked Prometheus metrics expectedMetrics := ` @@ -815,6 +815,11 @@ func TestIngester_Push_DecreaseInactiveSeries(t *testing.T) { cfg.IngesterRing.JoinAfter = 0 i, err := prepareIngesterWithBlocksStorage(t, cfg, registry) + currentTime := time.Now() + currentNow := func() time.Time { + return currentTime + } + i.now = currentNow require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -852,7 +857,8 @@ func TestIngester_Push_DecreaseInactiveSeries(t *testing.T) { // Update active series the after the idle timeout (in the future). // This will remove inactive series. - i.updateActiveSeries(time.Now().Add(cfg.ActiveSeriesMetricsIdleTimeout)) + currentTime = currentTime.Add(cfg.ActiveSeriesMetricsIdleTimeout + 1*time.Second) + i.updateActiveSeries() // Check tracked Prometheus metrics expectedMetrics := ` @@ -3865,7 +3871,7 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { }) pushSingleSampleWithMetadata(t, i) - i.updateActiveSeries(time.Now()) + i.updateActiveSeries() require.Equal(t, int64(1), i.seriesCount.Load()) @@ -3906,7 +3912,7 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { }) require.Greater(t, testutil.ToFloat64(i.metrics.idleTsdbChecks.WithLabelValues(string(tsdbIdleClosed))), float64(0)) - i.updateActiveSeries(time.Now()) + i.updateActiveSeries() require.Equal(t, int64(0), i.seriesCount.Load()) // Flushing removed all series from memory. // Verify that user has disappeared from metrics. @@ -3931,7 +3937,7 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { // Pushing another sample will recreate TSDB. pushSingleSampleWithMetadata(t, i) - i.updateActiveSeries(time.Now()) + i.updateActiveSeries() // User is back. require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` @@ -5189,13 +5195,11 @@ func TestIngesterActiveSeries(t *testing.T) { }{ "successful push, should count active series": { test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - now := time.Now() - pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries(now) + ingester.updateActiveSeries() expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5216,13 +5220,11 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should cleanup metrics when tsdb closed": { test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - now := time.Now() - pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries(now) + ingester.updateActiveSeries() expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5247,13 +5249,17 @@ func TestIngesterActiveSeries(t *testing.T) { }, "should track custom matchers, removing when zero": { test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - firstPushTime := time.Now() + currentTime := time.Now() + currentNow := func() time.Time { + return currentTime + } + ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries(firstPushTime) + ingester.updateActiveSeries() expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5271,20 +5277,15 @@ func TestIngesterActiveSeries(t *testing.T) { // Check tracked Prometheus metrics require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // Sleep for one millisecond, this will make the append time of first push smaller than - // secondPushTime. This is something required to make the test deterministic - // (otherwise it _could_ be the same nanosecond theoretically, although unlikely in practice) - time.Sleep(time.Millisecond) - secondPushTime := time.Now() - // Sleep another millisecond to make sure that secondPushTime is strictly less than the append time of the second push. - time.Sleep(time.Millisecond) - + // Pushing second time to have entires which are not going to be purged + currentTime = currentTime.Add(10 * time.Second) pushWithUser(t, ingester, labelsToPush, userID, req) - // Update active series for metrics check in the future. + // Adding time to make the first batch of pushes idle. // We update them in the exact moment in time where append time of the first push is already considered idle, // while the second append happens after the purge timestamp. - ingester.updateActiveSeries(secondPushTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout - 5*time.Second) + ingester.updateActiveSeries() expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5300,20 +5301,19 @@ func TestIngesterActiveSeries(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) // Update active series again in a further future where no series are active anymore. - ingester.updateActiveSeries(time.Now().Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout) + ingester.updateActiveSeries() require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(""), metricNames...)) }, }, "successful push, active series disabled": { disableActiveSeries: true, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - now := time.Now() - pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries(now) + ingester.updateActiveSeries() expectedMetrics := `` @@ -5403,13 +5403,17 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { tenantLimits: nil, activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - firstPushTime := time.Now() + currentTime := time.Now() + currentNow := func() time.Time { + return currentTime + } + ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries(firstPushTime) + ingester.updateActiveSeries() expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5436,7 +5440,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { require.NoError(t, err) ingester.limits = override // First update reloads the config - ingester.updateActiveSeries(firstPushTime) + ingester.updateActiveSeries() expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5451,13 +5455,13 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - secondPushtime := time.Now() - // Sleep here to ensure that the second batch of push happens after purgeTime to have entries for custom trackers - time.Sleep(time.Millisecond) + // Adding time to second push to avoid purging it before exposing. + currentTime = currentTime.Add(10 * time.Second) pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) - // Adding idleTimeout to expose the metrics - ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) + // Adding idleTimeout - 1 to expose the metrics but not purge the pushes. + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout - time.Second) + ingester.updateActiveSeries() expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5477,13 +5481,17 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { activeSeriesConfig: activeSeriesDefaultConfig, tenantLimits: defaultActiveSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - firstPushTime := time.Now() + currentTime := time.Now() + currentNow := func() time.Time { + return currentTime + } + ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries(firstPushTime) + ingester.updateActiveSeries() expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5506,7 +5514,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { override, err := validation.NewOverrides(limits, nil) require.NoError(t, err) ingester.limits = override - ingester.updateActiveSeries(firstPushTime) + ingester.updateActiveSeries() expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5521,12 +5529,13 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - secondPushtime := time.Now() - // Sleep here to ensure that the second batch of push happens after purgeTime to have entries for custom trackers - time.Sleep(time.Millisecond) + // Adding time to second push to avoid purging it before exposing. + currentTime = currentTime.Add(10 * time.Second) pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) - ingester.updateActiveSeries(secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout)) + // Adding idleTimeout - 1 to expose the metrics but not purge the pushes. + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout - time.Second) + ingester.updateActiveSeries() expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5545,12 +5554,16 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { "changing runtime override should result in new metrics": { activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - firstPushTime := time.Now() + currentTime := time.Now() + currentNow := func() time.Time { + return currentTime + } + ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) // Update active series for metrics check. - ingester.updateActiveSeries(firstPushTime) + ingester.updateActiveSeries() expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5577,9 +5590,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { override, err := validation.NewOverrides(limits, activeSeriesTenantOverride) require.NoError(t, err) ingester.limits = override - // This will update the runtime config. - configReloadTime := time.Now() - ingester.updateActiveSeries(configReloadTime) + ingester.updateActiveSeries() expectedMetrics = ` # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. # TYPE cortex_ingester_active_series_loading gauge @@ -5587,14 +5598,12 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - secondPushtime := time.Now() - for _, label := range labelsToPush { - ctx := user.InjectOrgID(context.Background(), userID) - _, err := ingester.Push(ctx, req(label, secondPushtime)) - require.NoError(t, err) - } - updateTime := secondPushtime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout) - ingester.updateActiveSeries(updateTime) + // Adding time to second push to avoid purging it before exposing. + currentTime = currentTime.Add(10 * time.Second) + pushWithUser(t, ingester, labelsToPush, userID, req) + // Adding idleTimeout - 1 to expose the metrics but not purge the pushes. + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout - time.Second) + ingester.updateActiveSeries() expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5613,13 +5622,17 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { activeSeriesConfig: activeSeriesDefaultConfig, tenantLimits: defaultActiveSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { - firstPushTime := time.Now() + currentTime := time.Now() + currentNow := func() time.Time { + return currentTime + } + ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries(firstPushTime) + ingester.updateActiveSeries() expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5641,7 +5654,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { override, err := validation.NewOverrides(limits, nil) require.NoError(t, err) ingester.limits = override - ingester.updateActiveSeries(firstPushTime) + ingester.updateActiveSeries() expectedMetrics = ` # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. # TYPE cortex_ingester_active_series_loading gauge diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index af23e2e10a..59448af35e 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -211,7 +211,7 @@ func newIngesterMetrics( // Not registered automatically, but only if activeSeriesEnabled is true. activeSeriesLoading: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: "cortex_ingester_active_series_loading", - Help: "Indicating that active series configuration has been reloaded, and waiting to become stable.", + Help: "Indicates that active series configuration is being reloaded, and waiting to become stable. While this metric is non zero, values from active series metrics shouldn't be considered.", }, []string{"user"}), // Not registered automatically, but only if activeSeriesEnabled is true. From d247dec11d2f855abe023713c121b34c67872d93 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Mar 2022 12:57:19 +0100 Subject: [PATCH 66/91] Adding back ActiveSeriesCustomTrackersConfig to ingester for backward compatibility, with deprecation note --- .../reference-configuration-parameters.md | 15 +++++++++++++++ pkg/ingester/ingester.go | 7 ++++--- pkg/mimir/modules.go | 13 +++++++++++++ 3 files changed, 32 insertions(+), 3 deletions(-) diff --git a/docs/sources/configuration/reference-configuration-parameters.md b/docs/sources/configuration/reference-configuration-parameters.md index fe7d8120ec..436312bbcf 100644 --- a/docs/sources/configuration/reference-configuration-parameters.md +++ b/docs/sources/configuration/reference-configuration-parameters.md @@ -794,6 +794,21 @@ ring: # CLI flag: -ingester.active-series-metrics-idle-timeout [active_series_metrics_idle_timeout: | default = 10m] +# (advanced) [Deprecated] This config has been moved to the limits config, +# please set it there. Additional custom trackers for active metrics. If there +# are active series matching a provided matcher (map value), the count will be +# exposed in the custom trackers metric labeled using the tracker name (map +# key). Zero valued counts are not exposed (and removed when they go back to +# zero). +# Example: +# The following configuration will count the active series coming from dev and +# prod namespaces for each tenant and label them as {name="dev"} and +# {name="prod"} in the cortex_ingester_active_series_custom_tracker metric. +# active_series_custom_trackers: +# dev: '{namespace=~"dev-.*"}' +# prod: '{namespace=~"prod-.*"}' +[active_series_custom_trackers: | default = ] + # (experimental) Period with which to update per-tenant max exemplar limit. # CLI flag: -ingester.exemplars-update-period [exemplars_update_period: | default = 15s] diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 58146b689b..455be82da1 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -122,9 +122,10 @@ type Config struct { RateUpdatePeriod time.Duration `yaml:"rate_update_period" category:"advanced"` - ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` - ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` - ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesMetricsEnabled bool `yaml:"active_series_metrics_enabled" category:"advanced"` + ActiveSeriesMetricsUpdatePeriod time.Duration `yaml:"active_series_metrics_update_period" category:"advanced"` + ActiveSeriesMetricsIdleTimeout time.Duration `yaml:"active_series_metrics_idle_timeout" category:"advanced"` + ActiveSeriesCustomTrackers activeseries.CustomTrackersConfig `yaml:"active_series_custom_trackers" doc:"description=[Deprecated] This config has been moved to the limits config, please set it there. Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero)." category:"advanced"` ExemplarsUpdatePeriod time.Duration `yaml:"exemplars_update_period" category:"experimental"` diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 67b2061871..37d1a64ddc 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -9,6 +9,7 @@ import ( "context" "flag" "fmt" + "github.com/grafana/dskit/flagext" "net/http" "strconv" "time" @@ -233,6 +234,18 @@ func (t *Mimir) initRuntimeConfig() (services.Service, error) { } func (t *Mimir) initOverrides() (serv services.Service, err error) { + // TODO Remove in Mimir 2.2. + // Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. + // We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, + // unless it's also defined in the limits, which is invalid. + if !t.Cfg.Ingester.ActiveSeriesCustomTrackers.Empty() { + if !t.Cfg.LimitsConfig.ActiveSeriesCustomTrackersConfig.Empty() { + return nil, fmt.Errorf("can't define active series custom trackers in both ingester and limits config, please define them only in the limits") + } + level.Warn(util_log.Logger).Log("msg", "active_series_custom_trackers is defined as an ingester config, this location is deprecated, please move it to the limits config") + flagext.DeprecatedFlagsUsed.Inc() + t.Cfg.LimitsConfig.ActiveSeriesCustomTrackersConfig = t.Cfg.Ingester.ActiveSeriesCustomTrackers + } t.Overrides, err = validation.NewOverrides(t.Cfg.LimitsConfig, t.TenantLimits) // overrides don't have operational state, nor do they need to do anything more in starting/stopping phase, // so there is no need to return any service. From e551d87dd53979632de0510e70265fa74962528c Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Mar 2022 13:08:43 +0100 Subject: [PATCH 67/91] lintfix --- pkg/mimir/modules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 39e5b356a3..11af7d6089 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -9,12 +9,12 @@ import ( "context" "flag" "fmt" - "github.com/grafana/dskit/flagext" "net/http" "strconv" "time" "github.com/go-kit/log/level" + "github.com/grafana/dskit/flagext" "github.com/grafana/dskit/kv/codec" "github.com/grafana/dskit/kv/memberlist" "github.com/grafana/dskit/modules" From 21e7671ae89357c248f62b11a826d58bf5eca742 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 16 Mar 2022 13:37:30 +0100 Subject: [PATCH 68/91] Test fix --- pkg/ingester/ingester_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 875370c186..5e197c92ad 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -5449,7 +5449,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 - # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. + # HELP cortex_ingester_active_series_loading Indicates that active series configuration is being reloaded, and waiting to become stable. While this metric is non zero, values from active series metrics shouldn't be considered. # TYPE cortex_ingester_active_series_loading gauge cortex_ingester_active_series_loading{user="test_user"} 1 ` @@ -5523,7 +5523,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { # TYPE cortex_ingester_active_series_custom_tracker gauge cortex_ingester_active_series_custom_tracker{name="bool_is_true_flagbased",user="other_test_user"} 2 cortex_ingester_active_series_custom_tracker{name="bool_is_false_flagbased",user="other_test_user"} 2 - # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. + # HELP cortex_ingester_active_series_loading Indicates that active series configuration is being reloaded, and waiting to become stable. While this metric is non zero, values from active series metrics shouldn't be considered. # TYPE cortex_ingester_active_series_loading gauge cortex_ingester_active_series_loading{user="test_user"} 1 ` @@ -5592,7 +5592,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ingester.limits = override ingester.updateActiveSeries() expectedMetrics = ` - # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. + # HELP cortex_ingester_active_series_loading Indicates that active series configuration is being reloaded, and waiting to become stable. While this metric is non zero, values from active series metrics shouldn't be considered. # TYPE cortex_ingester_active_series_loading gauge cortex_ingester_active_series_loading{user="test_user"} 1 ` @@ -5656,7 +5656,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ingester.limits = override ingester.updateActiveSeries() expectedMetrics = ` - # HELP cortex_ingester_active_series_loading Indicating that active series configuration has been reloaded, and waiting to become stable. + # HELP cortex_ingester_active_series_loading Indicates that active series configuration is being reloaded, and waiting to become stable. While this metric is non zero, values from active series metrics shouldn't be considered. # TYPE cortex_ingester_active_series_loading gauge cortex_ingester_active_series_loading{user="test_user"} 1 cortex_ingester_active_series_loading{user="other_test_user"} 1 From 34ac0a5aec196c18402cb64c2c41c81962dcf226 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 17 Mar 2022 08:46:51 +0100 Subject: [PATCH 69/91] Rework based on Oleg's suggestions --- pkg/ingester/activeseries/active_series.go | 76 ++++---- .../activeseries/active_series_test.go | 174 +++++++++++------- pkg/ingester/activeseries/matchers_test.go | 4 +- 3 files changed, 148 insertions(+), 106 deletions(-) diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index cc73083ad4..2c7ba7a8e5 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -15,23 +15,23 @@ import ( ) const ( - numActiveSeriesStripes = 512 + numStripes = 512 ) // ActiveSeries is keeping track of recently active series for a single tenant. type ActiveSeries struct { - mu sync.RWMutex - stripes [numActiveSeriesStripes]activeSeriesStripe - matchers *Matchers - + mu sync.RWMutex + stripes [numStripes]seriesStripe + matchers *Matchers lastMatchersUpdate time.Time + // The duration after series become inactive. timeout time.Duration now func() time.Time } -// activeSeriesStripe holds a subset of the series timestamps for a single tenant. -type activeSeriesStripe struct { +// seriesStripe holds a subset of the series timestamps for a single tenant. +type seriesStripe struct { matchers *Matchers // Unix nanoseconds. Only used by purge. Zero = unknown. @@ -40,26 +40,26 @@ type activeSeriesStripe struct { oldestEntryTs atomic.Int64 mu sync.RWMutex - refs map[uint64][]activeSeriesEntry + refs map[uint64][]seriesEntry active int // Number of active entries in this stripe. Only decreased during purge or clear. activeMatching []int // Number of active entries in this stripe matching each matcher of the configured Matchers. } -// activeSeriesEntry holds a timestamp for single series. -type activeSeriesEntry struct { +// seriesEntry holds a timestamp for single series. +type seriesEntry struct { lbs labels.Labels nanos *atomic.Int64 // Unix timestamp in nanoseconds. Needs to be a pointer because we don't store pointers to entries in the stripe. matches []bool // Which matchers of Matchers does this series match } -func NewActiveSeries(asm *Matchers, idleTimeout time.Duration, now func() time.Time) *ActiveSeries { - c := &ActiveSeries{matchers: asm, timeout: idleTimeout, now: now} +func NewActiveSeries(asm *Matchers, timeout time.Duration, now func() time.Time) *ActiveSeries { + c := &ActiveSeries{matchers: asm, timeout: timeout, now: now} // Stripes are pre-allocated so that we only read on them and no lock is required. - for i := 0; i < numActiveSeriesStripes; i++ { - c.stripes[i] = activeSeriesStripe{ + for i := 0; i < numStripes; i++ { + c.stripes[i] = seriesStripe{ matchers: asm, - refs: map[uint64][]activeSeriesEntry{}, + refs: map[uint64][]seriesEntry{}, activeMatching: resizeAndClear(len(asm.MatcherNames()), nil), } } @@ -77,7 +77,7 @@ func (c *ActiveSeries) ReloadMatchers(asm *Matchers) { c.mu.Lock() defer c.mu.Unlock() - for i := 0; i < numActiveSeriesStripes; i++ { + for i := 0; i < numStripes; i++ { c.stripes[i].reinitialize(asm) } c.matchers = asm @@ -93,28 +93,28 @@ func (c *ActiveSeries) CurrentConfig() CustomTrackersConfig { // Updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. func (c *ActiveSeries) UpdateSeries(series labels.Labels, now time.Time, labelsCopy func(labels.Labels) labels.Labels) { fp := series.Hash() - stripeID := fp % numActiveSeriesStripes + stripeID := fp % numStripes c.stripes[stripeID].updateSeriesTimestamp(now, series, fp, labelsCopy) } -// Purge removes expired entries from the cache. This function is called by Active. +// purge removes expired entries from the cache. func (c *ActiveSeries) purge(keepUntil time.Time) { - for s := 0; s < numActiveSeriesStripes; s++ { + for s := 0; s < numStripes; s++ { c.stripes[s].purge(keepUntil) } } //nolint // Linter reports that this method is unused, but it is. func (c *ActiveSeries) clear() { - for s := 0; s < numActiveSeriesStripes; s++ { + for s := 0; s < numStripes; s++ { c.stripes[s].clear() } } // Active returns the total number of active series, as well as a slice of active series matching each one of the -// custom trackers provided (in the same order as custom trackers are defined). The third return value shows -// if enough time has passed since last reload to consider the result. +// custom trackers provided (in the same order as custom trackers are defined). +// The result is correct only if the third return value is true, which shows if enough time has passed since last reload. // This should be called periodically to avoid memory leaks. // Active cannot be called concurrently with ReloadMatchers. func (c *ActiveSeries) Active() (int, []int, bool) { @@ -122,17 +122,23 @@ func (c *ActiveSeries) Active() (int, []int, bool) { defer c.mu.Unlock() purgeTime := c.now().Add(-c.timeout) c.purge(purgeTime) + + if c.lastMatchersUpdate.After(purgeTime) { + return 0, nil, false + } + total := 0 totalMatching := resizeAndClear(len(c.matchers.MatcherNames()), nil) - for s := 0; s < numActiveSeriesStripes; s++ { + for s := 0; s < numStripes; s++ { total += c.stripes[s].getTotalAndUpdateMatching(totalMatching) } - return total, totalMatching, purgeTime.After(c.lastMatchersUpdate) + + return total, totalMatching, true } // getTotalAndUpdateMatching will return the total active series in the stripe and also update the slice provided // with each matcher's total. -func (s *activeSeriesStripe) getTotalAndUpdateMatching(matching []int) int { +func (s *seriesStripe) getTotalAndUpdateMatching(matching []int) int { s.mu.RLock() defer s.mu.RUnlock() @@ -144,7 +150,7 @@ func (s *activeSeriesStripe) getTotalAndUpdateMatching(matching []int) int { return s.active } -func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels.Labels, fingerprint uint64, labelsCopy func(labels.Labels) labels.Labels) { +func (s *seriesStripe) updateSeriesTimestamp(now time.Time, series labels.Labels, fingerprint uint64, labelsCopy func(labels.Labels) labels.Labels) { nowNanos := now.UnixNano() e := s.findEntryForSeries(fingerprint, series) @@ -170,7 +176,7 @@ func (s *activeSeriesStripe) updateSeriesTimestamp(now time.Time, series labels. } } -func (s *activeSeriesStripe) findEntryForSeries(fingerprint uint64, series labels.Labels) *atomic.Int64 { +func (s *seriesStripe) findEntryForSeries(fingerprint uint64, series labels.Labels) *atomic.Int64 { s.mu.RLock() defer s.mu.RUnlock() @@ -184,7 +190,7 @@ func (s *activeSeriesStripe) findEntryForSeries(fingerprint uint64, series label return nil } -func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, series labels.Labels, nowNanos int64, labelsCopy func(labels.Labels) labels.Labels) (*atomic.Int64, bool) { +func (s *seriesStripe) findOrCreateEntryForSeries(fingerprint uint64, series labels.Labels, nowNanos int64, labelsCopy func(labels.Labels) labels.Labels) (*atomic.Int64, bool) { s.mu.Lock() defer s.mu.Unlock() @@ -205,7 +211,7 @@ func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, seri } } - e := activeSeriesEntry{ + e := seriesEntry{ lbs: labelsCopy(series), nanos: atomic.NewInt64(nowNanos), matches: matches, @@ -217,31 +223,31 @@ func (s *activeSeriesStripe) findOrCreateEntryForSeries(fingerprint uint64, seri } //nolint // Linter reports that this method is unused, but it is. -func (s *activeSeriesStripe) clear() { +func (s *seriesStripe) clear() { s.mu.Lock() defer s.mu.Unlock() s.oldestEntryTs.Store(0) - s.refs = map[uint64][]activeSeriesEntry{} + s.refs = map[uint64][]seriesEntry{} s.active = 0 for i := range s.activeMatching { s.activeMatching[i] = 0 } } -// Reinitialize is more than clear that it assigns new matchers and corresponding size activeMatching slices. -func (s *activeSeriesStripe) reinitialize(asm *Matchers) { +// Reinitialize assigns new matchers and corresponding size activeMatching slices. +func (s *seriesStripe) reinitialize(asm *Matchers) { s.mu.Lock() defer s.mu.Unlock() s.oldestEntryTs.Store(0) - s.refs = map[uint64][]activeSeriesEntry{} + s.refs = map[uint64][]seriesEntry{} s.active = 0 s.matchers = asm s.activeMatching = resizeAndClear(len(asm.MatcherNames()), s.activeMatching) } -func (s *activeSeriesStripe) purge(keepUntil time.Time) { +func (s *seriesStripe) purge(keepUntil time.Time) { keepUntilNanos := keepUntil.UnixNano() if oldest := s.oldestEntryTs.Load(); oldest > 0 && keepUntilNanos <= oldest { // Nothing to do. diff --git a/pkg/ingester/activeseries/active_series_test.go b/pkg/ingester/activeseries/active_series_test.go index 408f01f4dd..345e020f80 100644 --- a/pkg/ingester/activeseries/active_series_test.go +++ b/pkg/ingester/activeseries/active_series_test.go @@ -22,28 +22,32 @@ import ( func copyFn(l labels.Labels) labels.Labels { return l } +const DefaultTimeout = 5 * time.Minute + func TestActiveSeries_UpdateSeries_NoMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - currentNow := time.Now - c := NewActiveSeries(&Matchers{}, 5*time.Minute, currentNow) + c := NewActiveSeries(&Matchers{}, DefaultTimeout, time.Now) allActive, activeMatching, valid := c.Active() assert.Equal(t, 0, allActive) assert.Nil(t, activeMatching) assert.True(t, valid) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, _, _ = c.Active() + allActive, _, valid = c.Active() assert.Equal(t, 1, allActive) + assert.True(t, valid) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, _, _ = c.Active() + allActive, _, valid = c.Active() assert.Equal(t, 1, allActive) + assert.True(t, valid) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, _, _ = c.Active() + allActive, _, valid = c.Active() assert.Equal(t, 2, allActive) + assert.True(t, valid) } func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { @@ -53,31 +57,35 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~"2|3"}`})) - currentNow := time.Now - c := NewActiveSeries(asm, 5*time.Minute, currentNow) - allActive, activeMatching, _ := c.Active() + c := NewActiveSeries(asm, DefaultTimeout, time.Now) + allActive, activeMatching, valid := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) + assert.True(t, valid) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching, _ = c.Active() + allActive, activeMatching, valid = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{0}, activeMatching) + assert.True(t, valid) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, activeMatching, _ = c.Active() + allActive, activeMatching, valid = c.Active() assert.Equal(t, 2, allActive) assert.Equal(t, []int{1}, activeMatching) + assert.True(t, valid) c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching, _ = c.Active() + allActive, activeMatching, valid = c.Active() assert.Equal(t, 3, allActive) assert.Equal(t, []int{2}, activeMatching) + assert.True(t, valid) c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching, _ = c.Active() + allActive, activeMatching, valid = c.Active() assert.Equal(t, 3, allActive) assert.Equal(t, []int{2}, activeMatching) + assert.True(t, valid) } func TestActiveSeries_ShouldCorrectlyHandleFingerprintCollisions(t *testing.T) { @@ -86,13 +94,13 @@ func TestActiveSeries_ShouldCorrectlyHandleFingerprintCollisions(t *testing.T) { ls2 := metric.Set("_", "KiqbryhzUpn").Labels() require.True(t, client.Fingerprint(ls1) == client.Fingerprint(ls2)) - currentNow := time.Now - c := NewActiveSeries(&Matchers{}, 5*time.Minute, currentNow) + c := NewActiveSeries(&Matchers{}, DefaultTimeout, time.Now) c.UpdateSeries(ls1, time.Now(), copyFn) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, _, _ := c.Active() + allActive, _, valid := c.Active() assert.Equal(t, 2, allActive) + assert.True(t, valid) } func TestActiveSeries_Purge_NoMatchers(t *testing.T) { @@ -107,10 +115,10 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl: %d", ttl), func(t *testing.T) { - currentNow := func() time.Time { + mockedNow := func() time.Time { return time.Unix(int64(ttl), 0) } - c := NewActiveSeries(&Matchers{}, 5*time.Minute, currentNow) + c := NewActiveSeries(&Matchers{}, DefaultTimeout, mockedNow) for i := 0; i < len(series); i++ { c.UpdateSeries(series[i], time.Unix(int64(i), 0), copyFn) @@ -122,9 +130,10 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { exp := len(series) - (ttl) // c.Active is not intended to purge - allActive, activeMatching, _ := c.Active() + allActive, activeMatching, valid := c.Active() assert.Equal(t, exp, allActive) assert.Nil(t, activeMatching) + assert.True(t, valid) }) } } @@ -143,10 +152,10 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl=%d", ttl), func(t *testing.T) { - currentNow := func() time.Time { + mockedNow := func() time.Time { return time.Unix(int64(ttl), 0) } - c := NewActiveSeries(asm, 5*time.Minute, currentNow) + c := NewActiveSeries(asm, 5*time.Minute, mockedNow) exp := len(series) - ttl expMatchingSeries := 0 @@ -165,9 +174,10 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { c.purge(time.Unix(int64(ttl), 0)) // c.Active is not intended to purge - allActive, activeMatching, _ := c.Active() + allActive, activeMatching, valid := c.Active() assert.Equal(t, exp, allActive) assert.Equal(t, []int{expMatchingSeries}, activeMatching) + assert.True(t, valid) }) } } @@ -178,28 +188,31 @@ func TestActiveSeries_PurgeOpt(t *testing.T) { ls2 := metric.Set("_", "KiqbryhzUpn").Labels() currentTime := time.Now() - currentNow := func() time.Time { + mockedNow := func() time.Time { return currentTime } - c := NewActiveSeries(&Matchers{}, 59*time.Second, currentNow) + c := NewActiveSeries(&Matchers{}, 59*time.Second, mockedNow) c.UpdateSeries(ls1, currentTime.Add(-2*time.Minute), copyFn) c.UpdateSeries(ls2, currentTime, copyFn) - allActive, _, _ := c.Active() + allActive, _, valid := c.Active() assert.Equal(t, 1, allActive) + assert.True(t, valid) c.UpdateSeries(ls1, currentTime.Add(-1*time.Minute), copyFn) c.UpdateSeries(ls2, currentTime, copyFn) - allActive, _, _ = c.Active() + allActive, _, valid = c.Active() assert.Equal(t, 1, allActive) + assert.True(t, valid) // This will *not* update the series, since there is already newer timestamp. c.UpdateSeries(ls2, currentTime.Add(-1*time.Minute), copyFn) - allActive, _, _ = c.Active() + allActive, _, valid = c.Active() assert.Equal(t, 1, allActive) + assert.True(t, valid) } func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { @@ -211,41 +224,45 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~.*}`})) currentTime := time.Now() - currentNow := func() time.Time { + mockedNow := func() time.Time { return currentTime } - c := NewActiveSeries(asm, 5*time.Minute, currentNow) + c := NewActiveSeries(asm, DefaultTimeout, mockedNow) - allActive, activeMatching, _ := c.Active() + allActive, activeMatching, valid := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) + assert.True(t, valid) - c.UpdateSeries(ls1, currentNow(), copyFn) - allActive, activeMatching, _ = c.Active() + c.UpdateSeries(ls1, currentTime, copyFn) + allActive, activeMatching, valid = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{1}, activeMatching) + assert.True(t, valid) c.ReloadMatchers(asm) - allActive, activeMatching, valid := c.Active() - assert.Equal(t, 0, allActive) - assert.Equal(t, []int{0}, activeMatching) + _, _, valid = c.Active() assert.False(t, valid) - c.UpdateSeries(ls1, currentNow(), copyFn) - c.UpdateSeries(ls2, currentNow(), copyFn) + // Adding timeout time to make Active results valid. + currentTime = currentTime.Add(DefaultTimeout) + c.UpdateSeries(ls1, currentTime, copyFn) + c.UpdateSeries(ls2, currentTime, copyFn) allActive, activeMatching, valid = c.Active() assert.Equal(t, 2, allActive) assert.Equal(t, []int{2}, activeMatching) - assert.False(t, valid) + assert.True(t, valid) asmWithLessMatchers := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{})) c.ReloadMatchers(asmWithLessMatchers) - c.UpdateSeries(ls3, currentNow(), copyFn) + // Adding timeout time to make Active results valid. + currentTime = currentTime.Add(DefaultTimeout) + c.UpdateSeries(ls3, currentTime, copyFn) allActive, activeMatching, valid = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int(nil), activeMatching) - assert.False(t, valid) + assert.True(t, valid) asmWithMoreMatchers := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "a": `{a="3"}`, @@ -253,15 +270,12 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { })) c.ReloadMatchers(asmWithMoreMatchers) - c.UpdateSeries(ls4, currentNow(), copyFn) + // Adding timeout time to make Active results valid. + currentTime = currentTime.Add(DefaultTimeout) + c.UpdateSeries(ls4, currentTime, copyFn) allActive, activeMatching, valid = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{0, 1}, activeMatching) - assert.False(t, valid) - - currentTime = currentTime.Add(5 * time.Minute).Add(1 * time.Second) - - _, _, valid = c.Active() assert.True(t, valid) } @@ -273,15 +287,21 @@ func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { "bar": `{a=~.+}`, })) - c := NewActiveSeries(asm, 5*time.Minute, time.Now) - allActive, activeMatching, _ := c.Active() + currentTime := time.Now() + mockedNow := func() time.Time { + return currentTime + } + c := NewActiveSeries(asm, DefaultTimeout, mockedNow) + allActive, activeMatching, valid := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) + assert.True(t, valid) - c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching, _ = c.Active() + c.UpdateSeries(ls1, currentTime, copyFn) + allActive, activeMatching, valid = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{1, 1}, activeMatching) + assert.True(t, valid) asm = NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "foo": `{a=~.+}`, @@ -289,10 +309,12 @@ func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { c.ReloadMatchers(asm) c.purge(time.Time{}) - - allActive, activeMatching, _ = c.Active() + // Adding timeout time to make Active results valid. + currentTime = currentTime.Add(DefaultTimeout) + allActive, activeMatching, valid = c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) + assert.True(t, valid) } func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { @@ -303,15 +325,22 @@ func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { "bar": `{a=~.+}`, })) - c := NewActiveSeries(asm, 5*time.Minute, time.Now) - allActive, activeMatching, _ := c.Active() + currentTime := time.Now() + mockedNow := func() time.Time { + return currentTime + } + + c := NewActiveSeries(asm, DefaultTimeout, mockedNow) + allActive, activeMatching, valid := c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) + assert.True(t, valid) - c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching, _ = c.Active() + c.UpdateSeries(ls1, currentTime, copyFn) + allActive, activeMatching, valid = c.Active() assert.Equal(t, 1, allActive) assert.Equal(t, []int{1, 1}, activeMatching) + assert.True(t, valid) asm = NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "foo": `{b=~.+}`, @@ -320,10 +349,13 @@ func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { c.ReloadMatchers(asm) c.purge(time.Time{}) + // Adding timeout time to make Active results valid. + currentTime = currentTime.Add(DefaultTimeout) - allActive, activeMatching, _ = c.Active() + allActive, activeMatching, valid = c.Active() assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) + assert.True(t, valid) } var activeSeriesTestGoroutines = []int{50, 100, 500} @@ -341,7 +373,7 @@ func benchmarkActiveSeriesConcurrencySingleSeries(b *testing.B, goroutines int) {Name: "a", Value: "a"}, } - c := NewActiveSeries(&Matchers{}, 5*time.Minute, time.Now) + c := NewActiveSeries(&Matchers{}, DefaultTimeout, time.Now) wg := &sync.WaitGroup{} start := make(chan struct{}) @@ -405,7 +437,7 @@ func BenchmarkActiveSeries_UpdateSeries(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - c := NewActiveSeries(&Matchers{}, 5*time.Minute, time.Now) + c := NewActiveSeries(&Matchers{}, DefaultTimeout, time.Now) for round := 0; round <= tt.nRounds; round++ { for ix := 0; ix < tt.nSeries; ix++ { c.UpdateSeries(series[ix], time.Unix(0, now), copyFn) @@ -429,8 +461,11 @@ func benchmarkPurge(b *testing.B, twice bool) { const numSeries = 10000 const numExpiresSeries = numSeries / 25 - currentNow := time.Now() - c := NewActiveSeries(&Matchers{}, 5*time.Minute, time.Now) + currentTime := time.Now() + mockedNow := func() time.Time { + return currentTime + } + c := NewActiveSeries(&Matchers{}, DefaultTimeout, mockedNow) series := [numSeries]labels.Labels{} for s := 0; s < numSeries; s++ { @@ -443,26 +478,27 @@ func benchmarkPurge(b *testing.B, twice bool) { // Prepare series for ix, s := range series { if ix < numExpiresSeries { - c.UpdateSeries(s, currentNow.Add(-time.Minute), copyFn) + c.UpdateSeries(s, currentTime.Add(-DefaultTimeout), copyFn) } else { - c.UpdateSeries(s, currentNow, copyFn) + c.UpdateSeries(s, currentTime, copyFn) } } - allActive, _, _ := c.Active() + allActive, _, valid := c.Active() assert.Equal(b, numSeries, allActive) + assert.True(b, valid) b.StartTimer() - // Purge everything - now := time.Now() - c.purge(now) - allActive, _, _ = c.Active() + // Active is going to purge everything + currentTime = currentTime.Add(DefaultTimeout) + allActive, _, valid = c.Active() assert.Equal(b, numSeries-numExpiresSeries, allActive) + assert.True(b, valid) if twice { - c.purge(now) - allActive, _, _ = c.Active() + allActive, _, valid = c.Active() assert.Equal(b, numSeries-numExpiresSeries, allActive) + assert.True(b, valid) } } } diff --git a/pkg/ingester/activeseries/matchers_test.go b/pkg/ingester/activeseries/matchers_test.go index 6100ce83bc..b693a4b02f 100644 --- a/pkg/ingester/activeseries/matchers_test.go +++ b/pkg/ingester/activeseries/matchers_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { +func TestMatcher_MatchesSeries(t *testing.T) { asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{ "bar_starts_with_1": `{bar=~"1.*"}`, "does_not_have_foo_label": `{foo=""}`, @@ -86,7 +86,7 @@ func TestActiveSeriesMatcher_MatchesSeries(t *testing.T) { } } -func TestActiveSeriesCustomTrackersConfigs_MalformedMatcher(t *testing.T) { +func TestCustomTrackersConfigs_MalformedMatcher(t *testing.T) { for _, matcher := range []string{ `{foo}`, `{foo=~"}`, From f7136fb7e28b2e480eac2e903b02299f89ef8395 Mon Sep 17 00:00:00 2001 From: Janos Gub Date: Thu, 17 Mar 2022 13:18:56 +0100 Subject: [PATCH 70/91] Update pkg/ingester/activeseries/active_series_test.go Co-authored-by: Oleg Zaytsev --- pkg/ingester/activeseries/active_series_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/activeseries/active_series_test.go b/pkg/ingester/activeseries/active_series_test.go index 345e020f80..bcf25aca3c 100644 --- a/pkg/ingester/activeseries/active_series_test.go +++ b/pkg/ingester/activeseries/active_series_test.go @@ -449,7 +449,7 @@ func BenchmarkActiveSeries_UpdateSeries(b *testing.B) { } } -func BenchmarkActiveSeries_Purge_once(b *testing.B) { +func BenchmarkActiveSeries_Active_once(b *testing.B) { benchmarkPurge(b, false) } From 7ca8b1e7802f92ea00bcc0bd82eadc33d3e166e2 Mon Sep 17 00:00:00 2001 From: Janos Gub Date: Thu, 17 Mar 2022 13:19:00 +0100 Subject: [PATCH 71/91] Update pkg/ingester/activeseries/active_series_test.go Co-authored-by: Oleg Zaytsev --- pkg/ingester/activeseries/active_series_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/activeseries/active_series_test.go b/pkg/ingester/activeseries/active_series_test.go index bcf25aca3c..d2ce1839d1 100644 --- a/pkg/ingester/activeseries/active_series_test.go +++ b/pkg/ingester/activeseries/active_series_test.go @@ -453,7 +453,7 @@ func BenchmarkActiveSeries_Active_once(b *testing.B) { benchmarkPurge(b, false) } -func BenchmarkActiveSeries_Purge_twice(b *testing.B) { +func BenchmarkActiveSeries_Active_twice(b *testing.B) { benchmarkPurge(b, true) } From e14d4dbd10d5b2e26fcf75bc21b8d45a5ce1f459 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 17 Mar 2022 15:24:00 +0100 Subject: [PATCH 72/91] Removing accidentally added file --- pkg/mimir/metrics-activity.log | Bin 1048576 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 pkg/mimir/metrics-activity.log diff --git a/pkg/mimir/metrics-activity.log b/pkg/mimir/metrics-activity.log deleted file mode 100644 index 9e0f96a2a253b173cb45b41868209a5d043e1437..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1048576 zcmeIuF#!Mo0K%a4Pi+Wah(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ T0|pEjFkrxd0RsjM82APT0Pp|- From 69054d395c3df37d80bf02b3e6c03259aa6aa938 Mon Sep 17 00:00:00 2001 From: Janos Gub Date: Fri, 18 Mar 2022 10:36:49 +0100 Subject: [PATCH 73/91] Update pkg/ingester/activeseries/active_series.go Co-authored-by: Mauro Stettler --- pkg/ingester/activeseries/active_series.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index 2c7ba7a8e5..2023ff200c 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -25,7 +25,7 @@ type ActiveSeries struct { matchers *Matchers lastMatchersUpdate time.Time - // The duration after series become inactive. + // The duration after which series become inactive. timeout time.Duration now func() time.Time } From 47326c0cc5a33b81d92d9bdbe0ae91b80944aafe Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 22 Mar 2022 17:40:52 +0100 Subject: [PATCH 74/91] Reworking active series trackers to get time from external calls --- pkg/ingester/activeseries/active_series.go | 13 +- .../activeseries/active_series_test.go | 118 ++++++++---------- pkg/ingester/ingester.go | 20 ++- pkg/ingester/ingester_test.go | 100 ++++++--------- 4 files changed, 102 insertions(+), 149 deletions(-) diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index 2023ff200c..622952cabc 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -27,7 +27,6 @@ type ActiveSeries struct { // The duration after which series become inactive. timeout time.Duration - now func() time.Time } // seriesStripe holds a subset of the series timestamps for a single tenant. @@ -52,8 +51,8 @@ type seriesEntry struct { matches []bool // Which matchers of Matchers does this series match } -func NewActiveSeries(asm *Matchers, timeout time.Duration, now func() time.Time) *ActiveSeries { - c := &ActiveSeries{matchers: asm, timeout: timeout, now: now} +func NewActiveSeries(asm *Matchers, timeout time.Duration) *ActiveSeries { + c := &ActiveSeries{matchers: asm, timeout: timeout} // Stripes are pre-allocated so that we only read on them and no lock is required. for i := 0; i < numStripes; i++ { @@ -73,7 +72,7 @@ func (c *ActiveSeries) CurrentMatcherNames() []string { return c.matchers.MatcherNames() } -func (c *ActiveSeries) ReloadMatchers(asm *Matchers) { +func (c *ActiveSeries) ReloadMatchers(asm *Matchers, now time.Time) { c.mu.Lock() defer c.mu.Unlock() @@ -81,7 +80,7 @@ func (c *ActiveSeries) ReloadMatchers(asm *Matchers) { c.stripes[i].reinitialize(asm) } c.matchers = asm - c.lastMatchersUpdate = c.now() + c.lastMatchersUpdate = now } func (c *ActiveSeries) CurrentConfig() CustomTrackersConfig { @@ -117,10 +116,10 @@ func (c *ActiveSeries) clear() { // The result is correct only if the third return value is true, which shows if enough time has passed since last reload. // This should be called periodically to avoid memory leaks. // Active cannot be called concurrently with ReloadMatchers. -func (c *ActiveSeries) Active() (int, []int, bool) { +func (c *ActiveSeries) Active(now time.Time) (int, []int, bool) { c.mu.Lock() defer c.mu.Unlock() - purgeTime := c.now().Add(-c.timeout) + purgeTime := now.Add(-c.timeout) c.purge(purgeTime) if c.lastMatchersUpdate.After(purgeTime) { diff --git a/pkg/ingester/activeseries/active_series_test.go b/pkg/ingester/activeseries/active_series_test.go index d2ce1839d1..91f07aec66 100644 --- a/pkg/ingester/activeseries/active_series_test.go +++ b/pkg/ingester/activeseries/active_series_test.go @@ -28,24 +28,24 @@ func TestActiveSeries_UpdateSeries_NoMatchers(t *testing.T) { ls1 := []labels.Label{{Name: "a", Value: "1"}} ls2 := []labels.Label{{Name: "a", Value: "2"}} - c := NewActiveSeries(&Matchers{}, DefaultTimeout, time.Now) - allActive, activeMatching, valid := c.Active() + c := NewActiveSeries(&Matchers{}, DefaultTimeout) + allActive, activeMatching, valid := c.Active(time.Now()) assert.Equal(t, 0, allActive) assert.Nil(t, activeMatching) assert.True(t, valid) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, _, valid = c.Active() + allActive, _, valid = c.Active(time.Now()) assert.Equal(t, 1, allActive) assert.True(t, valid) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, _, valid = c.Active() + allActive, _, valid = c.Active(time.Now()) assert.Equal(t, 1, allActive) assert.True(t, valid) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, _, valid = c.Active() + allActive, _, valid = c.Active(time.Now()) assert.Equal(t, 2, allActive) assert.True(t, valid) } @@ -57,32 +57,32 @@ func TestActiveSeries_UpdateSeries_WithMatchers(t *testing.T) { asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~"2|3"}`})) - c := NewActiveSeries(asm, DefaultTimeout, time.Now) - allActive, activeMatching, valid := c.Active() + c := NewActiveSeries(asm, DefaultTimeout) + allActive, activeMatching, valid := c.Active(time.Now()) assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) assert.True(t, valid) c.UpdateSeries(ls1, time.Now(), copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(time.Now()) assert.Equal(t, 1, allActive) assert.Equal(t, []int{0}, activeMatching) assert.True(t, valid) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(time.Now()) assert.Equal(t, 2, allActive) assert.Equal(t, []int{1}, activeMatching) assert.True(t, valid) c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(time.Now()) assert.Equal(t, 3, allActive) assert.Equal(t, []int{2}, activeMatching) assert.True(t, valid) c.UpdateSeries(ls3, time.Now(), copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(time.Now()) assert.Equal(t, 3, allActive) assert.Equal(t, []int{2}, activeMatching) assert.True(t, valid) @@ -94,11 +94,11 @@ func TestActiveSeries_ShouldCorrectlyHandleFingerprintCollisions(t *testing.T) { ls2 := metric.Set("_", "KiqbryhzUpn").Labels() require.True(t, client.Fingerprint(ls1) == client.Fingerprint(ls2)) - c := NewActiveSeries(&Matchers{}, DefaultTimeout, time.Now) + c := NewActiveSeries(&Matchers{}, DefaultTimeout) c.UpdateSeries(ls1, time.Now(), copyFn) c.UpdateSeries(ls2, time.Now(), copyFn) - allActive, _, valid := c.Active() + allActive, _, valid := c.Active(time.Now()) assert.Equal(t, 2, allActive) assert.True(t, valid) } @@ -115,10 +115,8 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl: %d", ttl), func(t *testing.T) { - mockedNow := func() time.Time { - return time.Unix(int64(ttl), 0) - } - c := NewActiveSeries(&Matchers{}, DefaultTimeout, mockedNow) + mockedTime := time.Unix(int64(ttl), 0) + c := NewActiveSeries(&Matchers{}, DefaultTimeout) for i := 0; i < len(series); i++ { c.UpdateSeries(series[i], time.Unix(int64(i), 0), copyFn) @@ -130,7 +128,7 @@ func TestActiveSeries_Purge_NoMatchers(t *testing.T) { exp := len(series) - (ttl) // c.Active is not intended to purge - allActive, activeMatching, valid := c.Active() + allActive, activeMatching, valid := c.Active(mockedTime) assert.Equal(t, exp, allActive) assert.Nil(t, activeMatching) assert.True(t, valid) @@ -152,10 +150,9 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { // Run the same test for increasing TTL values for ttl := 1; ttl <= len(series); ttl++ { t.Run(fmt.Sprintf("ttl=%d", ttl), func(t *testing.T) { - mockedNow := func() time.Time { - return time.Unix(int64(ttl), 0) - } - c := NewActiveSeries(asm, 5*time.Minute, mockedNow) + mockedTime := time.Unix(int64(ttl), 0) + + c := NewActiveSeries(asm, 5*time.Minute) exp := len(series) - ttl expMatchingSeries := 0 @@ -174,7 +171,7 @@ func TestActiveSeries_Purge_WithMatchers(t *testing.T) { c.purge(time.Unix(int64(ttl), 0)) // c.Active is not intended to purge - allActive, activeMatching, valid := c.Active() + allActive, activeMatching, valid := c.Active(mockedTime) assert.Equal(t, exp, allActive) assert.Equal(t, []int{expMatchingSeries}, activeMatching) assert.True(t, valid) @@ -188,29 +185,26 @@ func TestActiveSeries_PurgeOpt(t *testing.T) { ls2 := metric.Set("_", "KiqbryhzUpn").Labels() currentTime := time.Now() - mockedNow := func() time.Time { - return currentTime - } - c := NewActiveSeries(&Matchers{}, 59*time.Second, mockedNow) + c := NewActiveSeries(&Matchers{}, 59*time.Second) c.UpdateSeries(ls1, currentTime.Add(-2*time.Minute), copyFn) c.UpdateSeries(ls2, currentTime, copyFn) - allActive, _, valid := c.Active() + allActive, _, valid := c.Active(currentTime) assert.Equal(t, 1, allActive) assert.True(t, valid) c.UpdateSeries(ls1, currentTime.Add(-1*time.Minute), copyFn) c.UpdateSeries(ls2, currentTime, copyFn) - allActive, _, valid = c.Active() + allActive, _, valid = c.Active(currentTime) assert.Equal(t, 1, allActive) assert.True(t, valid) // This will *not* update the series, since there is already newer timestamp. c.UpdateSeries(ls2, currentTime.Add(-1*time.Minute), copyFn) - allActive, _, valid = c.Active() + allActive, _, valid = c.Active(currentTime) assert.Equal(t, 1, allActive) assert.True(t, valid) } @@ -224,42 +218,39 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { asm := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{"foo": `{a=~.*}`})) currentTime := time.Now() - mockedNow := func() time.Time { - return currentTime - } - c := NewActiveSeries(asm, DefaultTimeout, mockedNow) + c := NewActiveSeries(asm, DefaultTimeout) - allActive, activeMatching, valid := c.Active() + allActive, activeMatching, valid := c.Active(currentTime) assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) assert.True(t, valid) c.UpdateSeries(ls1, currentTime, copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(currentTime) assert.Equal(t, 1, allActive) assert.Equal(t, []int{1}, activeMatching) assert.True(t, valid) - c.ReloadMatchers(asm) - _, _, valid = c.Active() + c.ReloadMatchers(asm, currentTime) + _, _, valid = c.Active(currentTime) assert.False(t, valid) // Adding timeout time to make Active results valid. currentTime = currentTime.Add(DefaultTimeout) c.UpdateSeries(ls1, currentTime, copyFn) c.UpdateSeries(ls2, currentTime, copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(currentTime) assert.Equal(t, 2, allActive) assert.Equal(t, []int{2}, activeMatching) assert.True(t, valid) asmWithLessMatchers := NewMatchers(mustNewCustomTrackersConfigFromMap(t, map[string]string{})) - c.ReloadMatchers(asmWithLessMatchers) + c.ReloadMatchers(asmWithLessMatchers, currentTime) // Adding timeout time to make Active results valid. currentTime = currentTime.Add(DefaultTimeout) c.UpdateSeries(ls3, currentTime, copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(currentTime) assert.Equal(t, 1, allActive) assert.Equal(t, []int(nil), activeMatching) assert.True(t, valid) @@ -268,12 +259,12 @@ func TestActiveSeries_ReloadSeriesMatchers(t *testing.T) { "a": `{a="3"}`, "b": `{a="4"}`, })) - c.ReloadMatchers(asmWithMoreMatchers) + c.ReloadMatchers(asmWithMoreMatchers, currentTime) // Adding timeout time to make Active results valid. currentTime = currentTime.Add(DefaultTimeout) c.UpdateSeries(ls4, currentTime, copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(currentTime) assert.Equal(t, 1, allActive) assert.Equal(t, []int{0, 1}, activeMatching) assert.True(t, valid) @@ -288,17 +279,14 @@ func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { })) currentTime := time.Now() - mockedNow := func() time.Time { - return currentTime - } - c := NewActiveSeries(asm, DefaultTimeout, mockedNow) - allActive, activeMatching, valid := c.Active() + c := NewActiveSeries(asm, DefaultTimeout) + allActive, activeMatching, valid := c.Active(currentTime) assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) assert.True(t, valid) c.UpdateSeries(ls1, currentTime, copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(currentTime) assert.Equal(t, 1, allActive) assert.Equal(t, []int{1, 1}, activeMatching) assert.True(t, valid) @@ -307,11 +295,11 @@ func TestActiveSeries_ReloadSeriesMatchers_LessMatchers(t *testing.T) { "foo": `{a=~.+}`, })) - c.ReloadMatchers(asm) + c.ReloadMatchers(asm, currentTime) c.purge(time.Time{}) // Adding timeout time to make Active results valid. currentTime = currentTime.Add(DefaultTimeout) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(currentTime) assert.Equal(t, 0, allActive) assert.Equal(t, []int{0}, activeMatching) assert.True(t, valid) @@ -326,18 +314,15 @@ func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { })) currentTime := time.Now() - mockedNow := func() time.Time { - return currentTime - } - c := NewActiveSeries(asm, DefaultTimeout, mockedNow) - allActive, activeMatching, valid := c.Active() + c := NewActiveSeries(asm, DefaultTimeout) + allActive, activeMatching, valid := c.Active(currentTime) assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) assert.True(t, valid) c.UpdateSeries(ls1, currentTime, copyFn) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(currentTime) assert.Equal(t, 1, allActive) assert.Equal(t, []int{1, 1}, activeMatching) assert.True(t, valid) @@ -347,12 +332,12 @@ func TestActiveSeries_ReloadSeriesMatchers_SameSizeNewLabels(t *testing.T) { "bar": `{b=~.+}`, })) - c.ReloadMatchers(asm) + c.ReloadMatchers(asm, currentTime) c.purge(time.Time{}) // Adding timeout time to make Active results valid. currentTime = currentTime.Add(DefaultTimeout) - allActive, activeMatching, valid = c.Active() + allActive, activeMatching, valid = c.Active(currentTime) assert.Equal(t, 0, allActive) assert.Equal(t, []int{0, 0}, activeMatching) assert.True(t, valid) @@ -373,7 +358,7 @@ func benchmarkActiveSeriesConcurrencySingleSeries(b *testing.B, goroutines int) {Name: "a", Value: "a"}, } - c := NewActiveSeries(&Matchers{}, DefaultTimeout, time.Now) + c := NewActiveSeries(&Matchers{}, DefaultTimeout) wg := &sync.WaitGroup{} start := make(chan struct{}) @@ -437,7 +422,7 @@ func BenchmarkActiveSeries_UpdateSeries(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - c := NewActiveSeries(&Matchers{}, DefaultTimeout, time.Now) + c := NewActiveSeries(&Matchers{}, DefaultTimeout) for round := 0; round <= tt.nRounds; round++ { for ix := 0; ix < tt.nSeries; ix++ { c.UpdateSeries(series[ix], time.Unix(0, now), copyFn) @@ -462,10 +447,7 @@ func benchmarkPurge(b *testing.B, twice bool) { const numExpiresSeries = numSeries / 25 currentTime := time.Now() - mockedNow := func() time.Time { - return currentTime - } - c := NewActiveSeries(&Matchers{}, DefaultTimeout, mockedNow) + c := NewActiveSeries(&Matchers{}, DefaultTimeout) series := [numSeries]labels.Labels{} for s := 0; s < numSeries; s++ { @@ -484,19 +466,19 @@ func benchmarkPurge(b *testing.B, twice bool) { } } - allActive, _, valid := c.Active() + allActive, _, valid := c.Active(currentTime) assert.Equal(b, numSeries, allActive) assert.True(b, valid) b.StartTimer() // Active is going to purge everything currentTime = currentTime.Add(DefaultTimeout) - allActive, _, valid = c.Active() + allActive, _, valid = c.Active(currentTime) assert.Equal(b, numSeries-numExpiresSeries, allActive) assert.True(b, valid) if twice { - allActive, _, valid = c.Active() + allActive, _, valid = c.Active(currentTime) assert.Equal(b, numSeries-numExpiresSeries, allActive) assert.True(b, valid) } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 455be82da1..d703c41c2c 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -234,9 +234,6 @@ type Ingester struct { // Rate of pushed samples. Used to limit global samples push rate. ingestionRate *util_math.EwmaRate inflightPushRequests atomic.Int64 - - // Single source of time for making ingester testable. - now func() time.Time } func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus.Registerer, logger log.Logger) (*Ingester, error) { @@ -261,7 +258,6 @@ func newIngester(cfg Config, limits *validation.Overrides, registerer prometheus forceCompactTrigger: make(chan requestWithUsersAndCallback), shipTrigger: make(chan requestWithUsersAndCallback), seriesHashCache: hashcache.NewSeriesHashCache(cfg.BlocksStorageConfig.TSDB.SeriesHashCacheMaxBytes), - now: time.Now, }, nil } @@ -459,7 +455,7 @@ func (i *Ingester) updateLoop(ctx context.Context) error { i.applyExemplarsSettings() case <-activeSeriesTickerChan: - i.updateActiveSeries() + i.updateActiveSeries(time.Now()) case <-ctx.Done(): return nil @@ -469,12 +465,12 @@ func (i *Ingester) updateLoop(ctx context.Context) error { } } -func (i *Ingester) replaceMatchers(asm *activeseries.Matchers, userDB *userTSDB) { +func (i *Ingester) replaceMatchers(asm *activeseries.Matchers, userDB *userTSDB, now time.Time) { i.metrics.deletePerUserCustomTrackerMetrics(userDB.userID, userDB.activeSeries.CurrentMatcherNames()) - userDB.activeSeries.ReloadMatchers(asm) + userDB.activeSeries.ReloadMatchers(asm, now) } -func (i *Ingester) updateActiveSeries() { +func (i *Ingester) updateActiveSeries(now time.Time) { for _, userID := range i.getTSDBUsers() { userDB := i.getTSDB(userID) if userDB == nil { @@ -483,9 +479,9 @@ func (i *Ingester) updateActiveSeries() { newMatchersConfig := i.limits.ActiveSeriesCustomTrackersConfig(userID) if newMatchersConfig.String() != userDB.activeSeries.CurrentConfig().String() { - i.replaceMatchers(activeseries.NewMatchers(newMatchersConfig), userDB) + i.replaceMatchers(activeseries.NewMatchers(newMatchersConfig), userDB, now) } - allActive, activeMatching, valid := userDB.activeSeries.Active() + allActive, activeMatching, valid := userDB.activeSeries.Active(now) if !valid { // Active series config has been reloaded, exposing loading metric until MetricsIdleTimeout passes. i.metrics.activeSeriesLoading.WithLabelValues(userID).Set(1) @@ -610,7 +606,7 @@ func (i *Ingester) PushWithCleanup(ctx context.Context, req *mimirpb.WriteReques failedSamplesCount = 0 succeededExemplarsCount = 0 failedExemplarsCount = 0 - startAppend = i.now() + startAppend = time.Now() sampleOutOfBoundsCount = 0 sampleOutOfOrderCount = 0 newValueForTimestampCount = 0 @@ -1456,7 +1452,7 @@ func (i *Ingester) createTSDB(userID string) (*userTSDB, error) { userDB := &userTSDB{ userID: userID, - activeSeries: activeseries.NewActiveSeries(activeseries.NewMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout, i.now), + activeSeries: activeseries.NewActiveSeries(activeseries.NewMatchers(matchersConfig), i.cfg.ActiveSeriesMetricsIdleTimeout), seriesInMetric: newMetricCounter(i.limiter, i.cfg.getIgnoreSeriesLimitForMetricNamesMap()), ingestedAPISamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), ingestedRuleSamples: util_math.NewEWMARate(0.2, i.cfg.RateUpdatePeriod), diff --git a/pkg/ingester/ingester_test.go b/pkg/ingester/ingester_test.go index 5e197c92ad..62e8ff8f1c 100644 --- a/pkg/ingester/ingester_test.go +++ b/pkg/ingester/ingester_test.go @@ -696,7 +696,7 @@ func TestIngester_Push(t *testing.T) { // Update active series for metrics check. if !testData.disableActiveSeries { - i.updateActiveSeries() + i.updateActiveSeries(time.Now()) } // Append additional metrics to assert on. @@ -765,7 +765,7 @@ func TestIngester_Push_ShouldCorrectlyTrackMetricsInMultiTenantScenario(t *testi } // Update active series for metrics check. - i.updateActiveSeries() + i.updateActiveSeries(time.Now()) // Check tracked Prometheus metrics expectedMetrics := ` @@ -816,10 +816,6 @@ func TestIngester_Push_DecreaseInactiveSeries(t *testing.T) { i, err := prepareIngesterWithBlocksStorage(t, cfg, registry) currentTime := time.Now() - currentNow := func() time.Time { - return currentTime - } - i.now = currentNow require.NoError(t, err) require.NoError(t, services.StartAndAwaitRunning(context.Background(), i)) defer services.StopAndAwaitTerminated(context.Background(), i) //nolint:errcheck @@ -858,7 +854,7 @@ func TestIngester_Push_DecreaseInactiveSeries(t *testing.T) { // Update active series the after the idle timeout (in the future). // This will remove inactive series. currentTime = currentTime.Add(cfg.ActiveSeriesMetricsIdleTimeout + 1*time.Second) - i.updateActiveSeries() + i.updateActiveSeries(currentTime) // Check tracked Prometheus metrics expectedMetrics := ` @@ -3871,7 +3867,7 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { }) pushSingleSampleWithMetadata(t, i) - i.updateActiveSeries() + i.updateActiveSeries(time.Now()) require.Equal(t, int64(1), i.seriesCount.Load()) @@ -3912,7 +3908,7 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { }) require.Greater(t, testutil.ToFloat64(i.metrics.idleTsdbChecks.WithLabelValues(string(tsdbIdleClosed))), float64(0)) - i.updateActiveSeries() + i.updateActiveSeries(time.Now()) require.Equal(t, int64(0), i.seriesCount.Load()) // Flushing removed all series from memory. // Verify that user has disappeared from metrics. @@ -3937,7 +3933,7 @@ func TestIngesterCompactAndCloseIdleTSDB(t *testing.T) { // Pushing another sample will recreate TSDB. pushSingleSampleWithMetadata(t, i) - i.updateActiveSeries() + i.updateActiveSeries(time.Now()) // User is back. require.NoError(t, testutil.GatherAndCompare(r, strings.NewReader(` @@ -5199,7 +5195,7 @@ func TestIngesterActiveSeries(t *testing.T) { pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries() + ingester.updateActiveSeries(time.Now()) expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5224,7 +5220,7 @@ func TestIngesterActiveSeries(t *testing.T) { pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries() + ingester.updateActiveSeries(time.Now()) expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5250,16 +5246,11 @@ func TestIngesterActiveSeries(t *testing.T) { "should track custom matchers, removing when zero": { test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { currentTime := time.Now() - currentNow := func() time.Time { - return currentTime - } - - ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5278,14 +5269,14 @@ func TestIngesterActiveSeries(t *testing.T) { require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) // Pushing second time to have entires which are not going to be purged - currentTime = currentTime.Add(10 * time.Second) + currentTime = time.Now() pushWithUser(t, ingester, labelsToPush, userID, req) // Adding time to make the first batch of pushes idle. // We update them in the exact moment in time where append time of the first push is already considered idle, // while the second append happens after the purge timestamp. - currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout - 5*time.Second) - ingester.updateActiveSeries() + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout) + ingester.updateActiveSeries(currentTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5302,7 +5293,7 @@ func TestIngesterActiveSeries(t *testing.T) { // Update active series again in a further future where no series are active anymore. currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout) - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(""), metricNames...)) }, }, @@ -5313,7 +5304,7 @@ func TestIngesterActiveSeries(t *testing.T) { pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries() + ingester.updateActiveSeries(time.Now()) expectedMetrics := `` @@ -5404,16 +5395,12 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { currentTime := time.Now() - currentNow := func() time.Time { - return currentTime - } - ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5439,8 +5426,9 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { override, err := validation.NewOverrides(limits, activeSeriesTenantOverride) require.NoError(t, err) ingester.limits = override + currentTime = time.Now() // First update reloads the config - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5455,13 +5443,13 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // Adding time to second push to avoid purging it before exposing. - currentTime = currentTime.Add(10 * time.Second) + // Saving time before second push to avoid purging it before exposing. + currentTime = time.Now() pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) - // Adding idleTimeout - 1 to expose the metrics but not purge the pushes. - currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout - time.Second) - ingester.updateActiveSeries() + // Adding idleTimeout to expose the metrics but not purge the pushes. + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout) + ingester.updateActiveSeries(currentTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5482,16 +5470,12 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { tenantLimits: defaultActiveSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { currentTime := time.Now() - currentNow := func() time.Time { - return currentTime - } - ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5514,7 +5498,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { override, err := validation.NewOverrides(limits, nil) require.NoError(t, err) ingester.limits = override - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5529,13 +5513,13 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // Adding time to second push to avoid purging it before exposing. - currentTime = currentTime.Add(10 * time.Second) + // Saving time before second push to avoid purging it before exposing. + currentTime = time.Now() pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) - // Adding idleTimeout - 1 to expose the metrics but not purge the pushes. - currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout - time.Second) - ingester.updateActiveSeries() + // Adding idleTimeout to expose the metrics but not purge the pushes. + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout) + ingester.updateActiveSeries(currentTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5555,15 +5539,11 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { activeSeriesConfig: activeSeriesDefaultConfig, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { currentTime := time.Now() - currentNow := func() time.Time { - return currentTime - } - ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) // Update active series for metrics check. - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5590,7 +5570,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { override, err := validation.NewOverrides(limits, activeSeriesTenantOverride) require.NoError(t, err) ingester.limits = override - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics = ` # HELP cortex_ingester_active_series_loading Indicates that active series configuration is being reloaded, and waiting to become stable. While this metric is non zero, values from active series metrics shouldn't be considered. # TYPE cortex_ingester_active_series_loading gauge @@ -5598,12 +5578,12 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { ` require.NoError(t, testutil.GatherAndCompare(gatherer, strings.NewReader(expectedMetrics), metricNames...)) - // Adding time to second push to avoid purging it before exposing. - currentTime = currentTime.Add(10 * time.Second) + // Saving time before second push to avoid purging it before exposing. + currentTime = time.Now() pushWithUser(t, ingester, labelsToPush, userID, req) - // Adding idleTimeout - 1 to expose the metrics but not purge the pushes. - currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout - time.Second) - ingester.updateActiveSeries() + // Adding idleTimeout to expose the metrics but not purge the pushes. + currentTime = currentTime.Add(ingester.cfg.ActiveSeriesMetricsIdleTimeout) + ingester.updateActiveSeries(currentTime) expectedMetrics = ` # HELP cortex_ingester_active_series Number of currently active series per user. # TYPE cortex_ingester_active_series gauge @@ -5623,16 +5603,12 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { tenantLimits: defaultActiveSeriesTenantOverride, test: func(t *testing.T, ingester *Ingester, gatherer prometheus.Gatherer) { currentTime := time.Now() - currentNow := func() time.Time { - return currentTime - } - ingester.now = currentNow pushWithUser(t, ingester, labelsToPush, userID, req) pushWithUser(t, ingester, labelsToPush, userID2, req) // Update active series for metrics check. - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics := ` # HELP cortex_ingester_active_series Number of currently active series per user. @@ -5654,7 +5630,7 @@ func TestIngesterActiveSeriesConfigChanges(t *testing.T) { override, err := validation.NewOverrides(limits, nil) require.NoError(t, err) ingester.limits = override - ingester.updateActiveSeries() + ingester.updateActiveSeries(currentTime) expectedMetrics = ` # HELP cortex_ingester_active_series_loading Indicates that active series configuration is being reloaded, and waiting to become stable. While this metric is non zero, values from active series metrics shouldn't be considered. # TYPE cortex_ingester_active_series_loading gauge From fd15e6ad362e73fee31048c025b01ba93d8af84b Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 24 Mar 2022 11:47:27 +0100 Subject: [PATCH 75/91] Addressing review on comments and minor cleanup --- pkg/ingester/activeseries/active_series.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index 622952cabc..38a1203215 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -26,6 +26,7 @@ type ActiveSeries struct { lastMatchersUpdate time.Time // The duration after which series become inactive. + // Also used to determine if enough time has passed since configuration reload for valid results. timeout time.Duration } @@ -56,11 +57,7 @@ func NewActiveSeries(asm *Matchers, timeout time.Duration) *ActiveSeries { // Stripes are pre-allocated so that we only read on them and no lock is required. for i := 0; i < numStripes; i++ { - c.stripes[i] = seriesStripe{ - matchers: asm, - refs: map[uint64][]seriesEntry{}, - activeMatching: resizeAndClear(len(asm.MatcherNames()), nil), - } + c.stripes[i].reinitialize(asm) } return c @@ -89,7 +86,7 @@ func (c *ActiveSeries) CurrentConfig() CustomTrackersConfig { return c.matchers.Config() } -// Updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. +// UpdateSeries updates series timestamp to 'now'. Function is called to make a copy of labels if entry doesn't exist yet. func (c *ActiveSeries) UpdateSeries(series labels.Labels, now time.Time, labelsCopy func(labels.Labels) labels.Labels) { fp := series.Hash() stripeID := fp % numStripes @@ -114,8 +111,7 @@ func (c *ActiveSeries) clear() { // Active returns the total number of active series, as well as a slice of active series matching each one of the // custom trackers provided (in the same order as custom trackers are defined). // The result is correct only if the third return value is true, which shows if enough time has passed since last reload. -// This should be called periodically to avoid memory leaks. -// Active cannot be called concurrently with ReloadMatchers. +// This should be called periodically to avoid unbounded memory growth. func (c *ActiveSeries) Active(now time.Time) (int, []int, bool) { c.mu.Lock() defer c.mu.Unlock() From 9e0a0e2b65b33e1b0561eb3e7dec54d33f469bed Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 29 Mar 2022 08:10:18 +0200 Subject: [PATCH 76/91] Changing active series deprecation override location and adding unit tests for it --- pkg/mimir/metrics-activity.log | Bin 0 -> 1048576 bytes pkg/mimir/modules.go | 26 ++++++++------- pkg/mimir/modules_test.go | 57 +++++++++++++++++++++++++++++++++ 3 files changed, 71 insertions(+), 12 deletions(-) create mode 100644 pkg/mimir/metrics-activity.log diff --git a/pkg/mimir/metrics-activity.log b/pkg/mimir/metrics-activity.log new file mode 100644 index 0000000000000000000000000000000000000000..9e0f96a2a253b173cb45b41868209a5d043e1437 GIT binary patch literal 1048576 zcmeIuF#!Mo0K%a4Pi+Wah(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ T0|pEjFkrxd0RsjM82APT0Pp|- literal 0 HcmV?d00001 diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 11af7d6089..662703b5a9 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -210,6 +210,20 @@ func (t *Mimir) initRing() (serv services.Service, err error) { } func (t *Mimir) initRuntimeConfig() (services.Service, error) { + // TODO Remove in Mimir 2.2. + // Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. + // We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, + // unless it's also defined in the limits, which is invalid. + // This needs to be set before setting default limits for unmarshalling. + if !t.Cfg.Ingester.ActiveSeriesCustomTrackers.Empty() { + if !t.Cfg.LimitsConfig.ActiveSeriesCustomTrackersConfig.Empty() { + return nil, fmt.Errorf("can't define active series custom trackers in both ingester and limits config, please define them only in the limits") + } + level.Warn(util_log.Logger).Log("msg", "active_series_custom_trackers is defined as an ingester config, this location is deprecated, please move it to the limits config") + flagext.DeprecatedFlagsUsed.Inc() + t.Cfg.LimitsConfig.ActiveSeriesCustomTrackersConfig = t.Cfg.Ingester.ActiveSeriesCustomTrackers + } + if t.Cfg.RuntimeConfig.LoadPath == "" { // no need to initialize module if load path is empty return nil, nil @@ -233,18 +247,6 @@ func (t *Mimir) initRuntimeConfig() (services.Service, error) { } func (t *Mimir) initOverrides() (serv services.Service, err error) { - // TODO Remove in Mimir 2.2. - // Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. - // We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, - // unless it's also defined in the limits, which is invalid. - if !t.Cfg.Ingester.ActiveSeriesCustomTrackers.Empty() { - if !t.Cfg.LimitsConfig.ActiveSeriesCustomTrackersConfig.Empty() { - return nil, fmt.Errorf("can't define active series custom trackers in both ingester and limits config, please define them only in the limits") - } - level.Warn(util_log.Logger).Log("msg", "active_series_custom_trackers is defined as an ingester config, this location is deprecated, please move it to the limits config") - flagext.DeprecatedFlagsUsed.Inc() - t.Cfg.LimitsConfig.ActiveSeriesCustomTrackersConfig = t.Cfg.Ingester.ActiveSeriesCustomTrackers - } t.Overrides, err = validation.NewOverrides(t.Cfg.LimitsConfig, t.TenantLimits) // overrides don't have operational state, nor do they need to do anything more in starting/stopping phase, // so there is no need to return any service. diff --git a/pkg/mimir/modules_test.go b/pkg/mimir/modules_test.go index e52dd5b465..f98f12d73a 100644 --- a/pkg/mimir/modules_test.go +++ b/pkg/mimir/modules_test.go @@ -16,6 +16,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/weaveworks/common/server" + + "github.com/grafana/mimir/pkg/ingester/activeseries" ) func changeTargetConfig(c *Config) { @@ -212,3 +214,58 @@ func TestMultiKVSetup(t *testing.T) { }) } } + +func TestActiveSeriesOverrides(t *testing.T) { + tests := map[string]struct { + config Config + }{ + "Override with runtime config specified": { + config: func() Config { + cfg := Config{} + flagext.DefaultValues(&cfg) + // Adding loadpath to hit default value set + cfg.RuntimeConfig.LoadPath = "testpath" + + trackersConfig, err := activeseries.NewCustomTrackersConfig(map[string]string{ + "bool_is_true_flag-based": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + }) + require.Nil(t, err) + cfg.Ingester.ActiveSeriesCustomTrackers = trackersConfig + return cfg + }(), + }, + "Override without runtime path specified": { + config: func() Config { + cfg := Config{} + flagext.DefaultValues(&cfg) + + trackersConfig, err := activeseries.NewCustomTrackersConfig(map[string]string{ + "bool_is_true_flag-based": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + }) + require.Nil(t, err) + cfg.Ingester.ActiveSeriesCustomTrackers = trackersConfig + return cfg + }(), + }, + } + for test, testData := range tests { + t.Run(test, func(t *testing.T) { + prepareGlobalMetricsRegistry(t) + // Set to 0 to find any free port. + cfg := testData.config + cfg.Server.HTTPListenPort = 0 + cfg.Server.GRPCListenPort = 0 + c, err := New(cfg) + require.Nil(t, err) + _, err = c.ModuleManager.InitModuleServices(Overrides) + require.NoError(t, err) + defer c.Server.Stop() + + defaultActiveSeriesConfig := c.Overrides.ActiveSeriesCustomTrackersConfig("nonexistent") + assert.False(t, defaultActiveSeriesConfig.Empty()) + + }) + } +} From 5d12046915da62aea31cc95a34a8d43320417717 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 29 Mar 2022 11:06:53 +0200 Subject: [PATCH 77/91] Adding TODO to remove module unit tests with Mimir 2.2 and adding higher level test to test runtime override case --- pkg/mimir/mimir_test.go | 62 +++++++++++++++++++++++++++++++++++++++ pkg/mimir/modules_test.go | 7 ++++- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/pkg/mimir/mimir_test.go b/pkg/mimir/mimir_test.go index 21c2022064..bd11a416da 100644 --- a/pkg/mimir/mimir_test.go +++ b/pkg/mimir/mimir_test.go @@ -11,6 +11,7 @@ import ( "flag" "fmt" "io" + "io/ioutil" "net" "net/http" "os" @@ -40,6 +41,7 @@ import ( "github.com/grafana/mimir/pkg/distributor" "github.com/grafana/mimir/pkg/frontend/v1/frontendv1pb" "github.com/grafana/mimir/pkg/ingester" + "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/ruler" "github.com/grafana/mimir/pkg/ruler/rulestore" "github.com/grafana/mimir/pkg/scheduler/schedulerpb" @@ -427,6 +429,66 @@ func TestFlagDefaults(t *testing.T) { require.Equal(t, 10*time.Second, c.Server.GRPCServerMinTimeBetweenPings) } +// TODO Remove in Mimir 2.2. +// Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. +// We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, +// unless it's also defined in the limits, which is invalid. +// This needs to be set before setting default limits for unmarshalling. +// For more context see https://github.com/grafana/mimir/pull/1188#discussion_r830129443 +func TestActiveSeriesDeprecationDefaultOverrideWithSomeRuntimeOverrides(t *testing.T) { + yamlContent := ` +overrides: + '1234': + ingestion_burst_size: 15000 + ingestion_rate: 1500 + max_global_series_per_metric: 7000 + max_global_series_per_user: 15000 + ruler_max_rule_groups_per_tenant: 20 + ruler_max_rules_per_rule_group: 20 +` + prepareGlobalMetricsRegistry(t) + + cfg := Config{} + + // This sets default values from flags to the config. + flagext.RegisterFlagsWithLogger(log.NewNopLogger(), &cfg) + + // Creating test file with runtime overrides. + tmpDir := t.TempDir() + cfg.RuntimeConfig.LoadPath = filepath.Join(tmpDir, "overrides.yml") + err := ioutil.WriteFile(cfg.RuntimeConfig.LoadPath, []byte(yamlContent), 777) + require.NoError(t, err, "Failed to write test override.yml.") + + // Setting up deprecated value as an ingester config value. + cfg.Ingester.ActiveSeriesCustomTrackers, err = activeseries.NewCustomTrackersConfig(map[string]string{ + "bool_is_true_flag-based": `{bool="true"}`, + "bool_is_false_flagbased": `{bool="false"}`, + }) + require.NoError(t, err) + + cfg.Target = []string{RuntimeConfig} + cfg.Server = getServerConfig(t) + require.NoError(t, cfg.Server.LogFormat.Set("logfmt")) + require.NoError(t, cfg.Server.LogLevel.Set("debug")) + util_log.InitLogger(&cfg.Server) + + c, err := New(cfg) + require.NoError(t, err) + + errCh := make(chan error) + go func() { + errCh <- c.Run() + }() + + // Waiting for RuntimeConfig to load config from file. + test.Poll(t, cfg.RuntimeConfig.ReloadPeriod, true, func() interface{} { + return c.RuntimeConfig != nil + }) + assert.NotNil(t, c.RuntimeConfig.GetConfig()) + assert.False(t, c.TenantLimits.ByUserID("1234").ActiveSeriesCustomTrackersConfig.Empty(), + "Default deserialization value for active series trackers should be set from ingester config.") +} + // Generates server config, with gRPC listening on random port. func getServerConfig(t *testing.T) server.Config { grpcHost, grpcPortNum := getHostnameAndRandomPort(t) diff --git a/pkg/mimir/modules_test.go b/pkg/mimir/modules_test.go index f98f12d73a..ae19855a2d 100644 --- a/pkg/mimir/modules_test.go +++ b/pkg/mimir/modules_test.go @@ -215,11 +215,16 @@ func TestMultiKVSetup(t *testing.T) { } } +// TODO Remove in Mimir 2.2. +// Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. +// We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, +// unless it's also defined in the limits, which is invalid. +// This needs to be set before setting default limits for unmarshalling. func TestActiveSeriesOverrides(t *testing.T) { tests := map[string]struct { config Config }{ - "Override with runtime config specified": { + "Override with runtime path specified": { config: func() Config { cfg := Config{} flagext.DefaultValues(&cfg) From f24c6d2f372c3146c827082090dc9b432e28f30b Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 29 Mar 2022 11:08:51 +0200 Subject: [PATCH 78/91] Removing accidentally commited file (again...) --- pkg/mimir/metrics-activity.log | Bin 1048576 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 pkg/mimir/metrics-activity.log diff --git a/pkg/mimir/metrics-activity.log b/pkg/mimir/metrics-activity.log deleted file mode 100644 index 9e0f96a2a253b173cb45b41868209a5d043e1437..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1048576 zcmeIuF#!Mo0K%a4Pi+Wah(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ z0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VK zfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5 zV8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM z7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b* z1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd z0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwA zz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEj zFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r z3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@ T0|pEjFkrxd0RsjM82APT0Pp|- From df0bb8ddaf949f3a3a9d1a8498aa6df4af9f30d6 Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 29 Mar 2022 14:01:50 +0200 Subject: [PATCH 79/91] Removing unused method clear. --- pkg/ingester/activeseries/active_series.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index 38a1203215..eb81ca7d2e 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -101,13 +101,6 @@ func (c *ActiveSeries) purge(keepUntil time.Time) { } } -//nolint // Linter reports that this method is unused, but it is. -func (c *ActiveSeries) clear() { - for s := 0; s < numStripes; s++ { - c.stripes[s].clear() - } -} - // Active returns the total number of active series, as well as a slice of active series matching each one of the // custom trackers provided (in the same order as custom trackers are defined). // The result is correct only if the third return value is true, which shows if enough time has passed since last reload. From c0149a76c9b96a89689856c1274a00cac4cafd5f Mon Sep 17 00:00:00 2001 From: Janos Date: Tue, 29 Mar 2022 14:33:05 +0200 Subject: [PATCH 80/91] Updating config-descriptor --- cmd/mimir/config-descriptor.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index c750e97ff2..84645341bb 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -2307,10 +2307,9 @@ "kind": "field", "name": "active_series_custom_trackers", "required": false, - "desc": "Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero).", + "desc": "[Deprecated] This config has been moved to the limits config, please set it there. Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero).", "fieldValue": null, "fieldDefaultValue": {}, - "fieldFlag": "ingester.active-series-custom-trackers", "fieldType": "map of tracker name (string) to matcher (string)", "fieldCategory": "advanced" }, @@ -2625,6 +2624,17 @@ "fieldType": "int", "fieldCategory": "experimental" }, + { + "kind": "field", + "name": "active_series_custom_trackers_config", + "required": false, + "desc": "Additional custom trackers for active metrics. If there are active series matching a provided matcher (map value), the count will be exposed in the custom trackers metric labeled using the tracker name (map key). Zero valued counts are not exposed (and removed when they go back to zero).", + "fieldValue": null, + "fieldDefaultValue": {}, + "fieldFlag": "ingester.active-series-custom-trackers", + "fieldType": "map of tracker name (string) to matcher (string)", + "fieldCategory": "advanced" + }, { "kind": "field", "name": "max_fetched_chunks_per_query", From 583d9dabac01cabae18ce71426563cd41ea43bbc Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 30 Mar 2022 09:54:19 +0200 Subject: [PATCH 81/91] WIP Fixing data race --- pkg/mimir/mimir_test.go | 46 +++++++++++++++++++++++++++++------------ 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/pkg/mimir/mimir_test.go b/pkg/mimir/mimir_test.go index bd11a416da..794a080b36 100644 --- a/pkg/mimir/mimir_test.go +++ b/pkg/mimir/mimir_test.go @@ -27,6 +27,7 @@ import ( "github.com/grafana/dskit/kv" "github.com/grafana/dskit/services" "github.com/grafana/dskit/test" + "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -168,6 +169,7 @@ func TestMimir(t *testing.T) { } func TestMimirServerShutdownWithActivityTrackerEnabled(t *testing.T) { + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) prepareGlobalMetricsRegistry(t) cfg := Config{} @@ -385,6 +387,7 @@ func TestGrpcAuthMiddleware(t *testing.T) { } func TestFlagDefaults(t *testing.T) { + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) c := Config{} f := flag.NewFlagSet("test", flag.PanicOnError) @@ -429,6 +432,19 @@ func TestFlagDefaults(t *testing.T) { require.Equal(t, 10*time.Second, c.Server.GRPCServerMinTimeBetweenPings) } +// TODO Remove in Mimir 2.2. +// Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. +// We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, +// unless it's also defined in the limits, which is invalid. +// This needs to be set before setting default limits for unmarshalling. +// For more context see https://github.com/grafana/mimir/pull/1188#discussion_r830129443 +func (t *Mimir) initTest() (services.Service, error) { + if t.Overrides.ActiveSeriesCustomTrackersConfig("1235").Empty() { + return nil, errors.New("Active series config should not be empty!") + } + return nil, nil +} + // TODO Remove in Mimir 2.2. // Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. // We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, @@ -436,9 +452,11 @@ func TestFlagDefaults(t *testing.T) { // This needs to be set before setting default limits for unmarshalling. // For more context see https://github.com/grafana/mimir/pull/1188#discussion_r830129443 func TestActiveSeriesDeprecationDefaultOverrideWithSomeRuntimeOverrides(t *testing.T) { + prepareGlobalMetricsRegistry(t) + flag.CommandLine = flag.NewFlagSet(os.Args[0], flag.ExitOnError) yamlContent := ` overrides: - '1234': + '1235': ingestion_burst_size: 15000 ingestion_rate: 1500 max_global_series_per_metric: 7000 @@ -446,8 +464,7 @@ overrides: ruler_max_rule_groups_per_tenant: 20 ruler_max_rules_per_rule_group: 20 ` - prepareGlobalMetricsRegistry(t) - + TEST_MODULE_NAME := "test" cfg := Config{} // This sets default values from flags to the config. @@ -456,17 +473,17 @@ overrides: // Creating test file with runtime overrides. tmpDir := t.TempDir() cfg.RuntimeConfig.LoadPath = filepath.Join(tmpDir, "overrides.yml") - err := ioutil.WriteFile(cfg.RuntimeConfig.LoadPath, []byte(yamlContent), 777) + err := ioutil.WriteFile(cfg.RuntimeConfig.LoadPath, []byte(yamlContent), 0777) require.NoError(t, err, "Failed to write test override.yml.") - // Setting up deprecated value as an ingester config value. + // Setting up tracker config value as an deprecated ingester config. cfg.Ingester.ActiveSeriesCustomTrackers, err = activeseries.NewCustomTrackersConfig(map[string]string{ "bool_is_true_flag-based": `{bool="true"}`, "bool_is_false_flagbased": `{bool="false"}`, }) require.NoError(t, err) - cfg.Target = []string{RuntimeConfig} + cfg.Target = []string{TEST_MODULE_NAME} cfg.Server = getServerConfig(t) require.NoError(t, cfg.Server.LogFormat.Set("logfmt")) require.NoError(t, cfg.Server.LogLevel.Set("debug")) @@ -474,19 +491,22 @@ overrides: c, err := New(cfg) require.NoError(t, err) + c.ModuleManager.RegisterModule(TEST_MODULE_NAME, c.initTest) + c.ModuleManager.AddDependency(TEST_MODULE_NAME, Overrides) errCh := make(chan error) go func() { errCh <- c.Run() }() - // Waiting for RuntimeConfig to load config from file. - test.Poll(t, cfg.RuntimeConfig.ReloadPeriod, true, func() interface{} { - return c.RuntimeConfig != nil - }) - assert.NotNil(t, c.RuntimeConfig.GetConfig()) - assert.False(t, c.TenantLimits.ByUserID("1234").ActiveSeriesCustomTrackersConfig.Empty(), - "Default deserialization value for active series trackers should be set from ingester config.") + select { + case <-time.After(5 * time.Second): + proc, err := os.FindProcess(os.Getpid()) + require.NoError(t, err) + require.NoError(t, proc.Signal(syscall.SIGINT)) + case err := <-errCh: + require.NoError(t, err, "Active series deprecation override not in place!") + } } // Generates server config, with gRPC listening on random port. From 273179ad97fb3206be87c5d6cbc0a4773b3319d9 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 30 Mar 2022 14:21:00 +0200 Subject: [PATCH 82/91] Fixing mimir_test.go --- pkg/mimir/mimir_test.go | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/pkg/mimir/mimir_test.go b/pkg/mimir/mimir_test.go index 794a080b36..64de1f71a7 100644 --- a/pkg/mimir/mimir_test.go +++ b/pkg/mimir/mimir_test.go @@ -25,6 +25,7 @@ import ( "github.com/go-kit/log" "github.com/grafana/dskit/flagext" "github.com/grafana/dskit/kv" + "github.com/grafana/dskit/modules" "github.com/grafana/dskit/services" "github.com/grafana/dskit/test" "github.com/pkg/errors" @@ -439,10 +440,18 @@ func TestFlagDefaults(t *testing.T) { // This needs to be set before setting default limits for unmarshalling. // For more context see https://github.com/grafana/mimir/pull/1188#discussion_r830129443 func (t *Mimir) initTest() (services.Service, error) { - if t.Overrides.ActiveSeriesCustomTrackersConfig("1235").Empty() { - return nil, errors.New("Active series config should not be empty!") - } - return nil, nil + + return services.NewTimerService(time.Second, + nil, + func(_ context.Context) error { + time.Sleep(100 * time.Millisecond) + if t.Overrides.ActiveSeriesCustomTrackersConfig("1235").Empty() { + return errors.New("Active series config should not be empty!") + } else { + return modules.ErrStopProcess + } + }, + nil), nil } // TODO Remove in Mimir 2.2. @@ -491,6 +500,7 @@ overrides: c, err := New(cfg) require.NoError(t, err) + // Creating a test module to ensure that runtime config check happens after initialization. c.ModuleManager.RegisterModule(TEST_MODULE_NAME, c.initTest) c.ModuleManager.AddDependency(TEST_MODULE_NAME, Overrides) @@ -501,9 +511,7 @@ overrides: select { case <-time.After(5 * time.Second): - proc, err := os.FindProcess(os.Getpid()) - require.NoError(t, err) - require.NoError(t, proc.Signal(syscall.SIGINT)) + require.Fail(t, "Mimir didn't stop in time") case err := <-errCh: require.NoError(t, err, "Active series deprecation override not in place!") } From c5311bbd77e4f560a3c80a4276bfb1f55a6fca10 Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 30 Mar 2022 14:28:32 +0200 Subject: [PATCH 83/91] Referencing issue in test to explain why sleep is needed --- pkg/mimir/mimir_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/mimir/mimir_test.go b/pkg/mimir/mimir_test.go index 64de1f71a7..7ee36a94e9 100644 --- a/pkg/mimir/mimir_test.go +++ b/pkg/mimir/mimir_test.go @@ -444,6 +444,7 @@ func (t *Mimir) initTest() (services.Service, error) { return services.NewTimerService(time.Second, nil, func(_ context.Context) error { + // Sleep to avoid issue https://github.com/grafana/dskit/issues/151 . time.Sleep(100 * time.Millisecond) if t.Overrides.ActiveSeriesCustomTrackersConfig("1235").Empty() { return errors.New("Active series config should not be empty!") From a8a230f0fe949d63852102d0161845505817ddbd Mon Sep 17 00:00:00 2001 From: Janos Date: Wed, 30 Mar 2022 14:39:50 +0200 Subject: [PATCH 84/91] Lint fix --- pkg/mimir/mimir_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pkg/mimir/mimir_test.go b/pkg/mimir/mimir_test.go index 7ee36a94e9..dff79c4e32 100644 --- a/pkg/mimir/mimir_test.go +++ b/pkg/mimir/mimir_test.go @@ -447,10 +447,9 @@ func (t *Mimir) initTest() (services.Service, error) { // Sleep to avoid issue https://github.com/grafana/dskit/issues/151 . time.Sleep(100 * time.Millisecond) if t.Overrides.ActiveSeriesCustomTrackersConfig("1235").Empty() { - return errors.New("Active series config should not be empty!") - } else { - return modules.ErrStopProcess + return errors.New("active series config should not be empty") } + return modules.ErrStopProcess }, nil), nil } @@ -474,7 +473,7 @@ overrides: ruler_max_rule_groups_per_tenant: 20 ruler_max_rules_per_rule_group: 20 ` - TEST_MODULE_NAME := "test" + TestModuleName := "test" cfg := Config{} // This sets default values from flags to the config. @@ -493,7 +492,7 @@ overrides: }) require.NoError(t, err) - cfg.Target = []string{TEST_MODULE_NAME} + cfg.Target = []string{TestModuleName} cfg.Server = getServerConfig(t) require.NoError(t, cfg.Server.LogFormat.Set("logfmt")) require.NoError(t, cfg.Server.LogLevel.Set("debug")) @@ -502,8 +501,9 @@ overrides: c, err := New(cfg) require.NoError(t, err) // Creating a test module to ensure that runtime config check happens after initialization. - c.ModuleManager.RegisterModule(TEST_MODULE_NAME, c.initTest) - c.ModuleManager.AddDependency(TEST_MODULE_NAME, Overrides) + c.ModuleManager.RegisterModule(TestModuleName, c.initTest) + err = c.ModuleManager.AddDependency(TestModuleName, Overrides) + require.NoError(t, err) errCh := make(chan error) go func() { From 737a08b077f73f371385ccf5e2d0e35d6dc9a1df Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 31 Mar 2022 09:44:43 +0200 Subject: [PATCH 85/91] Adding entry to CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d53c7bbdaa..39c12bde0e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * [CHANGE] Compactor: No longer upload debug meta files to object storage. #1257 * [FEATURE] Ruler: Allow setting `evaluation_delay` for each rule group via rules group configuration file. #1474 * [FEATURE] Distributor: Added the ability to forward specifics metrics to alternative remote_write API endpoints. #1052 +* [FEATURE] Ingester: Active series custom trackers now supports runtime override. The configuration has been moved to limit config, the ingester config has been deprecated. #1188 * [ENHANCEMENT] Ruler: Add more detailed query information to ruler query stats logging. #1411 * [ENHANCEMENT] Admin: Admin API now has some styling. #1482 #1549 * [BUGFIX] Query-frontend: do not shard queries with a subquery unless the subquery is inside a shardable aggregation function call. #1542 From 444d0a989754b302759003a95babf37ff348a363 Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 31 Mar 2022 09:47:34 +0200 Subject: [PATCH 86/91] Fixup for the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39c12bde0e..0b8a16dd8e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * [CHANGE] Compactor: No longer upload debug meta files to object storage. #1257 * [FEATURE] Ruler: Allow setting `evaluation_delay` for each rule group via rules group configuration file. #1474 * [FEATURE] Distributor: Added the ability to forward specifics metrics to alternative remote_write API endpoints. #1052 -* [FEATURE] Ingester: Active series custom trackers now supports runtime override. The configuration has been moved to limit config, the ingester config has been deprecated. #1188 +* [FEATURE] Ingester: Active series custom trackers now supports runtime and tenant-specific overrides. The configuration has been moved to limit config, the ingester config has been deprecated. #1188 * [ENHANCEMENT] Ruler: Add more detailed query information to ruler query stats logging. #1411 * [ENHANCEMENT] Admin: Admin API now has some styling. #1482 #1549 * [BUGFIX] Query-frontend: do not shard queries with a subquery unless the subquery is inside a shardable aggregation function call. #1542 From 5fb103f0d64e191c8589e0fd8fd0c7937d641aae Mon Sep 17 00:00:00 2001 From: Janos Date: Thu, 31 Mar 2022 09:52:40 +0200 Subject: [PATCH 87/91] More precise wording for the changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b8a16dd8e..dc87873b13 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ * [CHANGE] Compactor: No longer upload debug meta files to object storage. #1257 * [FEATURE] Ruler: Allow setting `evaluation_delay` for each rule group via rules group configuration file. #1474 * [FEATURE] Distributor: Added the ability to forward specifics metrics to alternative remote_write API endpoints. #1052 -* [FEATURE] Ingester: Active series custom trackers now supports runtime and tenant-specific overrides. The configuration has been moved to limit config, the ingester config has been deprecated. #1188 +* [FEATURE] Ingester: Active series custom trackers now supports runtime tenant-specific overrides. The configuration has been moved to limit config, the ingester config has been deprecated. #1188 * [ENHANCEMENT] Ruler: Add more detailed query information to ruler query stats logging. #1411 * [ENHANCEMENT] Admin: Admin API now has some styling. #1482 #1549 * [BUGFIX] Query-frontend: do not shard queries with a subquery unless the subquery is inside a shardable aggregation function call. #1542 From b04a320a8f7dbc34b9fd664fb43cb657d63746d2 Mon Sep 17 00:00:00 2001 From: Janos Gub Date: Thu, 31 Mar 2022 17:43:43 +0200 Subject: [PATCH 88/91] Comment fix based on Oleg's suggestion Co-authored-by: Oleg Zaytsev --- pkg/mimir/modules.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 662703b5a9..2224ab173a 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -214,7 +214,7 @@ func (t *Mimir) initRuntimeConfig() (services.Service, error) { // Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. // We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, // unless it's also defined in the limits, which is invalid. - // This needs to be set before setting default limits for unmarshalling. + // This needs to be set before setting default limits for unmarshalling. if !t.Cfg.Ingester.ActiveSeriesCustomTrackers.Empty() { if !t.Cfg.LimitsConfig.ActiveSeriesCustomTrackersConfig.Empty() { return nil, fmt.Errorf("can't define active series custom trackers in both ingester and limits config, please define them only in the limits") From 28e3b2a7f45df1bcfcc2d3256dd790b8d0d3d708 Mon Sep 17 00:00:00 2001 From: Janos Date: Fri, 1 Apr 2022 08:38:36 +0200 Subject: [PATCH 89/91] Lintfix after online editor conflict resolve --- pkg/ingester/ingester.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 3d6e9e560f..87b1becf36 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -49,7 +49,7 @@ import ( "github.com/grafana/dskit/tenant" - "github.com/grafana/mimir/pkg/ingester/activeseries" + "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/ingester/client" "github.com/grafana/mimir/pkg/mimirpb" "github.com/grafana/mimir/pkg/storage/bucket" From a043d12286f6e4e26f2cab537f71dc75d8ecce44 Mon Sep 17 00:00:00 2001 From: Janos Date: Mon, 4 Apr 2022 08:54:40 +0200 Subject: [PATCH 90/91] Modifications based on pstibrany's comments --- cmd/mimir/usage.go | 2 ++ pkg/ingester/activeseries/active_series.go | 14 ++++++-------- pkg/mimir/mimir_test.go | 11 +++-------- pkg/mimir/modules.go | 2 +- pkg/mimir/modules_test.go | 2 +- 5 files changed, 13 insertions(+), 18 deletions(-) diff --git a/cmd/mimir/usage.go b/cmd/mimir/usage.go index 542692c101..bd1c2343d6 100644 --- a/cmd/mimir/usage.go +++ b/cmd/mimir/usage.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/dskit/flagext" + "github.com/grafana/mimir/pkg/ingester/activeseries" "github.com/grafana/mimir/pkg/mimir" "github.com/grafana/mimir/pkg/util/fieldcategory" ) @@ -150,6 +151,7 @@ func parseStructure(structure interface{}, fields map[uintptr]reflect.StructFiel // then gets confused. var ignoredStructTypes = []reflect.Type{ reflect.TypeOf(flagext.Secret{}), + reflect.TypeOf(activeseries.CustomTrackersConfig{}), } func ignoreStructType(fieldType reflect.Type) bool { diff --git a/pkg/ingester/activeseries/active_series.go b/pkg/ingester/activeseries/active_series.go index eb81ca7d2e..27219d5890 100644 --- a/pkg/ingester/activeseries/active_series.go +++ b/pkg/ingester/activeseries/active_series.go @@ -245,8 +245,8 @@ func (s *seriesStripe) purge(keepUntil time.Time) { s.mu.Lock() defer s.mu.Unlock() - active := 0 - activeMatching := resizeAndClear(len(s.activeMatching), s.activeMatching) + s.active = 0 + s.activeMatching = resizeAndClear(len(s.activeMatching), s.activeMatching) oldest := int64(math.MaxInt64) for fp, entries := range s.refs { @@ -259,10 +259,10 @@ func (s *seriesStripe) purge(keepUntil time.Time) { continue } - active++ + s.active++ for i, ok := range entries[0].matches { if ok { - activeMatching[i]++ + s.activeMatching[i]++ } } if ts < oldest { @@ -290,11 +290,11 @@ func (s *seriesStripe) purge(keepUntil time.Time) { if cnt := len(entries); cnt == 0 { delete(s.refs, fp) } else { - active += cnt + s.active += cnt for i := range entries { for i, ok := range entries[i].matches { if ok { - activeMatching[i]++ + s.activeMatching[i]++ } } } @@ -308,8 +308,6 @@ func (s *seriesStripe) purge(keepUntil time.Time) { } else { s.oldestEntryTs.Store(oldest) } - s.active = active - s.activeMatching = activeMatching } func resizeAndClear(l int, prev []int) []int { diff --git a/pkg/mimir/mimir_test.go b/pkg/mimir/mimir_test.go index dff79c4e32..a6cae907b0 100644 --- a/pkg/mimir/mimir_test.go +++ b/pkg/mimir/mimir_test.go @@ -433,15 +433,10 @@ func TestFlagDefaults(t *testing.T) { require.Equal(t, 10*time.Second, c.Server.GRPCServerMinTimeBetweenPings) } -// TODO Remove in Mimir 2.2. -// Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. -// We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, -// unless it's also defined in the limits, which is invalid. -// This needs to be set before setting default limits for unmarshalling. -// For more context see https://github.com/grafana/mimir/pull/1188#discussion_r830129443 +// TODO Remove in Mimir 2.3. func (t *Mimir) initTest() (services.Service, error) { - return services.NewTimerService(time.Second, + return services.NewBasicService( nil, func(_ context.Context) error { // Sleep to avoid issue https://github.com/grafana/dskit/issues/151 . @@ -454,7 +449,7 @@ func (t *Mimir) initTest() (services.Service, error) { nil), nil } -// TODO Remove in Mimir 2.2. +// TODO Remove in Mimir 2.3. // Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. // We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, // unless it's also defined in the limits, which is invalid. diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 2224ab173a..c1f4181b90 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -210,7 +210,7 @@ func (t *Mimir) initRing() (serv services.Service, err error) { } func (t *Mimir) initRuntimeConfig() (services.Service, error) { - // TODO Remove in Mimir 2.2. + // TODO Remove in Mimir 2.3. // Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. // We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, // unless it's also defined in the limits, which is invalid. diff --git a/pkg/mimir/modules_test.go b/pkg/mimir/modules_test.go index ae19855a2d..c584755248 100644 --- a/pkg/mimir/modules_test.go +++ b/pkg/mimir/modules_test.go @@ -215,7 +215,7 @@ func TestMultiKVSetup(t *testing.T) { } } -// TODO Remove in Mimir 2.2. +// TODO Remove in Mimir 2.3. // Previously ActiveSeriesCustomTrackers was an ingester config, now it's in LimitsConfig. // We provide backwards compatibility for it by parsing the old YAML location and copying it to LimitsConfig here, // unless it's also defined in the limits, which is invalid. From 48bb8ea17b253e75ba05c1bd2cd208dfd7fbed37 Mon Sep 17 00:00:00 2001 From: Janos Date: Mon, 4 Apr 2022 09:29:50 +0200 Subject: [PATCH 91/91] Fixing reference-help --- cmd/mimir/help.txt.tmpl | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/mimir/help.txt.tmpl b/cmd/mimir/help.txt.tmpl index f2371f2834..62f1962170 100644 --- a/cmd/mimir/help.txt.tmpl +++ b/cmd/mimir/help.txt.tmpl @@ -263,8 +263,6 @@ Usage of ./cmd/mimir/mimir: Print basic help. -help-all Print help, also including advanced and experimental parameters. - -ingester.active-series-custom-trackers value - Additional active series metrics, matching the provided matchers. Matchers should be in form :, like 'foobar:{foo="bar"}'. Multiple matchers can be provided either providing the flag multiple times or providing multiple semicolon-separated values to a single flag. -ingester.max-global-metadata-per-metric int The maximum number of metadata per metric, across the cluster. 0 to disable. -ingester.max-global-metadata-per-user int