Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Fixed panic when Mimir is running in single binary mode and results cache is enabled #1704

Merged
merged 2 commits into from
Apr 14, 2022
Merged

Fixed panic when Mimir is running in single binary mode and results cache is enabled #1704

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8769ad7
Fixed panic when Mimir is running in single binary mode and results c…
pracucci Apr 14, 2022
39aa35e
Added integration test
pracucci Apr 14, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
* [ENHANCEMENT] Admin: Admin API now has some styling. #1482 #1549
* [ENHANCEMENT] Alertmanager: added `insight=true` field to alertmanager dispatch logs. #1379
* [BUGFIX] Query-frontend: do not shard queries with a subquery unless the subquery is inside a shardable aggregation function call. #1542
* [BUGFIX] Query-frontend: added `component=query-frontend` label to results cache memcached metrics to fix a panic when Mimir is running in single binary mode and results cache is enabled. #1704
* [BUGFIX] Mimir: services' status content-type is now correctly set to `text/html`. #1575
* [BUGFIX] Multikv: Fix panic when using using runtime config to set primary KV store used by `multi` KV. #1587
* [BUGFIX] Multikv: Fix watching for runtime config changes in `multi` KV store in ruler and querier. #1665
Expand Down
60 changes: 60 additions & 0 deletions integration/single_binary_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
// SPDX-License-Identifier: AGPL-3.0-only
//go:build requires_docker
// +build requires_docker

package integration

import (
"testing"

"github.com/grafana/e2e"
e2ecache "github.com/grafana/e2e/cache"
e2edb "github.com/grafana/e2e/db"
"github.com/stretchr/testify/require"

"github.com/grafana/mimir/integration/e2emimir"
)

func TestMimirShouldStartInSingleBinaryModeWithAllMemcachedConfigured(t *testing.T) {
s, err := e2e.NewScenario(networkName)
require.NoError(t, err)
defer s.Close()

// Start dependencies.
consul := e2edb.NewConsul()
minio := e2edb.NewMinio(9000, blocksBucketName)
memcached := e2ecache.NewMemcached()
require.NoError(t, s.StartAndWaitReady(consul, minio, memcached))

flags := mergeFlags(BlocksStorageFlags(), map[string]string{
// Memcached.
"-query-frontend.cache-results": "true",
"-query-frontend.results-cache.backend": "memcached",
"-query-frontend.results-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort),
"-blocks-storage.bucket-store.metadata-cache.backend": "memcached",
"-blocks-storage.bucket-store.metadata-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort),
"-blocks-storage.bucket-store.index-cache.backend": "memcached",
"-blocks-storage.bucket-store.index-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort),
"-blocks-storage.bucket-store.chunks-cache.backend": "memcached",
"-blocks-storage.bucket-store.chunks-cache.memcached.addresses": "dns+" + memcached.NetworkEndpoint(e2ecache.MemcachedPort),
// Ingester.
"-ingester.ring.store": "consul",
"-ingester.ring.consul.hostname": consul.NetworkHTTPEndpoint(),
// Distributor.
"-ingester.ring.replication-factor": "2",
"-distributor.ring.store": "consul",
"-distributor.ring.consul.hostname": consul.NetworkHTTPEndpoint(),
// Store-gateway.
"-store-gateway.sharding-ring.store": "consul",
"-store-gateway.sharding-ring.consul.hostname": consul.NetworkHTTPEndpoint(),
"-store-gateway.sharding-ring.replication-factor": "1",
// Compactor.
"-compactor.ring.store": "consul",
"-compactor.ring.consul.hostname": consul.NetworkHTTPEndpoint(),
"-compactor.cleanup-interval": "2s", // Update bucket index often.
})

// Ensure Mimir successfully starts.
mimir := e2emimir.NewSingleBinary("mimir-1", e2e.MergeFlags(DefaultSingleBinaryFlags(), flags))
require.NoError(t, s.StartAndWaitReady(mimir))
}
5 changes: 5 additions & 0 deletions pkg/frontend/querymiddleware/results_cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/prometheus/prometheus/model/timestamp"
"github.com/prometheus/prometheus/promql"
"github.com/prometheus/prometheus/promql/parser"
"github.com/thanos-io/thanos/pkg/extprom"
"github.com/uber/jaeger-client-go"

"github.com/grafana/mimir/pkg/cache"
Expand Down Expand Up @@ -82,6 +83,10 @@ func errUnsupportedResultsCacheBackend(unsupportedBackend string) error {

// newResultsCache creates a new results cache based on the input configuration.
func newResultsCache(cfg ResultsCacheConfig, logger log.Logger, reg prometheus.Registerer) (cache.Cache, error) {
// Add the "component" label similarly to other components, so that metrics don't clash and have the same labels set
// when running in single binary mode.
reg = extprom.WrapRegistererWith(prometheus.Labels{"component": "query-frontend"}, reg)

client, err := cache.CreateClient("frontend-cache", cfg.BackendConfig, logger, reg)
if err != nil {
return nil, err
Expand Down