Alertmanager: Allow usage with local storage type, with appropriate warnings.

CHANGELOG.md

@@ -76,6 +76,7 @@
 * [BUGFIX] Multikv: Fix watching for runtime config changes in `multi` KV store in ruler and querier. #1665
 * [BUGFIX] Memcached: allow to use CNAME DNS records for the memcached backend addresses. #1654
 * [BUGFIX] Querier: fixed temporary partial query results when shuffle sharding is enabled and hash ring backend storage is flushed / reset. #1829
+* [BUGFUX] Alertmanager: Allow usage with `-alertmanager-storage.backend=local`. Note that when using this storage type, the Alertmanager is not able persist state remotely, so it not recommended for production use. #1836
 
 ### Mixin
 

integration/alertmanager_test.go

@@ -125,6 +125,43 @@ func TestAlertmanager(t *testing.T) {
 	require.Equal(t, "Accept-Encoding", res.Header.Get("Vary"))
 }
 
+func TestAlertmanagerLocalStore(t *testing.T) {
+	s, err := e2e.NewScenario(networkName)
+	require.NoError(t, err)
+	defer s.Close()
+
+	consul := e2edb.NewConsul()
+	require.NoError(t, s.StartAndWaitReady(consul))
+
+	require.NoError(t, writeFileToSharedDir(s, "alertmanager_configs/user-1.yaml", []byte(mimirAlertmanagerUserConfigYaml)))
+
+	alertmanager := e2emimir.NewAlertmanager(
+		"alertmanager",
+		mergeFlags(
+			AlertmanagerFlags(),
+			AlertmanagerLocalFlags(),
+			AlertmanagerShardingFlags(consul.NetworkHTTPEndpoint(), 1),
+		),
+	)
+	require.NoError(t, s.StartAndWaitReady(alertmanager))
+	require.NoError(t, alertmanager.WaitSumMetrics(e2e.Equals(1), "cortex_alertmanager_config_last_reload_successful"))
+	require.NoError(t, alertmanager.WaitSumMetrics(e2e.Greater(0), "cortex_alertmanager_config_hash"))
+
+	c, err := e2emimir.NewClient("", "", alertmanager.HTTPEndpoint(), "", "user-1")
+	require.NoError(t, err)
+
+	cfg, err := c.GetAlertmanagerConfig(context.Background())
+	require.NoError(t, err)
+
+	// Ensure the returned status config matches alertmanager_test_fixtures/user-1.yaml
+	require.NotNil(t, cfg)
+	require.Equal(t, "example_receiver", cfg.Route.Receiver)
+	require.Len(t, cfg.Route.GroupByStr, 1)
+	require.Equal(t, "example_groupby", cfg.Route.GroupByStr[0])
+	require.Len(t, cfg.Receivers, 1)
+	require.Equal(t, "example_receiver", cfg.Receivers[0].Name)
+}
+
 func TestAlertmanagerStoreAPI(t *testing.T) {
 	s, err := e2e.NewScenario(networkName)
 	require.NoError(t, err)

integration/configs.go

@@ -10,10 +10,12 @@ package integration
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"text/template"
 
+	e2e "github.com/grafana/e2e"
 	e2edb "github.com/grafana/e2e/db"
 )
 
@@ -95,6 +97,13 @@ var (
 		}
 	}
 
+	AlertmanagerLocalFlags = func() map[string]string {
+		return map[string]string{
+			"-alertmanager-storage.backend":    "local",
+			"-alertmanager-storage.local.path": filepath.Join(e2e.ContainerSharedDir, "alertmanager_configs"),
+		}
+	}
+
 	AlertmanagerS3Flags = func() map[string]string {
 		return map[string]string{
 			"-alertmanager-storage.backend":              "s3",

pkg/alertmanager/alertstore/config.go

@@ -26,13 +26,3 @@ func (cfg *Config) RegisterFlags(f *flag.FlagSet) {
 	cfg.Local.RegisterFlagsWithPrefix(prefix, f)
 	cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "alertmanager", f)
 }
-
-// IsFullStateSupported returns if the given configuration supports access to FullState objects.
-func (cfg *Config) IsFullStateSupported() bool {
-	for _, backend := range bucket.SupportedBackends {
-		if cfg.Backend == backend {
-			return true
-		}
-	}
-	return false
-}

pkg/alertmanager/alertstore/local/store.go

@@ -108,12 +108,12 @@ func (f *Store) DeleteAlertConfig(_ context.Context, user string) error {
 
 // ListUsersWithFullState implements alertstore.AlertStore.
 func (f *Store) ListUsersWithFullState(ctx context.Context) ([]string, error) {
-	return nil, errState
+	return []string{}, nil
 }
 
 // GetFullState implements alertstore.AlertStore.
 func (f *Store) GetFullState(ctx context.Context, user string) (alertspb.FullStateDesc, error) {
-	return alertspb.FullStateDesc{}, errState
+	return alertspb.FullStateDesc{}, alertspb.ErrNotFound
 }
 
 // SetFullState implements alertstore.AlertStore.

pkg/alertmanager/alertstore/local/store_test.go

@@ -108,6 +108,30 @@ func TestStore_GetAlertConfigs(t *testing.T) {
 	}
 }
 
+func TestStore_FullState(t *testing.T) {
+	ctx := context.Background()
+	store, _ := prepareLocalStore(t)
+
+	// FullState not persisted - List always returns no users.
+
+	configs, err := store.ListUsersWithFullState(ctx)
+	require.NoError(t, err)
+	assert.Empty(t, configs)
+
+	// FullState not persisted - Get always returns NotFound.
+
+	_, err = store.GetFullState(ctx, "user-1")
+	require.ErrorIs(t, err, alertspb.ErrNotFound)
+
+	// Any attempt to write the store fails.
+
+	err = store.SetFullState(ctx, "user-1", alertspb.FullStateDesc{})
+	require.ErrorIs(t, err, errState)
+
+	err = store.DeleteFullState(ctx, "user-1")
+	require.ErrorIs(t, err, errState)
+}
+
 func prepareLocalStore(t *testing.T) (store *Store, storeDir string) {
 	var err error
 

pkg/alertmanager/alertstore/store.go

@@ -55,6 +55,7 @@ type AlertStore interface {
 // NewAlertStore returns a alertmanager store backend client based on the provided cfg.
 func NewAlertStore(ctx context.Context, cfg Config, cfgProvider bucket.TenantConfigProvider, logger log.Logger, reg prometheus.Registerer) (AlertStore, error) {
 	if cfg.Backend == local.Name {
+		level.Warn(logger).Log("msg", "-alertmanager-storage.backend=local is not suitable for persisting alertmanager state between replicas (silences, notifications); you should switch to an external object store for production use")
 		return local.NewStore(cfg.Local)
 	}
 

pkg/alertmanager/multitenant.go

@@ -58,7 +58,6 @@ const (
 var (
 	errEmptyExternalURL                    = errors.New("-alertmanager.web.external-url cannot be empty")
 	errInvalidExternalURL                  = errors.New("the configured external URL is invalid: should not end with /")
-	errShardingUnsupportedStorage          = errors.New("the configured alertmanager storage backend is not supported when sharding is enabled")
 	errZoneAwarenessEnabledWithoutZoneInfo = errors.New("the configured alertmanager has zone awareness enabled but zone is not set")
 	errNotUploadingFallback                = errors.New("not uploading fallback configuration")
 )
@@ -129,9 +128,6 @@ func (cfg *MultitenantAlertmanagerConfig) Validate(storageCfg alertstore.Config)
 		return err
 	}
 
-	if !storageCfg.IsFullStateSupported() {
-		return errShardingUnsupportedStorage
-	}
 	if cfg.ShardingRing.ZoneAwarenessEnabled && cfg.ShardingRing.InstanceZone == "" {
 		return errZoneAwarenessEnabledWithoutZoneInfo
 	}

pkg/alertmanager/multitenant_test.go

@@ -165,11 +165,11 @@ func TestMultitenantAlertmanagerConfig_Validate(t *testing.T) {
 			},
 			expected: nil,
 		},
-		"should fail if new storage store configuration given with local type": {
+		"should succeed if new storage store configuration given with local type": {
 			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {
 				storageCfg.Backend = "local"
 			},
-			expected: errShardingUnsupportedStorage,
+			expected: nil,
 		},
 		"should fail if zone aware is enabled but zone is not set": {
 			setup: func(t *testing.T, cfg *MultitenantAlertmanagerConfig, storageCfg *alertstore.Config) {