diff --git a/cmd/metaconvert/main.go b/cmd/metaconvert/main.go index 2c037f3a49..404290966d 100644 --- a/cmd/metaconvert/main.go +++ b/cmd/metaconvert/main.go @@ -42,8 +42,9 @@ func main() { cfg := config{} cfg.LogLevel.RegisterFlags(flag.CommandLine) + initLogger := log.NewDefaultLogger(cfg.LogLevel, cfg.LogFormat) cfg.LogFormat.RegisterFlags(flag.CommandLine) - cfg.BucketConfig.RegisterFlags(flag.CommandLine) + cfg.BucketConfig.RegisterFlags(flag.CommandLine, initLogger) flag.BoolVar(&cfg.DryRun, "dry-run", false, "Don't make changes; only report what needs to be done") flag.StringVar(&cfg.Tenant, "tenant", "", "Tenant to process") diff --git a/cmd/mimir/config-descriptor.json b/cmd/mimir/config-descriptor.json index a03be5bc58..be42c52303 100644 --- a/cmd/mimir/config-descriptor.json +++ b/cmd/mimir/config-descriptor.json @@ -4204,10 +4204,9 @@ "kind": "field", "name": "msi_resource", "required": false, - "desc": "If set, this URL is used instead of https://\u003cstorage-account-name\u003e.\u003cendpoint-suffix\u003e for obtaining ServicePrincipalToken from MSI.", + "desc": "", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "blocks-storage.azure.msi-resource", "fieldType": "string", "fieldCategory": "advanced" }, @@ -8018,10 +8017,9 @@ "kind": "field", "name": "msi_resource", "required": false, - "desc": "If set, this URL is used instead of https://\u003cstorage-account-name\u003e.\u003cendpoint-suffix\u003e for obtaining ServicePrincipalToken from MSI.", + "desc": "", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "ruler-storage.azure.msi-resource", "fieldType": "string", "fieldCategory": "advanced" }, @@ -9347,10 +9345,9 @@ "kind": "field", "name": "msi_resource", "required": false, - "desc": "If set, this URL is used instead of https://\u003cstorage-account-name\u003e.\u003cendpoint-suffix\u003e for obtaining ServicePrincipalToken from MSI.", + "desc": "", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "alertmanager-storage.azure.msi-resource", "fieldType": "string", "fieldCategory": "advanced" }, @@ -10585,10 +10582,9 @@ "kind": "field", "name": "msi_resource", "required": false, - "desc": "If set, this URL is used instead of https://\u003cstorage-account-name\u003e.\u003cendpoint-suffix\u003e for obtaining ServicePrincipalToken from MSI.", + "desc": "", "fieldValue": null, "fieldDefaultValue": "", - "fieldFlag": "common.storage.azure.msi-resource", "fieldType": "string", "fieldCategory": "advanced" }, diff --git a/cmd/mimir/help-all.txt.tmpl b/cmd/mimir/help-all.txt.tmpl index 07831a4e39..36429de4c6 100644 --- a/cmd/mimir/help-all.txt.tmpl +++ b/cmd/mimir/help-all.txt.tmpl @@ -13,8 +13,6 @@ Usage of ./cmd/mimir/mimir: Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN. If set to empty string, default endpoint suffix is used. -alertmanager-storage.azure.max-retries int Number of retries for recoverable errors (default 20) - -alertmanager-storage.azure.msi-resource string - If set, this URL is used instead of https://. for obtaining ServicePrincipalToken from MSI. -alertmanager-storage.azure.user-assigned-id string User assigned identity. If empty, then System assigned identity is used. -alertmanager-storage.backend string @@ -258,8 +256,6 @@ Usage of ./cmd/mimir/mimir: Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN. If set to empty string, default endpoint suffix is used. -blocks-storage.azure.max-retries int Number of retries for recoverable errors (default 20) - -blocks-storage.azure.msi-resource string - If set, this URL is used instead of https://. for obtaining ServicePrincipalToken from MSI. -blocks-storage.azure.user-assigned-id string User assigned identity. If empty, then System assigned identity is used. -blocks-storage.backend string @@ -531,8 +527,6 @@ Usage of ./cmd/mimir/mimir: Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN. If set to empty string, default endpoint suffix is used. -common.storage.azure.max-retries int Number of retries for recoverable errors (default 20) - -common.storage.azure.msi-resource string - If set, this URL is used instead of https://. for obtaining ServicePrincipalToken from MSI. -common.storage.azure.user-assigned-id string User assigned identity. If empty, then System assigned identity is used. -common.storage.backend string @@ -1370,8 +1364,6 @@ Usage of ./cmd/mimir/mimir: Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN. If set to empty string, default endpoint suffix is used. -ruler-storage.azure.max-retries int Number of retries for recoverable errors (default 20) - -ruler-storage.azure.msi-resource string - If set, this URL is used instead of https://. for obtaining ServicePrincipalToken from MSI. -ruler-storage.azure.user-assigned-id string User assigned identity. If empty, then System assigned identity is used. -ruler-storage.backend string diff --git a/docs/sources/operators-guide/configure/reference-configuration-parameters/index.md b/docs/sources/operators-guide/configure/reference-configuration-parameters/index.md index a651f613b4..b3eff0b62f 100644 --- a/docs/sources/operators-guide/configure/reference-configuration-parameters/index.md +++ b/docs/sources/operators-guide/configure/reference-configuration-parameters/index.md @@ -3567,10 +3567,7 @@ The `azure_storage_backend` block configures the connection to Azure object stor # CLI flag: -.azure.max-retries [max_retries: | default = 20] -# (advanced) If set, this URL is used instead of -# https://. for obtaining -# ServicePrincipalToken from MSI. -# CLI flag: -.azure.msi-resource +# (advanced) [msi_resource: | default = ""] # (advanced) User assigned identity. If empty, then System assigned identity is diff --git a/pkg/alertmanager/alertstore/config.go b/pkg/alertmanager/alertstore/config.go index a0310ff9c0..a7779b47ae 100644 --- a/pkg/alertmanager/alertstore/config.go +++ b/pkg/alertmanager/alertstore/config.go @@ -8,6 +8,8 @@ package alertstore import ( "flag" + "github.com/go-kit/log" + "github.com/grafana/mimir/pkg/alertmanager/alertstore/local" "github.com/grafana/mimir/pkg/storage/bucket" ) @@ -19,10 +21,10 @@ type Config struct { } // RegisterFlags registers the backend storage config. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { +func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { prefix := "alertmanager-storage." cfg.StorageBackendConfig.ExtraBackends = []string{local.Name} cfg.Local.RegisterFlagsWithPrefix(prefix, f) - cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "alertmanager", f) + cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "alertmanager", f, logger) } diff --git a/pkg/mimir/mimir.go b/pkg/mimir/mimir.go index c8c1e738d1..c78812bb70 100644 --- a/pkg/mimir/mimir.go +++ b/pkg/mimir/mimir.go @@ -154,22 +154,22 @@ func (c *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { c.LimitsConfig.RegisterFlags(f) c.Worker.RegisterFlags(f) c.Frontend.RegisterFlags(f, logger) - c.BlocksStorage.RegisterFlags(f) + c.BlocksStorage.RegisterFlags(f, logger) c.Compactor.RegisterFlags(f, logger) c.StoreGateway.RegisterFlags(f, logger) c.TenantFederation.RegisterFlags(f) c.Ruler.RegisterFlags(f, logger) - c.RulerStorage.RegisterFlags(f) + c.RulerStorage.RegisterFlags(f, logger) c.Alertmanager.RegisterFlags(f, logger) - c.AlertmanagerStorage.RegisterFlags(f) + c.AlertmanagerStorage.RegisterFlags(f, logger) c.RuntimeConfig.RegisterFlags(f) c.MemberlistKV.RegisterFlags(f) c.ActivityTracker.RegisterFlags(f) c.QueryScheduler.RegisterFlags(f) c.UsageStats.RegisterFlags(f) - c.Common.RegisterFlags(f) + c.Common.RegisterFlags(f, logger) } func (c *Config) CommonConfigInheritance() CommonConfigInheritance { @@ -576,8 +576,8 @@ type CommonConfigInheritance struct { } // RegisterFlags registers flag. -func (c *CommonConfig) RegisterFlags(f *flag.FlagSet) { - c.Storage.RegisterFlagsWithPrefix("common.storage.", f) +func (c *CommonConfig) RegisterFlags(f *flag.FlagSet, logger log.Logger) { + c.Storage.RegisterFlagsWithPrefix("common.storage.", f, logger) } // configWithCustomCommonUnmarshaler unmarshals config with custom unmarshaler for the `common` field. diff --git a/pkg/mimir/mimir_config_test.go b/pkg/mimir/mimir_config_test.go index 0d36ece941..ff6b3571b6 100644 --- a/pkg/mimir/mimir_config_test.go +++ b/pkg/mimir/mimir_config_test.go @@ -61,7 +61,7 @@ type customExtendedConfig struct { func (c *customExtendedConfig) RegisterFlags(f *flag.FlagSet, logger log.Logger) { c.MimirConfig.RegisterFlags(f, logger) - c.CustomStorage.RegisterFlagsWithPrefix("custom-storage", f) + c.CustomStorage.RegisterFlagsWithPrefix("custom-storage", f, logger) } func (c *customExtendedConfig) CommonConfigInheritance() mimir.CommonConfigInheritance { diff --git a/pkg/mimirtool/commands/bucket_validation.go b/pkg/mimirtool/commands/bucket_validation.go index f7757a1799..ca30da0008 100644 --- a/pkg/mimirtool/commands/bucket_validation.go +++ b/pkg/mimirtool/commands/bucket_validation.go @@ -105,19 +105,19 @@ func (b *BucketValidationCommand) Register(app *kingpin.Application, _ EnvVarNam } func (b *BucketValidationCommand) validate(k *kingpin.ParseContext) error { + b.logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) if b.bucketConfigHelp { - b.printBucketConfigHelp() + b.printBucketConfigHelp(b.logger) return nil } - err := b.parseBucketConfig() + err := b.parseBucketConfig(b.logger) if err != nil { return errors.Wrap(err, "error when parsing bucket config") } b.setObjectNames() b.objectContent = "testData" - b.logger = log.NewLogfmtLogger(log.NewSyncWriter(os.Stderr)) ctx := context.Background() bucketClient, err := bucket.NewClient(ctx, b.cfg, "testClient", b.logger, nil) @@ -163,9 +163,9 @@ func (b *BucketValidationCommand) validate(k *kingpin.ParseContext) error { return nil } -func (b *BucketValidationCommand) printBucketConfigHelp() { +func (b *BucketValidationCommand) printBucketConfigHelp(logger log.Logger) { fs := flag.NewFlagSet("bucket-config", flag.ContinueOnError) - b.cfg.RegisterFlags(fs) + b.cfg.RegisterFlags(fs, logger) fmt.Fprintf(fs.Output(), ` The following help text describes the arguments @@ -179,9 +179,9 @@ mimirtool bucket-validation --bucket-config='-backend=s3 -s3.endpoint=localhost: fs.Usage() } -func (b *BucketValidationCommand) parseBucketConfig() error { +func (b *BucketValidationCommand) parseBucketConfig(logger log.Logger) error { fs := flag.NewFlagSet("bucket-config", flag.ContinueOnError) - b.cfg.RegisterFlags(fs) + b.cfg.RegisterFlags(fs, logger) err := fs.Parse(strings.Split(b.bucketConfig, " ")) if err != nil { return err diff --git a/pkg/ruler/rulestore/config.go b/pkg/ruler/rulestore/config.go index 8ababafd90..700c904a9e 100644 --- a/pkg/ruler/rulestore/config.go +++ b/pkg/ruler/rulestore/config.go @@ -9,6 +9,7 @@ import ( "flag" "reflect" + "github.com/go-kit/log" "github.com/grafana/dskit/flagext" "github.com/grafana/mimir/pkg/ruler/rulestore/local" @@ -22,12 +23,12 @@ type Config struct { } // RegisterFlags registers the backend storage config. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { +func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { prefix := "ruler-storage." cfg.StorageBackendConfig.ExtraBackends = []string{local.Name} cfg.Local.RegisterFlagsWithPrefix(prefix, f) - cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "ruler", f) + cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "ruler", f, logger) } // IsDefaults returns true if the storage options have not been set. diff --git a/pkg/storage/bucket/client.go b/pkg/storage/bucket/client.go index e1fc3c5111..8323d7a4d8 100644 --- a/pkg/storage/bucket/client.go +++ b/pkg/storage/bucket/client.go @@ -81,15 +81,15 @@ func (cfg *StorageBackendConfig) supportedBackends() []string { } // RegisterFlags registers the backend storage config. -func (cfg *StorageBackendConfig) RegisterFlags(f *flag.FlagSet) { - cfg.RegisterFlagsWithPrefix("", f) +func (cfg *StorageBackendConfig) RegisterFlags(f *flag.FlagSet, logger log.Logger) { + cfg.RegisterFlagsWithPrefix("", f, logger) } -func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet) { +func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet, logger log.Logger) { cfg.RegisteredFlags = util.TrackRegisteredFlags(prefix, f, func(prefix string, f *flag.FlagSet) { cfg.S3.RegisterFlagsWithPrefix(prefix, f) cfg.GCS.RegisterFlagsWithPrefix(prefix, f) - cfg.Azure.RegisterFlagsWithPrefix(prefix, f) + cfg.Azure.RegisterFlagsWithPrefix(prefix, f, logger) cfg.Swift.RegisterFlagsWithPrefix(prefix, f) cfg.Filesystem.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f) @@ -97,8 +97,8 @@ func (cfg *StorageBackendConfig) RegisterFlagsWithPrefixAndDefaultDirectory(pref }) } -func (cfg *StorageBackendConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "", f) +func (cfg *StorageBackendConfig) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet, logger log.Logger) { + cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "", f, logger) } func (cfg *StorageBackendConfig) Validate() error { @@ -127,17 +127,17 @@ type Config struct { } // RegisterFlags registers the backend storage config. -func (cfg *Config) RegisterFlags(f *flag.FlagSet) { - cfg.RegisterFlagsWithPrefix("", f) +func (cfg *Config) RegisterFlags(f *flag.FlagSet, logger log.Logger) { + cfg.RegisterFlagsWithPrefix("", f, logger) } -func (cfg *Config) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet) { - cfg.StorageBackendConfig.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f) +func (cfg *Config) RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir string, f *flag.FlagSet, logger log.Logger) { + cfg.StorageBackendConfig.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, dir, f, logger) f.StringVar(&cfg.StoragePrefix, prefix+"storage-prefix", "", "Prefix for all objects stored in the backend storage. For simplicity, it may only contain digits and English alphabet letters.") } -func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet) { - cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "", f) +func (cfg *Config) RegisterFlagsWithPrefix(prefix string, f *flag.FlagSet, logger log.Logger) { + cfg.RegisterFlagsWithPrefixAndDefaultDirectory(prefix, "", f, logger) } func (cfg *Config) Validate() error { diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index bb0d205c95..3ada3acecb 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -12,6 +12,7 @@ import ( "time" "github.com/alecthomas/units" + "github.com/go-kit/log" "github.com/pkg/errors" "github.com/prometheus/prometheus/tsdb/chunks" "github.com/prometheus/prometheus/tsdb/wal" @@ -124,8 +125,8 @@ func (d *DurationList) ToMilliseconds() []int64 { } // RegisterFlags registers the TSDB flags -func (cfg *BlocksStorageConfig) RegisterFlags(f *flag.FlagSet) { - cfg.Bucket.RegisterFlagsWithPrefixAndDefaultDirectory("blocks-storage.", "blocks", f) +func (cfg *BlocksStorageConfig) RegisterFlags(f *flag.FlagSet, logger log.Logger) { + cfg.Bucket.RegisterFlagsWithPrefixAndDefaultDirectory("blocks-storage.", "blocks", f, logger) cfg.BucketStore.RegisterFlags(f) cfg.TSDB.RegisterFlags(f) } diff --git a/pkg/storage/tsdb/config_test.go b/pkg/storage/tsdb/config_test.go index b14acaa46e..97e893f48a 100644 --- a/pkg/storage/tsdb/config_test.go +++ b/pkg/storage/tsdb/config_test.go @@ -10,6 +10,7 @@ import ( "testing" "time" + "github.com/go-kit/log" "github.com/grafana/dskit/flagext" "github.com/stretchr/testify/assert" @@ -138,6 +139,7 @@ func TestConfig_Validate(t *testing.T) { func TestConfig_DurationList(t *testing.T) { t.Parallel() + nopLogger := log.NewNopLogger() tests := map[string]struct { cfg BlocksStorageConfig @@ -148,7 +150,7 @@ func TestConfig_DurationList(t *testing.T) { cfg: BlocksStorageConfig{}, expectedRanges: []int64{7200000}, f: func(c *BlocksStorageConfig) { - c.RegisterFlags(&flag.FlagSet{}) + c.RegisterFlags(&flag.FlagSet{}, nopLogger) }, }, "parse ranges correctly": { @@ -168,8 +170,8 @@ func TestConfig_DurationList(t *testing.T) { cfg: BlocksStorageConfig{}, expectedRanges: []int64{7200000}, f: func(c *BlocksStorageConfig) { - c.RegisterFlags(&flag.FlagSet{}) - c.RegisterFlags(&flag.FlagSet{}) + c.RegisterFlags(&flag.FlagSet{}, nopLogger) + c.RegisterFlags(&flag.FlagSet{}, nopLogger) }, }, } diff --git a/tools/compaction-planner/main.go b/tools/compaction-planner/main.go index ee8a8d407b..2a852d1bfc 100644 --- a/tools/compaction-planner/main.go +++ b/tools/compaction-planner/main.go @@ -38,8 +38,10 @@ func main() { sorting string }{} + logger := gokitlog.NewNopLogger() + // Loads bucket index, and plans compaction for all loaded meta files. - cfg.bucket.RegisterFlags(flag.CommandLine) + cfg.bucket.RegisterFlags(flag.CommandLine, logger) cfg.blockRanges = mimir_tsdb.DurationList{2 * time.Hour, 12 * time.Hour, 24 * time.Hour} flag.Var(&cfg.blockRanges, "block-ranges", "List of compaction time ranges.") flag.StringVar(&cfg.userID, "user", "", "User (tenant)") @@ -52,8 +54,6 @@ func main() { log.Fatalln("no user specified") } - logger := gokitlog.NewNopLogger() - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT) defer cancel() diff --git a/tools/list-deduplicated-blocks/main.go b/tools/list-deduplicated-blocks/main.go index 1bbd8eb4f7..dc9f897b9a 100644 --- a/tools/list-deduplicated-blocks/main.go +++ b/tools/list-deduplicated-blocks/main.go @@ -40,7 +40,8 @@ func main() { showCreationTime bool }{} - cfg.bucket.RegisterFlags(flag.CommandLine) + logger := gokitlog.NewNopLogger() + cfg.bucket.RegisterFlags(flag.CommandLine, logger) flag.StringVar(&cfg.userID, "user", "", "User (tenant)") flag.Parse() @@ -51,7 +52,6 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT) defer cancel() - logger := gokitlog.NewNopLogger() bkt, err := bucket.NewClient(ctx, cfg.bucket, "bucket", logger, nil) if err != nil { log.Fatalln("failed to create bucket:", err) diff --git a/tools/listblocks/main.go b/tools/listblocks/main.go index ef29dd36c2..9f19253bbb 100644 --- a/tools/listblocks/main.go +++ b/tools/listblocks/main.go @@ -44,8 +44,9 @@ type config struct { } func main() { + logger := gokitlog.NewNopLogger() cfg := config{} - cfg.bucket.RegisterFlags(flag.CommandLine) + cfg.bucket.RegisterFlags(flag.CommandLine, logger) flag.StringVar(&cfg.userID, "user", "", "User (tenant)") flag.BoolVar(&cfg.showDeleted, "show-deleted", false, "Show deleted blocks") flag.BoolVar(&cfg.showLabels, "show-labels", false, "Show block labels") @@ -68,7 +69,6 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT) defer cancel() - logger := gokitlog.NewNopLogger() bkt, err := bucket.NewClient(ctx, cfg.bucket, "bucket", logger, nil) if err != nil { log.Fatalln("failed to create bucket:", err) diff --git a/tools/markblocks/main.go b/tools/markblocks/main.go index 6e0504e582..cbcf27b9a8 100644 --- a/tools/markblocks/main.go +++ b/tools/markblocks/main.go @@ -42,13 +42,13 @@ func main() { ctx := context.Background() logger := log.WithPrefix(log.NewLogfmtLogger(os.Stderr), "time", log.DefaultTimestampUTC) - cfg := parseFlags() + cfg := parseFlags(logger) marker, filename := createMarker(cfg.mark, logger, cfg.details) ulids := validateTenantAndBlocks(logger, cfg.tenantID, cfg.blocks) uploadMarks(ctx, logger, ulids, marker, filename, cfg.dryRun, cfg.bucket, cfg.tenantID, cfg.allowPartialBlocks, cfg.concurrency) } -func parseFlags() config { +func parseFlags(logger log.Logger) config { var cfg config // We define two flag sets, one on basic straightforward flags of this cli, and the other one with all flags, @@ -91,7 +91,7 @@ func parseFlags() config { // We set only the `-backend` flag on the basicFlagSet, to make sure that user sees that there are more backends supported. // Then we register all bucket flags on the full flag set, which is the flag set we're parsing. basicFlagSet.StringVar(&cfg.bucket.Backend, "backend", bucket.Filesystem, fmt.Sprintf("Backend storage to use. Supported backends are: %s. Use -help-all to see help on backends configuration.", strings.Join(bucket.SupportedBackends, ", "))) - cfg.bucket.RegisterFlags(fullFlagSet) + cfg.bucket.RegisterFlags(fullFlagSet, logger) fullFlagSet.IntVar(&cfg.concurrency, "concurrency", 16, "How many markers to upload concurrently.")