Migrate to Thanos Objstore package at new location

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 ### Grafana Mimir
 
+* [CHANGE] Flag `-azure.msi-resource` is now ignored, and will be removed in Mimir 2.7. This setting is now made automatically by Azure. #2682
 * [ENHANCEMENT] Distributor: Add age filter to forwarding functionality, to not forward samples which are older than defined duration. If such samples are not ingested, `cortex_discarded_samples_total{reason="forwarded-sample-too-old"}` is increased. #3049 #3133
 * [ENHANCEMENT] Store-gateway: Reduce memory allocation when generating ids in index cache. #3179
 * [ENHANCEMENT] Query-frontend: truncate queries based on the configured creation grace period (`--validation.create-grace-period`) to avoid querying too far into the future. #3172

cmd/metaconvert/main.go

@@ -20,9 +20,9 @@ import (
 	gklog "github.com/go-kit/log"
 	"github.com/go-kit/log/level"
 	"github.com/pkg/errors"
+	"github.com/thanos-io/objstore"
 	"github.com/thanos-io/thanos/pkg/block"
 	"github.com/thanos-io/thanos/pkg/block/metadata"
-	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/weaveworks/common/logging"
 
 	"github.com/grafana/mimir/pkg/storage/bucket"
@@ -42,8 +42,9 @@ func main() {
 	cfg := config{}
 
 	cfg.LogLevel.RegisterFlags(flag.CommandLine)
+	initLogger := log.NewDefaultLogger(cfg.LogLevel, cfg.LogFormat)
 	cfg.LogFormat.RegisterFlags(flag.CommandLine)
-	cfg.BucketConfig.RegisterFlags(flag.CommandLine)
+	cfg.BucketConfig.RegisterFlags(flag.CommandLine, initLogger)
 
 	flag.BoolVar(&cfg.DryRun, "dry-run", false, "Don't make changes; only report what needs to be done")
 	flag.StringVar(&cfg.Tenant, "tenant", "", "Tenant to process")

cmd/mimir/config-descriptor.json

@@ -4603,17 +4603,6 @@
               "fieldType": "int",
               "fieldCategory": "advanced"
             },
-            {
-              "kind": "field",
-              "name": "msi_resource",
-              "required": false,
-              "desc": "If set, this URL is used instead of https://\u003cstorage-account-name\u003e.\u003cendpoint-suffix\u003e for obtaining ServicePrincipalToken from MSI.",
-              "fieldValue": null,
-              "fieldDefaultValue": "",
-              "fieldFlag": "blocks-storage.azure.msi-resource",
-              "fieldType": "string",
-              "fieldCategory": "advanced"
-            },
             {
               "kind": "field",
               "name": "user_assigned_id",
@@ -8560,17 +8549,6 @@
               "fieldType": "int",
               "fieldCategory": "advanced"
             },
-            {
-              "kind": "field",
-              "name": "msi_resource",
-              "required": false,
-              "desc": "If set, this URL is used instead of https://\u003cstorage-account-name\u003e.\u003cendpoint-suffix\u003e for obtaining ServicePrincipalToken from MSI.",
-              "fieldValue": null,
-              "fieldDefaultValue": "",
-              "fieldFlag": "ruler-storage.azure.msi-resource",
-              "fieldType": "string",
-              "fieldCategory": "advanced"
-            },
             {
               "kind": "field",
               "name": "user_assigned_id",
@@ -9934,17 +9912,6 @@
               "fieldType": "int",
               "fieldCategory": "advanced"
             },
-            {
-              "kind": "field",
-              "name": "msi_resource",
-              "required": false,
-              "desc": "If set, this URL is used instead of https://\u003cstorage-account-name\u003e.\u003cendpoint-suffix\u003e for obtaining ServicePrincipalToken from MSI.",
-              "fieldValue": null,
-              "fieldDefaultValue": "",
-              "fieldFlag": "alertmanager-storage.azure.msi-resource",
-              "fieldType": "string",
-              "fieldCategory": "advanced"
-            },
             {
               "kind": "field",
               "name": "user_assigned_id",
@@ -11634,17 +11601,6 @@
                   "fieldType": "int",
                   "fieldCategory": "advanced"
                 },
-                {
-                  "kind": "field",
-                  "name": "msi_resource",
-                  "required": false,
-                  "desc": "If set, this URL is used instead of https://\u003cstorage-account-name\u003e.\u003cendpoint-suffix\u003e for obtaining ServicePrincipalToken from MSI.",
-                  "fieldValue": null,
-                  "fieldDefaultValue": "",
-                  "fieldFlag": "common.storage.azure.msi-resource",
-                  "fieldType": "string",
-                  "fieldCategory": "advanced"
-                },
                 {
                   "kind": "field",
                   "name": "user_assigned_id",

cmd/mimir/help-all.txt.tmpl

@@ -13,8 +13,6 @@ Usage of ./cmd/mimir/mimir:
     	Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN. If set to empty string, default endpoint suffix is used.
   -alertmanager-storage.azure.max-retries int
     	Number of retries for recoverable errors (default 20)
-  -alertmanager-storage.azure.msi-resource string
-    	If set, this URL is used instead of https://<storage-account-name>.<endpoint-suffix> for obtaining ServicePrincipalToken from MSI.
   -alertmanager-storage.azure.user-assigned-id string
     	User assigned identity. If empty, then System assigned identity is used.
   -alertmanager-storage.backend string
@@ -263,8 +261,6 @@ Usage of ./cmd/mimir/mimir:
     	Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN. If set to empty string, default endpoint suffix is used.
   -blocks-storage.azure.max-retries int
     	Number of retries for recoverable errors (default 20)
-  -blocks-storage.azure.msi-resource string
-    	If set, this URL is used instead of https://<storage-account-name>.<endpoint-suffix> for obtaining ServicePrincipalToken from MSI.
   -blocks-storage.azure.user-assigned-id string
     	User assigned identity. If empty, then System assigned identity is used.
   -blocks-storage.backend string
@@ -535,8 +531,6 @@ Usage of ./cmd/mimir/mimir:
     	Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN. If set to empty string, default endpoint suffix is used.
   -common.storage.azure.max-retries int
     	Number of retries for recoverable errors (default 20)
-  -common.storage.azure.msi-resource string
-    	If set, this URL is used instead of https://<storage-account-name>.<endpoint-suffix> for obtaining ServicePrincipalToken from MSI.
   -common.storage.azure.user-assigned-id string
     	User assigned identity. If empty, then System assigned identity is used.
   -common.storage.backend string
@@ -1515,8 +1509,6 @@ Usage of ./cmd/mimir/mimir:
     	Azure storage endpoint suffix without schema. The account name will be prefixed to this value to create the FQDN. If set to empty string, default endpoint suffix is used.
   -ruler-storage.azure.max-retries int
     	Number of retries for recoverable errors (default 20)
-  -ruler-storage.azure.msi-resource string
-    	If set, this URL is used instead of https://<storage-account-name>.<endpoint-suffix> for obtaining ServicePrincipalToken from MSI.
   -ruler-storage.azure.user-assigned-id string
     	User assigned identity. If empty, then System assigned identity is used.
   -ruler-storage.backend string

docs/sources/operators-guide/configure/reference-configuration-parameters/index.md

@@ -3632,12 +3632,6 @@ The `azure_storage_backend` block configures the connection to Azure object stor
 # CLI flag: -<prefix>.azure.max-retries
 [max_retries: <int> | default = 20]
 
-# (advanced) If set, this URL is used instead of
-# https://<storage-account-name>.<endpoint-suffix> for obtaining
-# ServicePrincipalToken from MSI.
-# CLI flag: -<prefix>.azure.msi-resource
-[msi_resource: <string> | default = ""]
-
 # (advanced) User assigned identity. If empty, then System assigned identity is
 # used.
 # CLI flag: -<prefix>.azure.user-assigned-id

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/google/gopacket v1.1.19
 	github.com/gorilla/mux v1.8.0
 	github.com/grafana/dskit v0.0.0-20221004074252-e8057f223e41
-	github.com/grafana/e2e v0.1.1-0.20220519104354-1db01e4751fe
+	github.com/grafana/e2e v0.1.1-0.20221013164451-2c8db587f293
 	github.com/hashicorp/golang-lru v0.5.4
 	github.com/json-iterator/go v1.1.12
 	github.com/minio/minio-go/v7 v7.0.36
@@ -31,7 +31,7 @@ require (
 	github.com/prometheus/client_golang v1.13.0
 	github.com/prometheus/client_model v0.2.0
 	github.com/prometheus/common v0.37.0
-	github.com/prometheus/prometheus v1.8.2-0.20220308163432-03831554a519
+	github.com/prometheus/prometheus v1.8.2-0.20220620125440-d7e7b8e04b5e
 	github.com/segmentio/fasthash v0.0.0-20180216231524-a72b379d632e
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/afero v1.6.0
@@ -59,6 +59,7 @@ require (
 	github.com/grafana/regexp v0.0.0-20221005093135-b4c2bcb0a4b6
 	github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db
 	github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/prometheusremotewrite v0.54.0
+	github.com/thanos-io/objstore v0.0.0-20220809103346-8ef1f215e2bf
 	go.opentelemetry.io/collector/pdata v0.54.0
 	go.opentelemetry.io/otel v1.7.0
 	go.opentelemetry.io/otel/trace v1.7.0
@@ -72,33 +73,27 @@ require (
 	cloud.google.com/go/compute v1.7.0 // indirect
 	cloud.google.com/go/iam v0.3.0 // indirect
 	cloud.google.com/go/storage v1.22.1 // indirect
-	github.com/Azure/azure-pipeline-go v0.2.3 // indirect
-	github.com/Azure/azure-storage-blob-go v0.13.0 // indirect
-	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
-	github.com/Azure/go-autorest/autorest v0.11.27 // indirect
-	github.com/Azure/go-autorest/autorest/adal v0.9.20 // indirect
-	github.com/Azure/go-autorest/autorest/azure/auth v0.5.11 // indirect
-	github.com/Azure/go-autorest/autorest/azure/cli v0.4.5 // indirect
-	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
-	github.com/Azure/go-autorest/logger v0.2.1 // indirect
-	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.4.1 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 // indirect
 	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
-	github.com/OneOfOne/xxhash v1.2.6 // indirect
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/armon/go-metrics v0.3.10 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/aws/aws-sdk-go v1.44.109 // indirect
-	github.com/aws/aws-sdk-go-v2 v1.13.0 // indirect
-	github.com/aws/aws-sdk-go-v2/config v1.13.1 // indirect
-	github.com/aws/aws-sdk-go-v2/credentials v1.8.0 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.5 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 // indirect
-	github.com/aws/smithy-go v1.10.0 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.16.0 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.15.1 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.11.0 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.7 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.8 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.16.1 // indirect
+	github.com/aws/smithy-go v1.11.1 // indirect
 	github.com/benbjohnson/clock v1.3.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b // indirect
@@ -110,10 +105,10 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dennwc/varint v1.0.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/dimchansky/utfbom v1.1.1 // indirect
 	github.com/dlclark/regexp2 v1.4.0 // indirect
 	github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
+	github.com/efficientgo/tools/core v0.0.0-20220225185207-fe763185946b // indirect
 	github.com/facette/natsort v0.0.0-20181210072756-2cd4dd1e2dcb // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.3 // indirect
@@ -132,7 +127,7 @@ require (
 	github.com/go-redis/redis/v8 v8.11.4 // indirect
 	github.com/gofrs/uuid v4.3.0+incompatible // indirect
 	github.com/gogo/googleapis v1.4.1 // indirect
-	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
+	github.com/golang-jwt/jwt v3.2.1+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/google/btree v1.0.1 // indirect
 	github.com/google/gnostic v0.5.7-v3refs // indirect
@@ -165,9 +160,9 @@ require (
 	github.com/klauspost/compress v1.15.9 // indirect
 	github.com/klauspost/cpuid/v2 v2.1.1 // indirect
 	github.com/kr/pretty v0.3.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.12 // indirect
-	github.com/mattn/go-ieproxy v0.0.1 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.2 // indirect
 	github.com/miekg/dns v1.1.50 // indirect
@@ -180,8 +175,9 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/ncw/swift v1.0.52 // indirect
+	github.com/ncw/swift v1.0.53 // indirect
 	github.com/oklog/run v1.1.0 // indirect
+	github.com/pkg/browser v0.0.0-20210115035449-ce105d075bb4 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/prometheus/exporter-toolkit v0.7.1 // indirect
@@ -193,7 +189,6 @@ require (
 	github.com/sercand/kuberesolver v2.4.0+incompatible // indirect
 	github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 // indirect
 	github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546 // indirect
-	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/cobra v1.4.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/stretchr/objx v0.4.0 // indirect
@@ -239,7 +234,7 @@ replace github.com/prometheus/prometheus => github.com/grafana/mimir-prometheus
 
 // Out of order Support forces us to fork thanos because we've changed the ChunkReader interface.
 // Once the out of order support is upstreamed and Thanos has vendored it, we can remove this override.
-replace github.com/thanos-io/thanos => github.com/grafana/thanos v0.19.1-0.20220713162227-7bde03e4afa9
+replace github.com/thanos-io/thanos => github.com/grafana/thanos v0.19.1-0.20220809170552-5e929a767b26
 
 // Pin hashicorp depencencies since the Prometheus fork, go mod tries to update them.
 replace github.com/hashicorp/go-immutable-radix => github.com/hashicorp/go-immutable-radix v1.2.0