Add injectable middlewares to bucket client (cortexproject#3278)

* feat: add middleware config field

Signed-off-by: Jacob Lisi <jacob.t.lisi@gmail.com>

* fix: rename Middleware to Middlewares

Signed-off-by: Jacob Lisi <jacob.t.lisi@gmail.com>

* fix: rename Middleware to Middlewares

Signed-off-by: Jacob Lisi <jacob.t.lisi@gmail.com>

* address PR comments

Signed-off-by: Jacob Lisi <jacob.t.lisi@gmail.com>

pkg/storage/tsdb/bucket_client.go

@@ -32,7 +32,17 @@ func NewBucketClient(ctx context.Context, cfg BucketConfig, name string, logger
 		return nil, err
 	}
 
-	return objstore.NewTracingBucket(bucketWithMetrics(client, name, reg)), nil
+	client = objstore.NewTracingBucket(bucketWithMetrics(client, name, reg))
+
+	// Wrap the client with any provided middleware
+	for _, wrap := range cfg.Middlewares {
+		client, err = wrap(client)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return client, nil
 }
 
 func bucketWithMetrics(bucketClient objstore.Bucket, name string, reg prometheus.Registerer) objstore.Bucket {

pkg/storage/tsdb/config.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/alecthomas/units"
 	"github.com/pkg/errors"
+	"github.com/thanos-io/thanos/pkg/objstore"
 	"github.com/thanos-io/thanos/pkg/store"
 
 	"github.com/cortexproject/cortex/pkg/storage/backend/azure"
@@ -64,6 +65,10 @@ type BucketConfig struct {
 	GCS        gcs.Config        `yaml:"gcs"`
 	Azure      azure.Config      `yaml:"azure"`
 	Filesystem filesystem.Config `yaml:"filesystem"`
+
+	// Not used internally, meant to allow callers to wrap Buckets
+	// created using this config
+	Middlewares []func(objstore.Bucket) (objstore.Bucket, error) `yaml:"-"`
 }
 
 // BlocksStorageConfig holds the config information for the blocks storage.