Support multiple authentication methods for S3 (#320)

* tempodb: Upgrade to minio v7 and use credentials chain

Adds support for AWS IRSA, IAM roles, static credentials, etc.

* vendor: Upgrade to minio v7

* Update CHANGELOG

* docs/tempo/website/configuration: Document support S3 authentication methods

* example: Update s3 minio example to also show how to specify AWS S3 endpoint

* docs/tempo/website: Link out to upstream docs for minio/s3 authentication topics

CHANGELOG.md

@@ -5,6 +5,7 @@
 * [ENHANCEMENT] CI checks for vendored dependencies using `make vendor-check`. Update CONTRIBUTING.md to reflect the same before checking in files in a PR. [#274](https://github.com/grafana/tempo/pull/274)
 * [ENHANCEMENT] Add warnings for suspect configs. [#294](https://github.com/grafana/tempo/pull/294)
 * [ENHANCEMENT] Add command line flags for s3 credentials. [#308](https://github.com/grafana/tempo/pull/308)
+* [ENHANCEMENT] Support multiple authentication methods for S3 (IRSA, IAM role, static). [#320](https://github.com/grafana/tempo/pull/320)
 * [BUGFIX] S3 multi-part upload errors [#306](https://github.com/grafana/tempo/pull/325)
 * [BUGFIX] Increase Prometheus `notfound` metric on tempo-vulture. [#301](https://github.com/grafana/tempo/pull/301)
 * [BUGFIX] Return 404 if searching for a tenant id that does not exist in the backend. [#321](https://github.com/grafana/tempo/pull/321)

docs/tempo/website/configuration/_index.md

@@ -61,6 +61,15 @@ compactor:
 ### [Storage](https://github.com/grafana/tempo/blob/master/tempodb/config.go)
 The storage block is used to configure TempoDB.
 
+For the s3 backend, the following authentication methods are supported:
+
+- AWS env vars (static AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY)
+- Static access key and secret credentials specified in `storage.trace.s3.access_key` and `storage.trace.s3.secret_key`
+- Minio env vars (MINIO_ACCESS_KEY and MINIO_SECRET_KEY)
+- AWS shared credentials [config file](https://docs.aws.amazon.com/ses/latest/DeveloperGuide/create-shared-credentials-file.html)
+- Minio client credentials [config file](https://github.com/minio/mc/blob/master/docs/minio-client-configuration-files.md)
+- AWS IAM ([IRSA via WebIdentity](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html), [EC2 instance role](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/iam-roles-for-amazon-ec2.html))
+
 ```
 storage:
     trace:
@@ -88,4 +97,4 @@ memberlist:
     bind_port: 7946
     join_members:
       - gossip-ring.tracing-ops.svc.cluster.local:7946  # A DNS entry that lists all tempo components.  A "Headless" Cluster IP service in Kubernetes
-```
+```

example/docker-compose/etc/tempo-s3-minio.yaml

@@ -44,6 +44,9 @@ storage:
       access_key: tempo
       secret_key: supersecret
       insecure: true
+      # For using AWS, select the appropriate regional endpoint and region
+      # endpoint: s3.dualstack.us-west-2.amazonaws.com
+      # region: us-west-2
     pool:
       max_workers: 100                 # the worker pool mainly drives querying, but is also used for polling the blocklist
-      queue_depth: 10000
+      queue_depth: 10000

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/jaegertracing/jaeger v1.18.2-0.20200707061226-97d2319ff2be
 	github.com/jsternberg/zap-logfmt v1.0.0
 	github.com/karrick/godirwalk v1.16.1
-	github.com/minio/minio-go/v6 v6.0.56
+	github.com/minio/minio-go/v7 v7.0.5
 	github.com/olekukonko/tablewriter v0.0.2
 	github.com/open-telemetry/opentelemetry-proto v0.4.0
 	github.com/opentracing/opentracing-go v1.2.0

go.sum

@@ -1144,6 +1144,9 @@ github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0
 github.com/klauspost/compress v1.10.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
 github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
+github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
+github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
 github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg=
 github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08/go.mod h1:dFWs1zEqDjFtnBXsd1vPOZaLsESovai349994nHx3e0=
@@ -1277,10 +1280,14 @@ github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7
 github.com/miekg/dns v1.1.30 h1:Qww6FseFn8PRfw07jueqIXqodm0JKiiKuK0DeXSqfyo=
 github.com/miekg/dns v1.1.30/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
 github.com/minio/cli v1.20.0/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY=
+github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
+github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
 github.com/minio/minio-go/v6 v6.0.27-0.20190529152532-de69c0e465ed/go.mod h1:vaNT59cWULS37E+E9zkuN/BVnKHyXtVGS+b04Boc66Y=
 github.com/minio/minio-go/v6 v6.0.44/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg=
 github.com/minio/minio-go/v6 v6.0.56 h1:H4+v6UFV1V7VkEf1HjL15W9OvTL1Gy8EbMmjQZHqEbg=
 github.com/minio/minio-go/v6 v6.0.56/go.mod h1:KQMM+/44DSlSGSQWSfRrAZ12FVMmpWNuX37i2AX0jfI=
+github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4=
+github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw=
 github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
 github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
 github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
@@ -1561,6 +1568,7 @@ github.com/rogpeppe/go-internal v1.5.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE
 github.com/rs/cors v1.6.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
 github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik=
 github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU=
+github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -1907,6 +1915,8 @@ golang.org/x/crypto v0.0.0-20200320181102-891825fb96df/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 h1:DZhuSZLsGlFL4CmhA8BcRA0mnthyA/nZ00AqCUo7vHg=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
@@ -2356,6 +2366,8 @@ gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.52.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.55.0 h1:E8yzL5unfpW3M6fz/eB7Cb5MQAYSZ7GKo4Qth+N2sgQ=
 gopkg.in/ini.v1 v1.55.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
+gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/jcmturner/aescts.v1 v1.0.1 h1:cVVZBK2b1zY26haWB4vbBiZrfFQnfbTVrE3xZq6hrEw=
 gopkg.in/jcmturner/aescts.v1 v1.0.1/go.mod h1:nsR8qBOg+OucoIW+WMhB3GspUQXq9XorLnQb9XtvcOo=
 gopkg.in/jcmturner/dnsutils.v1 v1.0.1 h1:cIuC1OLRGZrld+16ZJvvZxVJeKPsvd5eUIvxfoN5hSM=

tempodb/backend/s3/compactor.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 
 	"github.com/grafana/tempo/tempodb/encoding/bloom"
+	"github.com/minio/minio-go/v7"
 
 	"github.com/go-kit/kit/log/level"
 	"github.com/google/uuid"
@@ -24,7 +25,7 @@ func (rw *readerWriter) MarkBlockCompacted(blockID uuid.UUID, tenantID string) e
 
 	metaFileName := util.MetaFileName(blockID, tenantID)
 	// copy meta.json to meta.compacted.json
-	_, err := rw.core.CopyObjectWithContext(
+	_, err := rw.core.CopyObject(
 		context.TODO(),
 		rw.cfg.Bucket,
 		metaFileName,
@@ -37,7 +38,7 @@ func (rw *readerWriter) MarkBlockCompacted(blockID uuid.UUID, tenantID string) e
 	}
 
 	// delete meta.json
-	return rw.core.RemoveObject(rw.cfg.Bucket, metaFileName)
+	return rw.core.RemoveObject(context.TODO(), rw.cfg.Bucket, metaFileName, minio.RemoveObjectOptions{})
 }
 
 func (rw *readerWriter) ClearBlock(blockID uuid.UUID, tenantID string) error {
@@ -59,7 +60,7 @@ func (rw *readerWriter) ClearBlock(blockID uuid.UUID, tenantID string) error {
 	delObjects = append(delObjects, util.IndexFileName(blockID, tenantID))
 	delObjects = append(delObjects, util.ObjectFileName(blockID, tenantID))
 	for _, obj := range delObjects {
-		err := rw.core.RemoveObject(rw.cfg.Bucket, obj)
+		err := rw.core.RemoveObject(context.TODO(), rw.cfg.Bucket, obj, minio.RemoveObjectOptions{})
 		if err != nil {
 			return errors.Wrapf(err, "error deleting obj from s3: %s", obj)
 		}

tempodb/backend/s3/s3.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"io"
 	"io/ioutil"
+	"net/http"
 	"strings"
 
 	log_util "github.com/cortexproject/cortex/pkg/util"
@@ -15,7 +16,8 @@ import (
 	"github.com/grafana/tempo/tempodb/backend"
 	"github.com/grafana/tempo/tempodb/backend/util"
 	"github.com/grafana/tempo/tempodb/encoding"
-	"github.com/minio/minio-go/v6"
+	"github.com/minio/minio-go/v7"
+	"github.com/minio/minio-go/v7/pkg/credentials"
 	"github.com/pkg/errors"
 )
 
@@ -32,7 +34,29 @@ type readerWriter struct {
 
 func New(cfg *Config) (backend.Reader, backend.Writer, backend.Compactor, error) {
 	l := log_util.Logger
-	core, err := minio.NewCore(cfg.Endpoint, cfg.AccessKey, cfg.SecretKey, !cfg.Insecure)
+	creds := credentials.NewChainCredentials([]credentials.Provider{
+		&credentials.EnvAWS{},
+		&credentials.Static{
+			Value: credentials.Value{
+				AccessKeyID:     cfg.AccessKey,
+				SecretAccessKey: cfg.SecretKey,
+			},
+		},
+		&credentials.EnvMinio{},
+		&credentials.FileAWSCredentials{},
+		&credentials.FileMinioClient{},
+		&credentials.IAM{
+			Client: &http.Client{
+				Transport: http.DefaultTransport,
+			},
+		},
+	})
+	opts := &minio.Options{
+		Secure: !cfg.Insecure,
+		Creds:  creds,
+		Region: cfg.Region,
+	}
+	core, err := minio.NewCore(cfg.Endpoint, opts)
 	if err != nil {
 		return nil, nil, nil, err
 	}
@@ -41,9 +65,9 @@ func New(cfg *Config) (backend.Reader, backend.Writer, backend.Compactor, error)
 	//client.SetCustomTransport(minio.DefaultTransport(!cfg.Insecure))
 
 	// make bucket name if doesn't exist already
-	err = core.MakeBucket(cfg.Bucket, cfg.Region)
+	err = core.MakeBucket(context.Background(), cfg.Bucket, minio.MakeBucketOptions{Region: cfg.Region})
 	if err != nil {
-		exists, errBucketExists := core.BucketExists(cfg.Bucket)
+		exists, errBucketExists := core.BucketExists(context.Background(), cfg.Bucket)
 		if errBucketExists == nil && !exists {
 			return nil, nil, nil, errors.Wrap(err, "cannot create bucket in s3, invalid permissions")
 		} else if errBucketExists != nil {
@@ -70,7 +94,7 @@ func (rw *readerWriter) Write(ctx context.Context, meta *encoding.BlockMeta, bBl
 	}
 
 	objName := util.ObjectFileName(meta.BlockID, meta.TenantID)
-	size, err := rw.core.FPutObjectWithContext(
+	info, err := rw.core.FPutObject(
 		ctx,
 		rw.cfg.Bucket,
 		objName,
@@ -81,7 +105,7 @@ func (rw *readerWriter) Write(ctx context.Context, meta *encoding.BlockMeta, bBl
 		return errors.Wrapf(err, "error writing object to s3 backend, object %s", objName)
 	}
 
-	level.Debug(rw.logger).Log("msg", "object uploaded to s3", "objectName", objName, "size", size)
+	level.Debug(rw.logger).Log("msg", "object uploaded to s3", "objectName", objName, "size", info.Size)
 
 	err = rw.WriteBlockMeta(ctx, nil, meta, bBloom, bIndex)
 	if err != nil {
@@ -104,7 +128,7 @@ func (rw *readerWriter) WriteBlockMeta(ctx context.Context, tracker backend.Appe
 		}
 		level.Debug(rw.logger).Log("msg", "marking compacted block complete", "parts", len(completeParts))
 		objName := util.ObjectFileName(meta.BlockID, meta.TenantID)
-		etag, err := rw.core.CompleteMultipartUploadWithContext(
+		etag, err := rw.core.CompleteMultipartUpload(
 			ctx,
 			rw.cfg.Bucket,
 			objName,
@@ -123,7 +147,7 @@ func (rw *readerWriter) WriteBlockMeta(ctx context.Context, tracker backend.Appe
 	}
 
 	for i, b := range bBloom {
-		size, err := rw.core.Client.PutObjectWithContext(
+		size, err := rw.core.Client.PutObject(
 			ctx,
 			rw.cfg.Bucket,
 			util.BloomFileName(blockID, tenantID, i),
@@ -137,7 +161,7 @@ func (rw *readerWriter) WriteBlockMeta(ctx context.Context, tracker backend.Appe
 		level.Debug(rw.logger).Log("msg", "block bloom uploaded to s3", "shard", i, "size", size)
 	}
 
-	size, err := rw.core.Client.PutObjectWithContext(
+	size, err := rw.core.Client.PutObject(
 		ctx,
 		rw.cfg.Bucket,
 		util.IndexFileName(blockID, tenantID),
@@ -156,7 +180,7 @@ func (rw *readerWriter) WriteBlockMeta(ctx context.Context, tracker backend.Appe
 	}
 
 	// write meta last.  this will prevent blocklist from returning a partial block
-	size, err = rw.core.Client.PutObjectWithContext(
+	size, err = rw.core.Client.PutObject(
 		ctx,
 		rw.cfg.Bucket,
 		util.MetaFileName(blockID, tenantID),
@@ -188,6 +212,7 @@ func (rw *readerWriter) AppendObject(ctx context.Context, tracker backend.Append
 		a = tracker.(AppenderTracker)
 	} else {
 		id, err := rw.core.NewMultipartUpload(
+			ctx,
 			rw.cfg.Bucket,
 			util.ObjectFileName(meta.BlockID, meta.TenantID),
 			options,
@@ -201,7 +226,7 @@ func (rw *readerWriter) AppendObject(ctx context.Context, tracker backend.Append
 	level.Debug(rw.logger).Log("msg", "appending object to s3", "objectName", util.ObjectFileName(meta.BlockID, meta.TenantID))
 
 	a.partNum++
-	objPart, err := rw.core.PutObjectPartWithContext(
+	objPart, err := rw.core.PutObjectPart(
 		ctx,
 		rw.cfg.Bucket,
 		util.ObjectFileName(meta.BlockID, meta.TenantID),
@@ -296,7 +321,7 @@ func (rw *readerWriter) Shutdown() {
 }
 
 func (rw *readerWriter) readAll(ctx context.Context, name string) ([]byte, error) {
-	reader, _, _, err := rw.core.GetObjectWithContext(ctx, rw.cfg.Bucket, name, minio.GetObjectOptions{})
+	reader, _, _, err := rw.core.GetObject(ctx, rw.cfg.Bucket, name, minio.GetObjectOptions{})
 	if err != nil {
 		// do not change or wrap this error
 		// we need to compare the specific err message
@@ -312,7 +337,7 @@ func (rw *readerWriter) readAll(ctx context.Context, name string) ([]byte, error
 }
 
 func (rw *readerWriter) readAllWithObjInfo(ctx context.Context, name string) ([]byte, minio.ObjectInfo, error) {
-	reader, info, _, err := rw.core.GetObjectWithContext(ctx, rw.cfg.Bucket, name, minio.GetObjectOptions{})
+	reader, info, _, err := rw.core.GetObject(ctx, rw.cfg.Bucket, name, minio.GetObjectOptions{})
 	if err != nil && err.Error() == s3KeyDoesNotExist {
 		return nil, minio.ObjectInfo{}, backend.ErrMetaDoesNotExist
 	} else if err != nil {
@@ -333,7 +358,7 @@ func (rw *readerWriter) readRange(ctx context.Context, objName string, offset in
 	if err != nil {
 		return errors.Wrap(err, "error setting headers for range read in s3")
 	}
-	reader, _, _, err := rw.core.GetObjectWithContext(ctx, rw.cfg.Bucket, objName, options)
+	reader, _, _, err := rw.core.GetObject(ctx, rw.cfg.Bucket, objName, options)
 	if err != nil {
 		return errors.Wrapf(err, "error in range read from s3 backend, bucket: %s, objName: %s", rw.cfg.Bucket, objName)
 	}