Merge branch 'master' into compaction-objects-processed-metric

CHANGELOG.md

@@ -3,6 +3,7 @@
 * [CHANGE] From path.Join to filepath.Join [#338](https://github.com/grafana/tempo/pull/338)
 * [CHANGE] Upgrade Cortex from v1.3.0 to v.1.4.0 [#341](https://github.com/grafana/tempo/pull/341)
 * [CHANGE] Compact more than 2 blocks at a time [#348](https://github.com/grafana/tempo/pull/348)
+* [ENHANCEMENT] Added OpenMetrics exemplar support. [#359](https://github.com/grafana/tempo/pull/359)
 * [ENHANCEMENT] Add tempodb_compaction_objects_combined metric. [#339](https://github.com/grafana/tempo/pull/339)
 * [ENHANCEMENT] Add tempodb_compaction_objects_processed_total metric. [#360](https://github.com/grafana/tempo/pull/360)
 * [ENHANCEMENT] Add tempodb_compaction_blocks_total metric. [#360](https://github.com/grafana/tempo/pull/360)

cmd/tempo-query/main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"io"
 	"strings"
 
 	"github.com/spf13/viper"
@@ -11,6 +12,7 @@ import (
 	"github.com/jaegertracing/jaeger/plugin/storage/grpc"
 	"github.com/jaegertracing/jaeger/storage/dependencystore"
 	"github.com/jaegertracing/jaeger/storage/spanstore"
+	jaeger_config "github.com/uber/jaeger-client-go/config"
 )
 
 func main() {
@@ -39,6 +41,12 @@ func main() {
 		}
 	}
 
+	closer, err := initJaeger("tempo-grpc-plugin")
+	if err != nil {
+		logger.Error("failed to init tracer", "error", err)
+	}
+	defer closer.Close()
+
 	cfg := &tempo.Config{}
 	cfg.InitFromViper(v)
 
@@ -61,3 +69,13 @@ func (p *plugin) SpanReader() spanstore.Reader {
 func (p *plugin) SpanWriter() spanstore.Writer {
 	return p.backend
 }
+
+func initJaeger(service string) (io.Closer, error) {
+	// .FromEnv() uses standard environment variables to allow for easy configuration
+	cfg, err := jaeger_config.FromEnv()
+	if err != nil {
+		return nil, err
+	}
+
+	return cfg.InitGlobalTracer(service)
+}

cmd/tempo-query/tempo/plugin.go

@@ -10,6 +10,8 @@ import (
 
 	"github.com/golang/protobuf/jsonpb"
 	"github.com/grafana/tempo/pkg/tempopb"
+	"github.com/opentracing/opentracing-go"
+	ot_log "github.com/opentracing/opentracing-go/log"
 	"github.com/weaveworks/common/user"
 	"google.golang.org/grpc/metadata"
 
@@ -36,11 +38,20 @@ func (b *Backend) GetDependencies(endTs time.Time, lookback time.Duration) ([]ja
 }
 func (b *Backend) GetTrace(ctx context.Context, traceID jaeger.TraceID) (*jaeger.Trace, error) {
 	hexID := fmt.Sprintf("%016x%016x", traceID.High, traceID.Low)
+
+	span, _ := opentracing.StartSpanFromContext(ctx, "GetTrace")
+	defer span.Finish()
+
 	req, err := http.NewRequestWithContext(ctx, "GET", b.tempoEndpoint+hexID, nil)
 	if err != nil {
 		return nil, err
 	}
 
+	if tracer := opentracing.GlobalTracer(); tracer != nil {
+		// this is not really loggable or anything we can react to.  just ignoring this error
+		_ = tracer.Inject(span.Context(), opentracing.HTTPHeaders, opentracing.HTTPHeadersCarrier(req.Header))
+	}
+
 	// currently Jaeger Query will only propagate bearer token to the grpc backend and no other headers
 	// so we are going to extract the tenant id from the header, if it exists and use it
 	tenantID, found := extractBearerToken(ctx)
@@ -69,6 +80,7 @@ func (b *Backend) GetTrace(ctx context.Context, traceID jaeger.TraceID) (*jaeger
 		return nil, fmt.Errorf("failed to unmarshal trace json, err: %w. Tempo response body: %s", err, string(body))
 	}
 
+	span.LogFields(ot_log.String("msg", "otlp to Jaeger"))
 	otTrace := ot_pdata.TracesFromOtlp(out.Batches)
 	jaegerBatches, err := ot_jaeger.InternalTracesToJaegerProto(otTrace)
 
@@ -81,8 +93,9 @@ func (b *Backend) GetTrace(ctx context.Context, traceID jaeger.TraceID) (*jaeger
 		ProcessMap: []jaeger.Trace_ProcessMapping{},
 	}
 
+	span.LogFields(ot_log.String("msg", "build process map"))
+	// otel proto conversion doesn't set jaeger processes
 	for _, batch := range jaegerBatches {
-		// otel proto conversion doesn't set jaeger spans for some reason.
 		for _, s := range batch.Spans {
 			s.Process = batch.Process
 		}

go.mod

@@ -34,7 +34,8 @@ require (
 	github.com/spf13/viper v1.7.1
 	github.com/stretchr/testify v1.6.1
 	github.com/uber-go/atomic v1.4.0
-	github.com/weaveworks/common v0.0.0-20200914083218-61ffdd448099
+	github.com/uber/jaeger-client-go v2.25.0+incompatible
+	github.com/weaveworks/common v0.0.0-20201029142910-313edde7a455
 	github.com/willf/bitset v1.1.10 // indirect
 	github.com/willf/bloom v2.0.3+incompatible
 	go.opencensus.io v0.22.3

go.sum

@@ -1832,6 +1832,8 @@ github.com/weaveworks/common v0.0.0-20200206153930-760e36ae819a/go.mod h1:6enWAq
 github.com/weaveworks/common v0.0.0-20200625145055-4b1847531bc9/go.mod h1:c98fKi5B9u8OsKGiWHLRKus6ToQ1Tubeow44ECO1uxY=
 github.com/weaveworks/common v0.0.0-20200914083218-61ffdd448099 h1:MS5M2antM8wzMUqVxIfAi+yb6yjXvDINRFvLnmNXeIw=
 github.com/weaveworks/common v0.0.0-20200914083218-61ffdd448099/go.mod h1:hz10LOsAdzC3K/iXaKoFxOKTDRgxJl+BTGX1GY+TzO4=
+github.com/weaveworks/common v0.0.0-20201029142910-313edde7a455 h1:2T3aOLLo+vSL5sGjIx0z+AicSSeKkXOswmKHE+qTrfU=
+github.com/weaveworks/common v0.0.0-20201029142910-313edde7a455/go.mod h1:ykzWac1LtVfOxdCK+jD754at1Ws9dKCwFeUzkFBffPs=
 github.com/weaveworks/promrus v1.2.0 h1:jOLf6pe6/vss4qGHjXmGz4oDJQA+AOCqEL3FvvZGz7M=
 github.com/weaveworks/promrus v1.2.0/go.mod h1:SaE82+OJ91yqjrE1rsvBWVzNZKcHYFtMUyS1+Ogs/KA=
 github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=

modules/querier/querier.go

@@ -36,7 +36,7 @@ var (
 		Namespace: "tempo",
 		Name:      "query_bytes_read",
 		Help:      "bytes read",
-		Buckets:   prometheus.ExponentialBuckets(1024, 2, 10),
+		Buckets:   prometheus.ExponentialBuckets(1024*1024, 2, 8),
 	}, []string{"layer"})
 	metricIngesterClients = promauto.NewGauge(prometheus.GaugeOpts{
 		Namespace: "tempo",

tempodb/tempodb.go

@@ -259,9 +259,6 @@ func (rw *readerWriter) Find(ctx context.Context, tenantID string, id encoding.I
 	}
 
 	foundBytes, err := rw.pool.RunJobs(derivedCtx, copiedBlocklist, func(ctx context.Context, payload interface{}) ([]byte, error) {
-		span, ctx := opentracing.StartSpanFromContext(ctx, "block.Find")
-		defer span.Finish()
-
 		meta := payload.(*encoding.BlockMeta)
 		shardKey := bloom.ShardKeyForTraceID(id)
 
@@ -278,15 +275,13 @@ func (rw *readerWriter) Find(ctx context.Context, tenantID string, id encoding.I
 			return nil, fmt.Errorf("error parsing bloom %v", err)
 		}
 
-		span.LogFields(ot_log.String("msg", "bloom"), ot_log.Int("bytes", len(bloomBytes)))
 		metrics.BloomFilterReads.Inc()
 		metrics.BloomFilterBytesRead.Add(int32(len(bloomBytes)))
 		if !filter.Test(id) {
 			return nil, nil
 		}
 
 		indexBytes, err := rw.r.Index(ctx, meta.BlockID, tenantID)
-		span.LogFields(ot_log.String("msg", "index"), ot_log.Int("bytes", len(indexBytes)))
 		metrics.IndexReads.Inc()
 		metrics.IndexBytesRead.Add(int32(len(indexBytes)))
 		if err != nil {
@@ -304,7 +299,6 @@ func (rw *readerWriter) Find(ctx context.Context, tenantID string, id encoding.I
 
 		objectBytes := make([]byte, record.Length)
 		err = rw.r.Object(ctx, meta.BlockID, tenantID, record.Start, objectBytes)
-		span.LogFields(ot_log.String("msg", "object"), ot_log.Int("bytes", len(objectBytes)))
 		metrics.BlockReads.Inc()
 		metrics.BlockBytesRead.Add(int32(len(objectBytes)))
 		if err != nil {

vendor/modules.txt

@@ -1091,6 +1091,7 @@ github.com/tommy-muehle/go-mnd/config
 ## explicit
 github.com/uber-go/atomic
 # github.com/uber/jaeger-client-go v2.25.0+incompatible
+## explicit
 github.com/uber/jaeger-client-go
 github.com/uber/jaeger-client-go/config
 github.com/uber/jaeger-client-go/internal/baggage
@@ -1122,7 +1123,7 @@ github.com/ultraware/funlen
 github.com/ultraware/whitespace
 # github.com/uudashr/gocognit v1.0.1
 github.com/uudashr/gocognit
-# github.com/weaveworks/common v0.0.0-20200914083218-61ffdd448099
+# github.com/weaveworks/common v0.0.0-20201029142910-313edde7a455
 ## explicit
 github.com/weaveworks/common/aws
 github.com/weaveworks/common/errors