tracing: track query sent to prometheus via remote read api (#2145)

* tracing: track query sent to prometheus via remote read api

Signed-off-by: Igor Wiedler <iwiedler@gitlab.com>

* add CHANGELOG entry

Signed-off-by: Igor Wiedler <iwiedler@gitlab.com>

CHANGELOG.md

@@ -28,6 +28,7 @@ We use *breaking* word for marking changes that are not backward compatible (rel
 - [#2030](https://github.com/thanos-io/thanos/pull/2030) Query: Add `thanos_proxy_store_empty_stream_responses_total` metric for number of empty responses from stores.
 - [#2049](https://github.com/thanos-io/thanos/pull/2049) Tracing: Support sampling on Elastic APM with new sample_rate setting.
 - [#2008](https://github.com/thanos-io/thanos/pull/2008) Querier, Receiver, Sidecar, Store: Add gRPC [health check](https://github.com/grpc/grpc/blob/master/doc/health-checking.md) endpoints.
+- [#2145](https://github.com/thanos-io/thanos/pull/2145) Tracing: track query sent to prometheus via remote read api.
 
 ### Changed
 

pkg/store/prometheus.go

@@ -432,6 +432,11 @@ func (p *PrometheusStore) startPromSeries(ctx context.Context, q *prompb.Query)
 		return nil, errors.Wrap(err, "marshal read request")
 	}
 
+	qjson, err := json.Marshal(q)
+	if err != nil {
+		return nil, errors.Wrap(err, "json encode query for tracing")
+	}
+
 	u := *p.base
 	u.Path = path.Join(u.Path, "api/v1/read")
 
@@ -442,13 +447,13 @@ func (p *PrometheusStore) startPromSeries(ctx context.Context, q *prompb.Query)
 	preq.Header.Add("Content-Encoding", "snappy")
 	preq.Header.Set("Content-Type", "application/x-stream-protobuf")
 	preq.Header.Set("User-Agent", userAgent)
-	spanReqDo, ctx := tracing.StartSpan(ctx, "query_prometheus_request")
+	spanReqDo, ctx := tracing.StartSpan(ctx, "query_prometheus_request", opentracing.Tag{Key: "prometheus.query", Value: string(qjson)})
 	preq = preq.WithContext(ctx)
 	presp, err := p.client.Do(preq)
+	spanReqDo.Finish()
 	if err != nil {
 		return nil, errors.Wrap(err, "send request")
 	}
-	spanReqDo.Finish()
 	if presp.StatusCode/100 != 2 {
 		// Best effort read.
 		b, err := ioutil.ReadAll(presp.Body)