Skip to content

Commit

Permalink
Add interface Tracer, add Opentelemetry (#41963)
Browse files Browse the repository at this point in the history
* Add interface Tracer, add Opentelemetry

* Fix lint

* Fix failing tests and return error if config not parsed fo opentelemetry

* Update defaults.ini

Add comment with jaeger url

* go mod tidy

* Remove comments that are not needed

* Move OpentracingSpan to tracing.go

* Add opentelemetry to sample.ini
  • Loading branch information
idafurjes committed Dec 1, 2021
1 parent be578e5 commit d993b12
Show file tree
Hide file tree
Showing 8 changed files with 181 additions and 11 deletions.
4 changes: 4 additions & 0 deletions conf/defaults.ini
Original file line number Diff line number Diff line change
Expand Up @@ -914,6 +914,10 @@ zipkin_propagation = false
# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
disable_shared_zipkin_spans = false

[tracing.opentelemetry.jaeger]
# jaeger destination (ex http://localhost:14268/api/traces)
address =

#################################### External Image Storage ##############
[external_image_storage]
# Used for uploading images to public servers so they can be included in slack/email messages.
Expand Down
4 changes: 4 additions & 0 deletions conf/sample.ini
Original file line number Diff line number Diff line change
Expand Up @@ -891,6 +891,10 @@
# Not disabling is the most common setting when using Zipkin elsewhere in your infrastructure.
;disable_shared_zipkin_spans = false

[tracing.opentelemetry.jaeger]
# jaeger destination (ex http://localhost:14268/api/traces)
; address = http://localhost:14268/api/traces

#################################### External image storage ##########################
[external_image_storage]
# Used for uploading images to public servers so they can be included in slack/email messages.
Expand Down
11 changes: 7 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -101,10 +101,10 @@ require (
github.com/yudai/gojsondiff v1.0.0
go.opentelemetry.io/collector v0.31.0
go.opentelemetry.io/collector/model v0.31.0
go.opentelemetry.io/otel v1.0.0
go.opentelemetry.io/otel v1.2.0
go.opentelemetry.io/otel/exporters/jaeger v1.0.0
go.opentelemetry.io/otel/sdk v1.0.0
go.opentelemetry.io/otel/trace v1.0.0
go.opentelemetry.io/otel/trace v1.2.0
golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e
golang.org/x/exp v0.0.0-20210220032938-85be41e4509f // indirect
golang.org/x/net v0.0.0-20210903162142-ad29c8ab022f
Expand Down Expand Up @@ -133,7 +133,6 @@ require (
cloud.google.com/go v0.94.1 // indirect
github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0 // indirect
github.com/Azure/go-autorest v14.2.0+incompatible // indirect
github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect
github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
Expand Down Expand Up @@ -189,7 +188,6 @@ require (
github.com/google/flatbuffers v2.0.0+incompatible // indirect
github.com/googleapis/gax-go/v2 v2.1.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect
github.com/grafana/grafana-google-sdk-go v0.0.0-20211104130251-b190293eaf58
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect
github.com/hashicorp/errwrap v1.0.0 // indirect
Expand Down Expand Up @@ -258,6 +256,11 @@ require (
gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
)

require (
github.com/Azure/go-autorest/autorest/adal v0.9.15 // indirect
github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f // indirect
)

// Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream
replace github.com/crewjam/saml => github.com/grafana/saml v0.0.0-20211007135653-aed1b2edd86b

Expand Down
6 changes: 4 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -2470,8 +2470,9 @@ go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.21.0/go.mod h1:
go.opentelemetry.io/contrib/zpages v0.0.0-20210722161726-7668016acb73/go.mod h1:NAkejuYm41lpyL43Fu1XdnCOYxN5NVV80/MJ03JQ/X8=
go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0=
go.opentelemetry.io/otel v1.0.0-RC1/go.mod h1:x9tRa9HK4hSSq7jf2TKbqFbtt58/TGk0f9XiEYISI1I=
go.opentelemetry.io/otel v1.0.0 h1:qTTn6x71GVBvoafHK/yaRUmFzI4LcONZD0/kXxl5PHI=
go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg=
go.opentelemetry.io/otel v1.2.0 h1:YOQDvxO1FayUcT9MIhJhgMyNO1WqoduiyvQHzGN0kUQ=
go.opentelemetry.io/otel v1.2.0/go.mod h1:aT17Fk0Z1Nor9e0uisf98LrntPGMnk4frBO9+dkf69I=
go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U=
go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ=
go.opentelemetry.io/otel/internal/metric v0.21.0/go.mod h1:iOfAaY2YycsXfYD4kaRSbLx2LKmfpKObWBEv9QK5zFo=
Expand All @@ -2481,8 +2482,9 @@ go.opentelemetry.io/otel/sdk v1.0.0-RC1/go.mod h1:kj6yPn7Pgt5ByRuwesbaWcRLA+V7BS
go.opentelemetry.io/otel/sdk v1.0.0 h1:BNPMYUONPNbLneMttKSjQhOTlFLOD9U22HNG1KrIN2Y=
go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM=
go.opentelemetry.io/otel/trace v1.0.0-RC1/go.mod h1:86UHmyHWFEtWjfWPSbu0+d0Pf9Q6e1U+3ViBOc+NXAg=
go.opentelemetry.io/otel/trace v1.0.0 h1:TSBr8GTEtKevYMG/2d21M989r5WJYVimhTHBKVEZuh4=
go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs=
go.opentelemetry.io/otel/trace v1.2.0 h1:Ys3iqbqZhcf28hHzrm5WAquMkDHNZTUkw7KHbuNjej0=
go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0=
go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
go.starlark.net v0.0.0-20200901195727-6e684ef5eeee/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU=
go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
Expand Down
4 changes: 2 additions & 2 deletions pkg/api/http_server.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ type HTTPServer struct {
SecretsService secrets.Service
DataSourcesService *datasources.Service
cleanUpService *cleanup.CleanUpService
tracingService *tracing.TracingService
tracingService tracing.Tracer
internalMetricsSvc *metrics.InternalMetricsService
updateChecker *updatechecker.Service
searchUsersService searchusers.Service
Expand All @@ -136,7 +136,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi
contextHandler *contexthandler.ContextHandler,
schemaService *schemaloader.SchemaLoaderService, alertNG *ngalert.AlertNG,
libraryPanelService librarypanels.Service, libraryElementService libraryelements.Service,
notificationService *notifications.NotificationService, tracingService *tracing.TracingService,
notificationService *notifications.NotificationService, tracingService tracing.Tracer,
internalMetricsSvc *metrics.InternalMetricsService, quotaService *quota.QuotaService,
socialService social.Service, oauthTokenService oauthtoken.OAuthTokenService,
encryptionService encryption.Internal, updateChecker *updatechecker.Service, searchUsersService searchusers.Service,
Expand Down
123 changes: 123 additions & 0 deletions pkg/infra/tracing/opentelemetry_tracing.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package tracing

import (
"context"
"time"

"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/setting"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/exporters/jaeger"
"go.opentelemetry.io/otel/sdk/resource"
tracesdk "go.opentelemetry.io/otel/sdk/trace"
semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
trace "go.opentelemetry.io/otel/trace"
)

type Tracer interface {
Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span)
Run(context.Context) error
}

type Span interface {
End()
SetAttributes(kv ...attribute.KeyValue)
}

var (
GlobalTracer trace.Tracer
)

type OpentelemetryTracingService struct {
enabled bool
address string
log log.Logger

tracerProvider *tracesdk.TracerProvider

Cfg *setting.Cfg
}

type OpentelemetrySpan struct {
span trace.Span
}

func (ots *OpentelemetryTracingService) parseSettingsOpentelemetry() error {
section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
if err != nil {
return err
}

ots.address = section.Key("address").MustString("")
if ots.address != "" {
ots.enabled = true
}

return nil
}

func (ots *OpentelemetryTracingService) initTracerProvider() (*tracesdk.TracerProvider, error) {
// Create the Jaeger exporter
exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
if err != nil {
return nil, err
}

tp := tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exp),
tracesdk.WithResource(resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceNameKey.String("grafana"),
attribute.String("environment", "production"),
)),
)

return tp, nil
}

func (ots *OpentelemetryTracingService) initOpentelemetryTracer() error {
tp, err := ots.initTracerProvider()
if err != nil {
return err
}
// Register our TracerProvider as the global so any imported
// instrumentation in the future will default to using it
// only if tracing is enabled
if ots.enabled {
otel.SetTracerProvider(tp)
}

ots.tracerProvider = tp
GlobalTracer = otel.GetTracerProvider().Tracer("component-main")

return nil
}

func (ots *OpentelemetryTracingService) Run(ctx context.Context) error {
<-ctx.Done()

ots.log.Info("Closing tracing")
ctxShutdown, cancel := context.WithTimeout(ctx, time.Second*5)
defer cancel()

if err := ots.tracerProvider.Shutdown(ctxShutdown); err != nil {
return err
}

return nil
}

func (ots *OpentelemetryTracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
ctx, span := GlobalTracer.Start(ctx, spanName)
oSpan := OpentelemetrySpan{span: span}
return ctx, oSpan
}

func (s OpentelemetrySpan) End() {
s.span.End()
}

func (s OpentelemetrySpan) SetAttributes(kv ...attribute.KeyValue) {
s.span.SetAttributes(kv...)
}
38 changes: 36 additions & 2 deletions pkg/infra/tracing/tracing.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,8 @@ import (

"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/setting"
"go.opentelemetry.io/otel/attribute"
trace "go.opentelemetry.io/otel/trace"

opentracing "github.com/opentracing/opentracing-go"
jaegercfg "github.com/uber/jaeger-client-go/config"
Expand All @@ -20,11 +22,12 @@ const (
envJaegerAgentPort = "JAEGER_AGENT_PORT"
)

func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
func ProvideService(cfg *setting.Cfg) (Tracer, error) {
ts := &TracingService{
Cfg: cfg,
log: log.New("tracing"),
}

if err := ts.parseSettings(); err != nil {
return nil, err
}
Expand All @@ -33,7 +36,16 @@ func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
return ts, ts.initGlobalTracer()
}

return ts, nil
ots := &OpentelemetryTracingService{
Cfg: cfg,
log: log.New("tracing"),
}

if err := ots.parseSettingsOpentelemetry(); err != nil {
return nil, err
}

return ots, ots.initOpentelemetryTracer()
}

type TracingService struct {
Expand All @@ -51,6 +63,10 @@ type TracingService struct {
Cfg *setting.Cfg
}

type OpentracingSpan struct {
span opentracing.Span
}

func (ts *TracingService) parseSettings() error {
var section, err = ts.Cfg.Raw.GetSection("tracing.jaeger")
if err != nil {
Expand Down Expand Up @@ -150,6 +166,24 @@ func (ts *TracingService) Run(ctx context.Context) error {
return nil
}

func (ts *TracingService) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
span, ctx := opentracing.StartSpanFromContext(ctx, spanName)
oSpan := OpentracingSpan{
span: span,
}
return ctx, oSpan
}

func (s OpentracingSpan) End() {
s.span.Finish()
}

func (s OpentracingSpan) SetAttributes(kv ...attribute.KeyValue) {
for k, v := range kv {
s.span.SetTag(fmt.Sprint(k), v)
}
}

func splitTagSettings(input string) map[string]string {
res := map[string]string{}

Expand Down
2 changes: 1 addition & 1 deletion pkg/server/backgroundsvcs/background_services.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ func ProvideBackgroundServiceRegistry(
rendering *rendering.RenderingService, tokenService models.UserTokenBackgroundService,
provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, pm *manager.PluginManager,
metrics *metrics.InternalMetricsService, usageStats *uss.UsageStats, updateChecker *updatechecker.Service,
tracing *tracing.TracingService, remoteCache *remotecache.RemoteCache,
tracing tracing.Tracer, remoteCache *remotecache.RemoteCache,
// Need to make sure these are initialized, is there a better place to put them?
_ *azuremonitor.Service, _ *cloudwatch.CloudWatchService, _ *elasticsearch.Service, _ *graphite.Service,
_ *influxdb.Service, _ *loki.Service, _ *opentsdb.Service, _ *prometheus.Service, _ *tempo.Service,
Expand Down

0 comments on commit d993b12

Please sign in to comment.