feat: first iteration on datadog tracing with spans

server/config/options.go

@@ -34,6 +34,7 @@ type Config struct {
 	Tracing      TracingConfig   `yaml:"tracing" json:"tracing"`
 	Profiling    ProfilingConfig `yaml:"profiling" json:"profiling"`
 	Metrics      MetricsConfig
+	DatadogTrace DatadogTraceConfig
 	FoundationDB FoundationDBConfig
 }
 
@@ -81,6 +82,11 @@ type MetricsConfig struct {
 	Search SearchMetricsConfig
 }
 
+type DatadogTraceConfig struct {
+	// Global switch
+	Enabled bool `mapstructure:"enabled" yaml:"enabled" json:"enabled"`
+}
+
 type GrpcMetricsConfig struct {
 	Enabled      bool `mapstructure:"enabled" yaml:"enabled" json:"enabled"`
 	Counters     bool `mapstructure:"counters" yaml:"counters" json:"counters"`
@@ -157,6 +163,9 @@ var DefaultConfig = Config{
 			ResponseTime: true,
 		},
 	},
+	DatadogTrace: DatadogTraceConfig{
+		Enabled: true,
+	},
 }
 
 // FoundationDBConfig keeps FoundationDB configuration parameters

server/midddleware/ddtrace.go

@@ -0,0 +1,84 @@
+// Copyright 2022 Tigris Data, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package middleware
+
+import (
+	"context"
+	"fmt"
+	"github.com/tigrisdata/tigris/server/metrics"
+	"github.com/tigrisdata/tigris/server/request"
+	ulog "github.com/tigrisdata/tigris/util/log"
+	"google.golang.org/grpc"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
+	"strings"
+)
+
+func getMethodName(fullMethod string) string {
+	return strings.Split(fullMethod, "/")[2]
+}
+
+func getServiceName(fullMethod string) string {
+	return strings.Split(fullMethod, "/")[1]
+}
+
+func getSpanName(fullMethod string) string {
+	return fmt.Sprintf("tigris.server.grpc.%s", getMethodName(fullMethod))
+}
+
+func getNamespaceName(ctx context.Context) string {
+	namespace, err := request.GetNamespace(ctx)
+	if ulog.E(err) && err == request.ErrNamespaceNotFound {
+		return metrics.DefaultReportedTigrisTenant
+	} else {
+		return namespace
+	}
+}
+
+func getTags(ctx context.Context, fullMethod string, serviceType string) map[string]string {
+	return map[string]string{
+		"method":            getMethodName(fullMethod),
+		"tigris_tenant":     getNamespaceName(ctx),
+		"grpc_service_name": getServiceName(fullMethod),
+		"grpc_service_type": serviceType,
+	}
+}
+
+func ddTraceUnary() func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+	return func(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) {
+		span := tracer.StartSpan(info.FullMethod)
+		ctx = tracer.ContextWithSpan(ctx, span)
+		for k, v := range getTags(ctx, info.FullMethod, "unary") {
+			span.SetTag(k, v)
+		}
+		defer span.Finish()
+		resp, err := handler(ctx, req)
+		return resp, err
+	}
+}
+
+func ddTraceStream() grpc.StreamServerInterceptor {
+	return func(srv interface{}, stream grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {
+		ctx := stream.Context()
+		span := tracer.StartSpan(getSpanName(info.FullMethod))
+		ctx = tracer.ContextWithSpan(ctx, span)
+		for k, v := range getTags(ctx, info.FullMethod, "stream") {
+			span.SetTag(k, v)
+		}
+		defer span.Finish()
+		wrapper := &recvWrapper{stream}
+		err := handler(srv, wrapper)
+		return err
+	}
+}

server/midddleware/metrics.go

@@ -17,16 +17,14 @@ package middleware
 import (
 	"context"
 	"errors"
-	"strconv"
-
-	"github.com/tigrisdata/tigris/server/request"
-	ulog "github.com/tigrisdata/tigris/util/log"
-	"github.com/uber-go/tally"
-
 	"github.com/apple/foundationdb/bindings/go/src/fdb"
 	api "github.com/tigrisdata/tigris/api/server/v1"
 	"github.com/tigrisdata/tigris/server/metrics"
+	"github.com/tigrisdata/tigris/server/request"
+	ulog "github.com/tigrisdata/tigris/util/log"
+	"github.com/uber-go/tally"
 	"google.golang.org/grpc"
+	"strconv"
 )
 
 // There is no need to handle metrics configuration in the interceptors themselves, because they are not

server/midddleware/middleware.go

@@ -84,6 +84,10 @@ func Get(config *config.Config, tenantMgr *metadata.TenantManager, txMgr *transa
 		streamInterceptors = append(streamInterceptors, metricsStreamServerInterceptorCounter())
 	}
 
+	if config.DatadogTrace.Enabled {
+		streamInterceptors = append(streamInterceptors, ddTraceStream())
+	}
+
 	streamInterceptors = append(streamInterceptors, []grpc.StreamServerInterceptor{
 		grpc_opentracing.StreamServerInterceptor(),
 		grpc_recovery.StreamServerInterceptor(),
@@ -117,6 +121,10 @@ func Get(config *config.Config, tenantMgr *metadata.TenantManager, txMgr *transa
 		unaryInterceptors = append(unaryInterceptors, metricsUnaryServerInterceptorCounters())
 	}
 
+	if config.DatadogTrace.Enabled {
+		unaryInterceptors = append(unaryInterceptors, ddTraceUnary())
+	}
+
 	unaryInterceptors = append(unaryInterceptors, []grpc.UnaryServerInterceptor{
 		grpc_opentracing.UnaryServerInterceptor(),
 		grpc_recovery.UnaryServerInterceptor(),

server/transaction/manager.go

@@ -16,6 +16,8 @@ package transaction
 
 import (
 	"context"
+	"github.com/tigrisdata/tigris/server/config"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 	"sync"
 
 	"github.com/google/uuid"
@@ -135,6 +137,12 @@ func (s *TxSession) GetTxCtx() *api.TransactionCtx {
 }
 
 func (s *TxSession) start(ctx context.Context) error {
+	if config.DefaultConfig.DatadogTrace.Enabled {
+		span, _ := tracer.StartSpanFromContext(ctx, "StartTxSession")
+		defer span.Finish()
+		span.SetTag("fdb_tx", "Start")
+	}
+
 	s.Lock()
 	defer s.Unlock()
 
@@ -229,6 +237,7 @@ func (s *TxSession) SetVersionstampedValue(ctx context.Context, key []byte, valu
 }
 
 func (s *TxSession) SetVersionstampedKey(ctx context.Context, key []byte, value []byte) error {
+
 	s.Lock()
 	defer s.Unlock()
 
@@ -240,6 +249,11 @@ func (s *TxSession) SetVersionstampedKey(ctx context.Context, key []byte, value
 }
 
 func (s *TxSession) Get(ctx context.Context, key []byte) ([]byte, error) {
+	if config.DefaultConfig.DatadogTrace.Enabled {
+		span, _ := tracer.StartSpanFromContext(ctx, "GetTxSession")
+		defer span.Finish()
+		span.SetTag("fdb_tx", "Get")
+	}
 	s.Lock()
 	defer s.Unlock()
 
@@ -251,6 +265,12 @@ func (s *TxSession) Get(ctx context.Context, key []byte) ([]byte, error) {
 }
 
 func (s *TxSession) Commit(ctx context.Context) error {
+	if config.DefaultConfig.DatadogTrace.Enabled {
+		span, _ := tracer.StartSpanFromContext(ctx, "CommitTxSession")
+		defer span.Finish()
+		span.SetTag("fdb_tx", "Commit")
+	}
+
 	s.Lock()
 	defer s.Unlock()
 

store/kv/kv.go

@@ -17,6 +17,7 @@ package kv
 import (
 	"context"
 	"errors"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 	"strconv"
 	"unsafe"
 
@@ -102,6 +103,11 @@ func measureLow(ctx context.Context, name string, f func() error) {
 		metrics.FdbRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets)
 		defer metrics.FdbRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets).Start().Stop()
 	}
+	if config.DefaultConfig.DatadogTrace.Enabled {
+		span, _ := tracer.StartSpanFromContext(ctx, name)
+		defer span.Finish()
+		span.SetTag("fdb_op_name", name)
+	}
 	err := f()
 	if err == nil {
 		// Request was ok

store/search/ts.go

@@ -26,6 +26,7 @@ import (
 	"github.com/typesense/typesense-go/typesense"
 	tsApi "github.com/typesense/typesense-go/typesense/api"
 	"github.com/uber-go/tally"
+	"gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer"
 	"io"
 	"net/http"
 )
@@ -93,6 +94,11 @@ func (m *storeImplWithMetrics) measure(ctx context.Context, name string, f func(
 		metrics.SearchRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets)
 		defer metrics.SearchRequests.Tagged(tags).Histogram("histogram", tally.DefaultBuckets).Start().Stop()
 	}
+	if config.DefaultConfig.DatadogTrace.Enabled {
+		span, _ := tracer.StartSpanFromContext(ctx, name)
+		defer span.Finish()
+		span.SetTag("search_op_name", name)
+	}
 	err := f()
 	if err == nil {
 		// Request was ok