diff --git a/.github/workflows/apps/appsec-test-contrib-submodules.sh b/.github/workflows/apps/appsec-test-contrib-submodules.sh index 8dbf577e48..bca4177520 100755 --- a/.github/workflows/apps/appsec-test-contrib-submodules.sh +++ b/.github/workflows/apps/appsec-test-contrib-submodules.sh @@ -41,7 +41,16 @@ fi $runner "$JUNIT_REPORT.xml" "." ./appsec/... ./internal/appsec/... -SCOPES=("gin-gonic/gin" "google.golang.org/grpc" "net/http" "gorilla/mux" "go-chi/chi" "go-chi/chi.v5" "labstack/echo.v4") +SCOPES=( + "gin-gonic/gin" \ + "google.golang.org/grpc" \ + "net/http" "gorilla/mux" \ + "go-chi/chi" "go-chi/chi.v5" \ + "labstack/echo.v4" \ + "99designs/gqlgen" \ + "graphql-go/graphql" \ + "graph-gophers/graphql-go" +) for SCOPE in "${SCOPES[@]}"; do contrib=$(basename "$SCOPE") echo "Running appsec tests for contrib/$SCOPE" diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 71b07825e7..c42ac619a1 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -30,7 +30,7 @@ jobs: native: strategy: matrix: - runs-on: [ macos-13, macos-12, macos-11, ubuntu-22.04, ubuntu-20.04 ] + runs-on: [ macos-14, macos-13, macos-12, ubuntu-22.04, ubuntu-20.04 ] go-version: [ "1.21", "1.20", "1.19" ] cgo_enabled: [ "0", "1" ] # test it compiles with and without cgo appsec_enabled: # test it compiles with and without appsec enabled @@ -44,22 +44,16 @@ jobs: cgocheck: # 1.21 deprecates the GODEBUG=cgocheck=2 value, replacing it with GOEXPERIMENT=cgocheck2 GOEXPERIMENT=cgocheck2 fail-fast: false + name: native ${{ toJSON(matrix) }} runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.ref }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - - name: Go modules cache - uses: actions/cache@v3 - with: - path: ~/go/pkg/mod - key: go-pkg-mod-${{ hashFiles('**/go.sum') }} - restore-keys: go-pkg-mod- - - name: go test shell: bash run: | @@ -74,7 +68,7 @@ jobs: files: ${{ env.JUNIT_REPORT }}*.xml tags: go:${{ matrix.go-version }},arch:${{ runner.arch }},os:${{ runner.os }} - # Tests cases were appsec end up being disable + # Tests cases were appsec end up being disabled waf-disabled: strategy: fail-fast: false @@ -87,20 +81,14 @@ jobs: include: - runs-on: windows-latest go-args: "" + name: disabled ${{ toJSON(matrix) }} runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 # TODO: rely on v4 which now provides github caching by default + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: stable - - name: Go modules cache - uses: actions/cache@v3 - with: - path: ~/go/pkg/mod - key: go-pkg-mod-${{ hashFiles('**/go.sum') }} - restore-keys: go-pkg-mod- - - name: go test shell: bash run: | @@ -118,6 +106,7 @@ jobs: # Same tests but on the official golang container for linux golang-linux-container: + name: golang-containers ${{ toJSON(matrix) }} runs-on: ubuntu-latest container: image: golang:${{ matrix.go-version }}-${{ matrix.distribution }} @@ -138,7 +127,7 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.ref }} # Install gcc and the libc headers on alpine images @@ -169,6 +158,7 @@ jobs: linux-arm64: runs-on: ubuntu-latest + name: linux/arm64 ${{ toJSON(matrix) }} strategy: matrix: cgo_enabled: # test it compiles with and without the cgo @@ -179,17 +169,17 @@ jobs: - false fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.ref }} - name: Go modules cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/go/pkg/mod key: go-pkg-mod-${{ hashFiles('**/go.sum') }} restore-keys: go-pkg-mod- - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - run: | diff --git a/.github/workflows/ecosystems-label-issue copy.yml b/.github/workflows/ecosystems-label-issue copy.yml new file mode 100644 index 0000000000..f63226c003 --- /dev/null +++ b/.github/workflows/ecosystems-label-issue copy.yml @@ -0,0 +1,17 @@ +name: Label APM Ecosystems issues +on: + issues: + types: + - reopened + - opened + - edited +jobs: + label_issues: + if: contains(github.event.issue.title, 'contrib') + runs-on: ubuntu-latest + steps: + # https://github.com/marketplace/actions/actions-ecosystem-add-labels + - name: add label + uses: actions-ecosystem/action-add-labels@v1 + with: + labels: apm:ecosystem diff --git a/.github/workflows/ecosystems-label.yml b/.github/workflows/ecosystems-label-pr.yml similarity index 73% rename from .github/workflows/ecosystems-label.yml rename to .github/workflows/ecosystems-label-pr.yml index 909c1db005..36f35b5422 100644 --- a/.github/workflows/ecosystems-label.yml +++ b/.github/workflows/ecosystems-label-pr.yml @@ -1,12 +1,5 @@ -name: Label APM Ecosystems issues +name: Label APM Ecosystems Pull Requests on: - issues: - paths: - - "contrib/**" - types: - - reopened - - opened - - edited pull_request: paths: - "contrib/**" diff --git a/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka.go b/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka.go index 8c7ae41542..21b89369f2 100644 --- a/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka.go +++ b/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka.go @@ -91,6 +91,7 @@ func (c *Consumer) traceEventsChannel(in chan kafka.Event) chan kafka.Event { setConsumeCheckpoint(c.cfg.dataStreamsEnabled, c.cfg.groupID, msg) } else if offset, ok := evt.(kafka.OffsetsCommitted); ok { commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, offset.Offsets, offset.Error) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, offset.Offsets) } out <- evt @@ -176,10 +177,22 @@ func (c *Consumer) Poll(timeoutMS int) (event kafka.Event) { c.prev = c.startSpan(msg) } else if offset, ok := evt.(kafka.OffsetsCommitted); ok { commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, offset.Offsets, offset.Error) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, offset.Offsets) } return evt } +func (c *Consumer) trackHighWatermark(dataStreamsEnabled bool, offsets []kafka.TopicPartition) { + if !dataStreamsEnabled { + return + } + for _, tp := range offsets { + if _, high, err := c.Consumer.GetWatermarkOffsets(*tp.Topic, tp.Partition); err == nil { + tracer.TrackKafkaHighWatermarkOffset("", *tp.Topic, tp.Partition, high) + } + } +} + // ReadMessage polls the consumer for a message. Message will be traced. func (c *Consumer) ReadMessage(timeout time.Duration) (*kafka.Message, error) { if c.prev != nil { @@ -199,6 +212,7 @@ func (c *Consumer) ReadMessage(timeout time.Duration) (*kafka.Message, error) { func (c *Consumer) Commit() ([]kafka.TopicPartition, error) { tps, err := c.Consumer.Commit() commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } @@ -206,6 +220,7 @@ func (c *Consumer) Commit() ([]kafka.TopicPartition, error) { func (c *Consumer) CommitMessage(msg *kafka.Message) ([]kafka.TopicPartition, error) { tps, err := c.Consumer.CommitMessage(msg) commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } @@ -213,6 +228,7 @@ func (c *Consumer) CommitMessage(msg *kafka.Message) ([]kafka.TopicPartition, er func (c *Consumer) CommitOffsets(offsets []kafka.TopicPartition) ([]kafka.TopicPartition, error) { tps, err := c.Consumer.CommitOffsets(offsets) commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } diff --git a/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka_test.go b/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka_test.go index 32dfa1e9eb..8efa05fc58 100644 --- a/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka_test.go +++ b/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka_test.go @@ -9,6 +9,7 @@ import ( "context" "errors" "os" + "strings" "testing" "time" @@ -17,6 +18,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + internaldsm "gopkg.in/DataDog/dd-trace-go.v1/internal/datastreams" "github.com/confluentinc/confluent-kafka-go/v2/kafka" "github.com/stretchr/testify/assert" @@ -76,6 +78,8 @@ func produceThenConsume(t *testing.T, consumerAction consumerActionFn, producerO msg2, err := consumerAction(c) require.NoError(t, err) + _, err = c.CommitMessage(msg2) + require.NoError(t, err) assert.Equal(t, msg1.String(), msg2.String()) err = c.Close() require.NoError(t, err) @@ -84,6 +88,21 @@ func produceThenConsume(t *testing.T, consumerAction consumerActionFn, producerO require.Len(t, spans, 2) // they should be linked via headers assert.Equal(t, spans[0].TraceID(), spans[1].TraceID()) + + if c.cfg.dataStreamsEnabled { + backlogs := mt.SentDSMBacklogs() + toMap := func(b []internaldsm.Backlog) map[string]struct{} { + m := make(map[string]struct{}) + for _, b := range backlogs { + m[strings.Join(b.Tags, "")] = struct{}{} + } + return m + } + backlogsMap := toMap(backlogs) + require.Contains(t, backlogsMap, "consumer_group:"+testGroupID+"partition:0"+"topic:"+testTopic+"type:kafka_commit") + require.Contains(t, backlogsMap, "partition:0"+"topic:"+testTopic+"type:kafka_high_watermark") + require.Contains(t, backlogsMap, "partition:0"+"topic:"+testTopic+"type:kafka_produce") + } return spans, msg2 } diff --git a/contrib/confluentinc/confluent-kafka-go/kafka/kafka.go b/contrib/confluentinc/confluent-kafka-go/kafka/kafka.go index c0c9a91c29..4de425ca56 100644 --- a/contrib/confluentinc/confluent-kafka-go/kafka/kafka.go +++ b/contrib/confluentinc/confluent-kafka-go/kafka/kafka.go @@ -91,6 +91,7 @@ func (c *Consumer) traceEventsChannel(in chan kafka.Event) chan kafka.Event { setConsumeCheckpoint(c.cfg.dataStreamsEnabled, c.cfg.groupID, msg) } else if offset, ok := evt.(kafka.OffsetsCommitted); ok { commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, offset.Offsets, offset.Error) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, offset.Offsets) } out <- evt @@ -176,10 +177,22 @@ func (c *Consumer) Poll(timeoutMS int) (event kafka.Event) { c.prev = c.startSpan(msg) } else if offset, ok := evt.(kafka.OffsetsCommitted); ok { commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, offset.Offsets, offset.Error) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, offset.Offsets) } return evt } +func (c *Consumer) trackHighWatermark(dataStreamsEnabled bool, offsets []kafka.TopicPartition) { + if !dataStreamsEnabled { + return + } + for _, tp := range offsets { + if _, high, err := c.Consumer.GetWatermarkOffsets(*tp.Topic, tp.Partition); err == nil { + tracer.TrackKafkaHighWatermarkOffset("", *tp.Topic, tp.Partition, high) + } + } +} + // ReadMessage polls the consumer for a message. Message will be traced. func (c *Consumer) ReadMessage(timeout time.Duration) (*kafka.Message, error) { if c.prev != nil { @@ -199,6 +212,7 @@ func (c *Consumer) ReadMessage(timeout time.Duration) (*kafka.Message, error) { func (c *Consumer) Commit() ([]kafka.TopicPartition, error) { tps, err := c.Consumer.Commit() commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } @@ -206,6 +220,7 @@ func (c *Consumer) Commit() ([]kafka.TopicPartition, error) { func (c *Consumer) CommitMessage(msg *kafka.Message) ([]kafka.TopicPartition, error) { tps, err := c.Consumer.CommitMessage(msg) commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } @@ -213,6 +228,7 @@ func (c *Consumer) CommitMessage(msg *kafka.Message) ([]kafka.TopicPartition, er func (c *Consumer) CommitOffsets(offsets []kafka.TopicPartition) ([]kafka.TopicPartition, error) { tps, err := c.Consumer.CommitOffsets(offsets) commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } diff --git a/contrib/confluentinc/confluent-kafka-go/kafka/kafka_test.go b/contrib/confluentinc/confluent-kafka-go/kafka/kafka_test.go index 2196beda41..4707a1e5ae 100644 --- a/contrib/confluentinc/confluent-kafka-go/kafka/kafka_test.go +++ b/contrib/confluentinc/confluent-kafka-go/kafka/kafka_test.go @@ -9,6 +9,7 @@ import ( "context" "errors" "os" + "strings" "testing" "time" @@ -17,6 +18,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + internaldsm "gopkg.in/DataDog/dd-trace-go.v1/internal/datastreams" "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/stretchr/testify/assert" @@ -76,6 +78,8 @@ func produceThenConsume(t *testing.T, consumerAction consumerActionFn, producerO msg2, err := consumerAction(c) require.NoError(t, err) + _, err = c.CommitMessage(msg2) + require.NoError(t, err) assert.Equal(t, msg1.String(), msg2.String()) err = c.Close() require.NoError(t, err) @@ -84,6 +88,21 @@ func produceThenConsume(t *testing.T, consumerAction consumerActionFn, producerO require.Len(t, spans, 2) // they should be linked via headers assert.Equal(t, spans[0].TraceID(), spans[1].TraceID()) + + if c.cfg.dataStreamsEnabled { + backlogs := mt.SentDSMBacklogs() + toMap := func(b []internaldsm.Backlog) map[string]struct{} { + m := make(map[string]struct{}) + for _, b := range backlogs { + m[strings.Join(b.Tags, "")] = struct{}{} + } + return m + } + backlogsMap := toMap(backlogs) + require.Contains(t, backlogsMap, "consumer_group:"+testGroupID+"partition:0"+"topic:"+testTopic+"type:kafka_commit") + require.Contains(t, backlogsMap, "partition:0"+"topic:"+testTopic+"type:kafka_high_watermark") + require.Contains(t, backlogsMap, "partition:0"+"topic:"+testTopic+"type:kafka_produce") + } return spans, msg2 } diff --git a/contrib/internal/httptrace/config_test.go b/contrib/internal/httptrace/config_test.go index c052292346..6b6856d01c 100644 --- a/contrib/internal/httptrace/config_test.go +++ b/contrib/internal/httptrace/config_test.go @@ -6,7 +6,6 @@ package httptrace import ( - "os" "testing" "github.com/stretchr/testify/require" @@ -50,9 +49,8 @@ func TestConfig(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - defer cleanEnv()() for k, v := range tc.env { - os.Setenv(k, v) + t.Setenv(k, v) } c := newConfig() require.Equal(t, tc.cfg.queryStringRegexp, c.queryStringRegexp) @@ -60,18 +58,3 @@ func TestConfig(t *testing.T) { }) } } - -func cleanEnv() func() { - env := map[string]string{ - envQueryStringDisabled: os.Getenv(envQueryStringDisabled), - envQueryStringRegexp: os.Getenv(envQueryStringRegexp), - } - for k := range env { - os.Unsetenv(k) - } - return func() { - for k, v := range env { - os.Setenv(k, v) - } - } -} diff --git a/contrib/labstack/echo.v4/echotrace.go b/contrib/labstack/echo.v4/echotrace.go index 3911036d60..ddbf53da08 100644 --- a/contrib/labstack/echo.v4/echotrace.go +++ b/contrib/labstack/echo.v4/echotrace.go @@ -89,7 +89,7 @@ func Middleware(opts ...Option) echo.MiddlewareFunc { } // serve the request to the next middleware err := next(c) - if err != nil { + if err != nil && !shouldIgnoreError(cfg, err) { // invokes the registered HTTP error handler c.Error(err) @@ -109,12 +109,16 @@ func Middleware(opts ...Option) echo.MiddlewareFunc { } } else if status := c.Response().Status; status > 0 { if cfg.isStatusError(status) { - finishOpts = append(finishOpts, tracer.WithError(fmt.Errorf("%d: %s", status, http.StatusText(status)))) + if statusErr := errorFromStatusCode(status); !shouldIgnoreError(cfg, statusErr) { + finishOpts = append(finishOpts, tracer.WithError(statusErr)) + } } span.SetTag(ext.HTTPCode, strconv.Itoa(status)) } else { if cfg.isStatusError(200) { - finishOpts = append(finishOpts, tracer.WithError(fmt.Errorf("%d: %s", 200, http.StatusText(200)))) + if statusErr := errorFromStatusCode(200); !shouldIgnoreError(cfg, statusErr) { + finishOpts = append(finishOpts, tracer.WithError(statusErr)) + } } span.SetTag(ext.HTTPCode, "200") } @@ -122,3 +126,11 @@ func Middleware(opts ...Option) echo.MiddlewareFunc { } } } + +func errorFromStatusCode(statusCode int) error { + return fmt.Errorf("%d: %s", statusCode, http.StatusText(statusCode)) +} + +func shouldIgnoreError(cfg *config, err error) bool { + return cfg.errCheck != nil && !cfg.errCheck(err) +} diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index 01eb5f873e..50af8802d1 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -589,6 +589,128 @@ func TestWithHeaderTags(t *testing.T) { }) } +func TestWithErrorCheck(t *testing.T) { + tests := []struct { + name string + err error + opts []Option + wantErr error + }{ + { + name: "ignore-4xx-404-error", + err: &echo.HTTPError{ + Code: http.StatusNotFound, + Message: "not found", + Internal: errors.New("not found"), + }, + opts: []Option{ + WithErrorCheck(func(err error) bool { + var he *echo.HTTPError + if errors.As(err, &he) { + // do not tag 4xx errors + return !(he.Code < 500 && he.Code >= 400) + } + return true + }), + }, + wantErr: nil, // 404 is returned, hence not tagged + }, + { + name: "ignore-4xx-500-error", + err: &echo.HTTPError{ + Code: http.StatusInternalServerError, + Message: "internal error", + Internal: errors.New("internal error"), + }, + opts: []Option{ + WithErrorCheck(func(err error) bool { + var he *echo.HTTPError + if errors.As(err, &he) { + // do not tag 4xx errors + return !(he.Code < 500 && he.Code >= 400) + } + return true + }), + }, + wantErr: &echo.HTTPError{ + Code: http.StatusInternalServerError, + Message: "internal error", + Internal: errors.New("internal error"), + }, // this is 500, tagged + }, + { + name: "ignore-none", + err: errors.New("any error"), + opts: []Option{ + WithErrorCheck(func(err error) bool { + return true + }), + }, + wantErr: errors.New("any error"), + }, + { + name: "ignore-all", + err: errors.New("any error"), + opts: []Option{ + WithErrorCheck(func(err error) bool { + return false + }), + }, + wantErr: nil, + }, + { + // withErrorCheck also runs for the errors created from the WithStatusCheck option. + name: "ignore-errors-from-status-check", + err: &echo.HTTPError{ + Code: http.StatusNotFound, + Message: "internal error", + Internal: errors.New("internal error"), + }, + opts: []Option{ + WithStatusCheck(func(statusCode int) bool { + return statusCode == http.StatusNotFound + }), + WithErrorCheck(func(err error) bool { + return false + }), + }, + wantErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mt := mocktracer.Start() + defer mt.Stop() + + router := echo.New() + router.Use(Middleware(tt.opts...)) + var called, traced bool + + // always return the specified error + router.GET("/err", func(c echo.Context) error { + _, traced = tracer.SpanFromContext(c.Request().Context()) + called = true + return tt.err + }) + r := httptest.NewRequest(http.MethodGet, "/err", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, r) + + assert.True(t, called) + assert.True(t, traced) + spans := mt.FinishedSpans() + require.Len(t, spans, 1) // fail at once if there is no span + + span := spans[0] + if tt.wantErr == nil { + assert.NotContains(t, span.Tags(), ext.Error) + return + } + assert.Equal(t, tt.wantErr, span.Tag(ext.Error)) + }) + } +} + func TestWithCustomTags(t *testing.T) { assert := assert.New(t) mt := mocktracer.Start() diff --git a/contrib/labstack/echo.v4/option.go b/contrib/labstack/echo.v4/option.go index 9ce8263890..2994d42c61 100644 --- a/contrib/labstack/echo.v4/option.go +++ b/contrib/labstack/echo.v4/option.go @@ -27,6 +27,7 @@ type config struct { isStatusError func(statusCode int) bool translateError func(err error) (*echo.HTTPError, bool) headerTags *internal.LockMap + errCheck func(error) bool tags map[string]interface{} } @@ -129,6 +130,14 @@ func WithHeaderTags(headers []string) Option { } } +// WithErrorCheck sets the func which determines if err would be ignored (if it returns true, the error is not tagged). +// This function also checks the errors created from the WithStatusCheck option. +func WithErrorCheck(errCheck func(error) bool) Option { + return func(cfg *config) { + cfg.errCheck = errCheck + } +} + // WithCustomTag will attach the value to the span tagged by the key. Standard // span tags cannot be replaced. func WithCustomTag(key string, value interface{}) Option { diff --git a/ddtrace/ddtrace.go b/ddtrace/ddtrace.go index c4d106458c..e311b5ff25 100644 --- a/ddtrace/ddtrace.go +++ b/ddtrace/ddtrace.go @@ -94,6 +94,25 @@ type SpanContext interface { ForeachBaggageItem(handler func(k, v string) bool) } +// SpanLink represents a reference to a span that exists outside of the trace. +// +//go:generate msgp -unexported -marshal=false -o=span_link_msgp.go -tests=false + +type SpanLink struct { + // TraceID represents the low 64 bits of the linked span's trace id. This field is required. + TraceID uint64 `msg:"trace_id" json:"trace_id"` + // TraceIDHigh represents the high 64 bits of the linked span's trace id. This field is only set if the linked span's trace id is 128 bits. + TraceIDHigh uint64 `msg:"trace_id_high,omitempty" json:"trace_id_high"` + // SpanID represents the linked span's span id. + SpanID uint64 `msg:"span_id" json:"span_id"` + // Attributes is a mapping of keys to string values. These values are used to add additional context to the span link. + Attributes map[string]string `msg:"attributes,omitempty" json:"attributes"` + // Tracestate is the tracestate of the linked span. This field is optional. + Tracestate string `msg:"tracestate,omitempty" json:"tracestate"` + // Flags represents the W3C trace flags of the linked span. This field is optional. + Flags uint32 `msg:"flags,omitempty" json:"flags"` +} + // StartSpanOption is a configuration option that can be used with a Tracer's StartSpan method. type StartSpanOption func(cfg *StartSpanConfig) @@ -144,6 +163,9 @@ type StartSpanConfig struct { // Context is the parent context where the span should be stored. Context context.Context + + // SpanLink represents a causal relationship between two spans. A span can have multiple links. + SpanLinks []SpanLink } // Logger implementations are able to log given messages that the tracer or profiler might output. diff --git a/ddtrace/mocktracer/data_streams.go b/ddtrace/mocktracer/data_streams.go new file mode 100644 index 0000000000..dcb7191e79 --- /dev/null +++ b/ddtrace/mocktracer/data_streams.go @@ -0,0 +1,45 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package mocktracer + +import ( + "compress/gzip" + "net/http" + + "github.com/tinylib/msgp/msgp" + + "gopkg.in/DataDog/dd-trace-go.v1/internal/datastreams" +) + +type mockDSMTransport struct { + backlogs []datastreams.Backlog +} + +// RoundTrip does nothing and returns a dummy response. +func (t *mockDSMTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // You can customize the dummy response if needed. + gzipReader, err := gzip.NewReader(req.Body) + if err != nil { + return nil, err + } + var p datastreams.StatsPayload + err = msgp.Decode(gzipReader, &p) + if err != nil { + return nil, err + } + for _, bucket := range p.Stats { + t.backlogs = append(t.backlogs, bucket.Backlogs...) + } + return &http.Response{ + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: req, + ContentLength: -1, + Body: http.NoBody, + }, nil +} diff --git a/ddtrace/mocktracer/mockspan.go b/ddtrace/mocktracer/mockspan.go index 506cb1145b..7609fa30c7 100644 --- a/ddtrace/mocktracer/mockspan.go +++ b/ddtrace/mocktracer/mockspan.go @@ -107,6 +107,7 @@ type mockspan struct { parentID uint64 context *spanContext tracer *mocktracer + links []ddtrace.SpanLink } // SetTag sets a given tag on the span. diff --git a/ddtrace/mocktracer/mocktracer.go b/ddtrace/mocktracer/mocktracer.go index 3b748f5c65..0a4252bfd1 100644 --- a/ddtrace/mocktracer/mocktracer.go +++ b/ddtrace/mocktracer/mocktracer.go @@ -37,6 +37,7 @@ type Tracer interface { // FinishedSpans returns the set of finished spans. FinishedSpans() []Span + SentDSMBacklogs() []datastreams.Backlog // Reset resets the spans and services recorded in the tracer. This is // especially useful when running tests in a loop, where a clean start @@ -63,11 +64,25 @@ type mocktracer struct { sync.RWMutex // guards below spans finishedSpans []Span openSpans map[uint64]Span + dsmTransport *mockDSMTransport + dsmProcessor *datastreams.Processor +} + +func (t *mocktracer) SentDSMBacklogs() []datastreams.Backlog { + t.dsmProcessor.Flush() + return t.dsmTransport.backlogs } func newMockTracer() *mocktracer { var t mocktracer t.openSpans = make(map[uint64]Span) + t.dsmTransport = &mockDSMTransport{} + client := &http.Client{ + Transport: t.dsmTransport, + } + t.dsmProcessor = datastreams.NewProcessor(&statsd.NoOpClient{}, "env", "service", "v1", &url.URL{Scheme: "http", Host: "agent-address"}, client, func() bool { return true }) + t.dsmProcessor.Start() + t.dsmProcessor.Flush() return &t } @@ -91,27 +106,8 @@ func (t *mocktracer) StartSpan(operationName string, opts ...ddtrace.StartSpanOp return span } -type noOpTransport struct{} - -// RoundTrip does nothing and returns a dummy response. -func (t *noOpTransport) RoundTrip(req *http.Request) (*http.Response, error) { - // You can customize the dummy response if needed. - return &http.Response{ - StatusCode: 200, - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - Request: req, - ContentLength: -1, - Body: http.NoBody, - }, nil -} - func (t *mocktracer) GetDataStreamsProcessor() *datastreams.Processor { - client := &http.Client{ - Transport: &noOpTransport{}, - } - return datastreams.NewProcessor(&statsd.NoOpClient{}, "env", "service", "v1", &url.URL{Scheme: "http", Host: "agent-address"}, client, func() bool { return true }) + return t.dsmProcessor } func (t *mocktracer) OpenSpans() []Span { diff --git a/ddtrace/opentelemetry/otel_test.go b/ddtrace/opentelemetry/otel_test.go index 81ea64d62a..78c17670f9 100644 --- a/ddtrace/opentelemetry/otel_test.go +++ b/ddtrace/opentelemetry/otel_test.go @@ -11,7 +11,6 @@ import ( "context" "net/http" "net/http/httptest" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -23,6 +22,7 @@ import ( ) func TestHttpDistributedTrace(t *testing.T) { + assert := assert.New(t) tp, payloads, cleanup := mockTracerProvider(t) defer cleanup() otel.SetTracerProvider(tp) @@ -33,11 +33,10 @@ func TestHttpDistributedTrace(t *testing.T) { w := otelhttp.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { receivedSpan := oteltrace.SpanFromContext(r.Context()) - assert.Equal(t, rootSpan.SpanContext().TraceID(), receivedSpan.SpanContext().TraceID()) + assert.Equal(rootSpan.SpanContext().TraceID(), receivedSpan.SpanContext().TraceID()) }), "testOperation") testServer := httptest.NewServer(w) defer testServer.Close() - c := http.Client{Transport: otelhttp.NewTransport(nil)} req, err := http.NewRequestWithContext(sctx, http.MethodGet, testServer.URL, nil) require.NoError(t, err) @@ -47,12 +46,11 @@ func TestHttpDistributedTrace(t *testing.T) { rootSpan.End() p := <-payloads - numSpans := strings.Count(p, "\"span_id\"") - assert.Equal(t, 3, numSpans) - assert.Contains(t, p, `"name":"internal"`) - assert.Contains(t, p, `"name":"server.request`) - assert.Contains(t, p, `"name":"client.request"`) - assert.Contains(t, p, `"resource":"testRootSpan"`) - assert.Contains(t, p, `"resource":"testOperation"`) - assert.Contains(t, p, `"resource":"HTTP GET"`) + assert.Len(p, 2) + assert.Equal("server.request", p[0][0]["name"]) + assert.Equal("internal", p[1][0]["name"]) + assert.Equal("client.request", p[1][1]["name"]) + assert.Equal("testOperation", p[0][0]["resource"]) + assert.Equal("testRootSpan", p[1][0]["resource"]) + assert.Equal("HTTP GET", p[1][1]["resource"]) } diff --git a/ddtrace/opentelemetry/span.go b/ddtrace/opentelemetry/span.go index 466a0fa7b3..61c197434a 100644 --- a/ddtrace/opentelemetry/span.go +++ b/ddtrace/opentelemetry/span.go @@ -10,6 +10,7 @@ import ( "errors" "strconv" "strings" + "sync" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" @@ -25,7 +26,8 @@ import ( var _ oteltrace.Span = (*span)(nil) type span struct { - noop.Span // https://pkg.go.dev/go.opentelemetry.io/otel/trace#hdr-API_Implementations + noop.Span // https://pkg.go.dev/go.opentelemetry.io/otel/trace#hdr-API_Implementations + mu sync.RWMutex `msg:"-"` // all fields are protected by this RWMutex DD tracer.Span finished bool attributes map[string]interface{} @@ -38,10 +40,14 @@ type span struct { func (s *span) TracerProvider() oteltrace.TracerProvider { return s.oteltracer.provider } func (s *span) SetName(name string) { + s.mu.Lock() + defer s.mu.Unlock() s.attributes[ext.SpanName] = strings.ToLower(name) } func (s *span) End(options ...oteltrace.SpanEndOption) { + s.mu.Lock() + defer s.mu.Unlock() if s.finished { return } @@ -157,6 +163,8 @@ type statusInfo struct { // value before (OK > Error > Unset), the code will not be changed. // The code and description are set once when the span is finished. func (s *span) SetStatus(code otelcodes.Code, description string) { + s.mu.Lock() + defer s.mu.Unlock() if code >= s.statusInfo.code { s.statusInfo = statusInfo{code, description} } @@ -175,6 +183,8 @@ func (s *span) SetStatus(code otelcodes.Code, description string) { // The list of reserved tags might be extended in the future. // Any other non-reserved tags will be set as provided. func (s *span) SetAttributes(kv ...attribute.KeyValue) { + s.mu.Lock() + defer s.mu.Unlock() for _, kv := range kv { if k, v := toReservedAttributes(string(kv.Key), kv.Value); k != "" { s.attributes[k] = v diff --git a/ddtrace/opentelemetry/span_test.go b/ddtrace/opentelemetry/span_test.go index f60a0a5f27..221bce8be6 100644 --- a/ddtrace/opentelemetry/span_test.go +++ b/ddtrace/opentelemetry/span_test.go @@ -8,6 +8,7 @@ package opentelemetry import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -16,6 +17,7 @@ import ( "testing" "time" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "gopkg.in/DataDog/dd-trace-go.v1/internal/httpmem" @@ -28,21 +30,41 @@ import ( oteltrace "go.opentelemetry.io/otel/trace" ) -func mockTracerProvider(t *testing.T, opts ...tracer.StartOption) (tp *TracerProvider, payloads chan string, cleanup func()) { - payloads = make(chan string) +type traces [][]map[string]interface{} + +func mockTracerProvider(t *testing.T, opts ...tracer.StartOption) (tp *TracerProvider, payloads chan traces, cleanup func()) { + payloads = make(chan traces) s, c := httpmem.ServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v0.4/traces": if h := r.Header.Get("X-Datadog-Trace-Count"); h == "0" { return } - buf, err := io.ReadAll(r.Body) + req := r.Clone(context.Background()) + defer req.Body.Close() + buf, err := io.ReadAll(req.Body) if err != nil || len(buf) == 0 { - t.Fatalf("Test agent: Error receiving traces") + t.Fatalf("Test agent: Error receiving traces: %v", err) + } + var payload bytes.Buffer + _, err = msgp.UnmarshalAsJSON(&payload, buf) + if err != nil { + t.Fatalf("Failed to unmarshal payload bytes as JSON: %v", err) + } + var tr [][]map[string]interface{} + err = json.Unmarshal(payload.Bytes(), &tr) + if err != nil || len(tr) == 0 { + t.Fatalf("Failed to unmarshal payload bytes as trace: %v", err) + } + payloads <- tr + default: + if r.Method == "GET" { + // Write an empty JSON object to the output, to avoid spurious decoding + // errors to be reported in the logs, which may lead someone + // investigating a test failure into the wrong direction. + w.Write([]byte("{}")) + return } - var js bytes.Buffer - msgp.UnmarshalAsJSON(&js, buf) - payloads <- js.String() } w.WriteHeader(200) })) @@ -50,24 +72,27 @@ func mockTracerProvider(t *testing.T, opts ...tracer.StartOption) (tp *TracerPro tp = NewTracerProvider(opts...) otel.SetTracerProvider(tp) return tp, payloads, func() { - s.Close() - tp.Shutdown() + if err := s.Close(); err != nil { + t.Fatalf("Test Agent server Close failure: %v", err) + } + if err := tp.Shutdown(); err != nil { + t.Fatalf("Tracer Provider shutdown failure: %v", err) + } } } -func waitForPayload(ctx context.Context, payloads chan string) (string, error) { +func waitForPayload(payloads chan traces) (traces, error) { select { - case <-ctx.Done(): - return "", fmt.Errorf("Timed out waiting for traces") case p := <-payloads: return p, nil + case <-time.After(10 * time.Second): + return nil, fmt.Errorf("Timed out waiting for traces") } } func TestSpanResourceNameDefault(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + ctx := context.Background() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") @@ -77,39 +102,96 @@ func TestSpanResourceNameDefault(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, `"name":"internal"`) - assert.Contains(p, `"resource":"OperationName"`) + p := traces[0] + assert.Equal("internal", p[0]["name"]) + assert.Equal("OperationName", p[0]["resource"]) } func TestSpanSetName(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() _, sp := tr.Start(ctx, "OldName") sp.SetName("NewName") sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, strings.ToLower("NewName")) + p := traces[0] + assert.Equal(strings.ToLower("NewName"), p[0]["name"]) } -func TestSpanEnd(t *testing.T) { +func TestSpanLink(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + + _, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() + + // Use traceID, spanID, and traceflags that can be unmarshalled from unint64 to float64 without loss of precision + traceID, _ := oteltrace.TraceIDFromHex("00000000000001c8000000000000007b") + spanID, _ := oteltrace.SpanIDFromHex("000000000000000f") + traceState, _ := oteltrace.ParseTraceState("dd_origin=ci") + remoteSpanContext := oteltrace.NewSpanContext( + oteltrace.SpanContextConfig{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: 0x0, + TraceState: traceState, + Remote: true, + }, + ) + + _, payloads, cleanup := mockTracerProvider(t) + tr := otel.Tracer("") + defer cleanup() + + // Create a span with a link to a remote span + _, decoratedSpan := tr.Start(context.Background(), "span_with_link", + oteltrace.WithLinks(oteltrace.Link{ + SpanContext: remoteSpanContext, + Attributes: []attribute.KeyValue{attribute.String("link.name", "alpha_transaction")}, + })) + decoratedSpan.End() + + // Flush span with link and unmarshal the payload to a map + tracer.Flush() + payload, err := waitForPayload(payloads) + if err != nil { + t.Fatalf(err.Error()) + } + assert.NotNil(payload) + // Ensure payload contains one trace and one span + assert.Len(payload[0], 1) + + // Convert the span_links field from type []map[string]interface{} to a struct + var spanLinks []ddtrace.SpanLink + spanLinkBytes, _ := json.Marshal(payload[0][0]["span_links"]) + json.Unmarshal(spanLinkBytes, &spanLinks) + assert.Len(spanLinks, 1) + + // Ensure the span link has the correct values + assert.Equal(spanLinks[0].TraceID, uint64(123)) + assert.Equal(spanLinks[0].TraceIDHigh, uint64(456)) + assert.Equal(spanLinks[0].SpanID, uint64(15)) + assert.Equal(spanLinks[0].Attributes, map[string]string{"link.name": "alpha_transaction"}) + assert.Equal(spanLinks[0].Tracestate, "dd_origin=ci") + assert.Equal(spanLinks[0].Flags, uint32(0x80000000)) +} + +func TestSpanEnd(t *testing.T) { + assert := assert.New(t) _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() @@ -139,22 +221,22 @@ func TestSpanEnd(t *testing.T) { } tracer.Flush() - payload, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } + p := traces[0] - assert.Contains(payload, name) - assert.NotContains(payload, ignoredName) - assert.Contains(payload, msg) - assert.NotContains(payload, ignoredMsg) - assert.Contains(payload, `"error":1`) // this should be an error span - + assert.Equal(name, p[0]["resource"]) + assert.Equal(ext.SpanKindInternal, p[0]["name"]) // default + assert.Equal(1.0, p[0]["error"]) // this should be an error span + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, msg) for k, v := range attributes { - assert.Contains(payload, fmt.Sprintf("\"%s\":\"%s\"", k, v)) + assert.Contains(meta, fmt.Sprintf("%s:%s", k, v)) } for k, v := range ignoredAttributes { - assert.NotContains(payload, fmt.Sprintf("\"%s\":\"%s\"", k, v)) + assert.NotContains(meta, fmt.Sprintf("%s:%s", k, v)) } } @@ -193,26 +275,25 @@ func TestSpanSetStatus(t *testing.T) { for _, test := range testData { t.Run(fmt.Sprintf("Setting Code: %d", test.code), func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - var sp oteltrace.Span testStatus := func() { sp.End() tracer.Flush() - payload, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } + p := traces[0] // An error description is set IFF the span has an error // status code value. Messages related to any other status code // are ignored. + meta := fmt.Sprintf("%v", p[0]["meta"]) if test.code == codes.Error { - assert.Contains(payload, test.msg) + assert.Contains(meta, test.msg) } else { - assert.NotContains(payload, test.msg) + assert.NotContains(meta, test.msg) } - assert.NotContains(payload, test.ignoredCode) + assert.NotContains(meta, test.ignoredCode) } _, sp = tr.Start(context.Background(), "test") sp.SetStatus(test.code, test.msg) @@ -229,8 +310,6 @@ func TestSpanSetStatus(t *testing.T) { func TestSpanContextWithStartOptions(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() @@ -262,27 +341,28 @@ func TestSpanContextWithStartOptions(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - if strings.Count(p, "span_id") != 2 { - t.Fatalf("payload does not contain two spans\n%s", p) - } - assert.Contains(p, `"service":"persisted_srv"`) - assert.Contains(p, `"resource":"persisted_ctx_rsc"`) - assert.Contains(p, `"span.kind":"producer"`) - assert.Contains(p, fmt.Sprint(spanID)) - assert.Contains(p, fmt.Sprint(startTime.UnixNano())) - assert.Contains(p, fmt.Sprint(duration.Nanoseconds())) + p := traces[0] + t.Logf("%v", p[0]) + assert.Len(p, 2) + assert.Equal("persisted_srv", p[0]["service"]) + assert.Equal("persisted_ctx_rsc", p[0]["resource"]) + assert.Equal(1234567890.0, p[0]["span_id"]) + assert.Equal("producer", p[0]["name"]) + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, "producer") + assert.Equal(float64(startTime.UnixNano()), p[0]["start"]) + assert.Equal(float64(duration.Nanoseconds()), p[0]["duration"]) assert.NotContains(p, "discarded") - assert.Equal(1, strings.Count(p, `"span_id":1234567890`)) + assert.NotEqual(1234567890.0, p[1]["span_id"]) } func TestSpanContextWithStartOptionsPriorityOrder(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() @@ -299,20 +379,21 @@ func TestSpanContextWithStartOptionsPriorityOrder(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, "persisted_ctx_rsc") - assert.Contains(p, "persisted_srv") - assert.Contains(p, `"span.kind":"producer"`) + p := traces[0] + assert.Equal("persisted_srv", p[0]["service"]) + assert.Equal("persisted_ctx_rsc", p[0]["resource"]) + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, "producer") assert.NotContains(p, "discarded") } func TestSpanEndOptionsPriorityOrder(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() @@ -331,100 +412,112 @@ func TestSpanEndOptionsPriorityOrder(t *testing.T) { EndOptions(sp, tracer.FinishTime(startTime.Add(time.Second*5))) // EndOptions timestamp should prevail sp.End(oteltrace.WithTimestamp(startTime.Add(time.Second * 3))) + duration := time.Second * 5 // making sure end options don't have effect after the span has returned - EndOptions(sp, tracer.FinishTime(startTime.Add(time.Second*2))) + EndOptions(sp, tracer.FinishTime(startTime.Add(duration))) sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, `"duration":5000000000,`) - assert.NotContains(p, `"duration":2000000000,`) - assert.NotContains(p, `"duration":1000000000,`) - assert.NotContains(p, `"duration":3000000000,`) + p := traces[0] + assert.Equal(float64(duration.Nanoseconds()), p[0]["duration"]) } func TestSpanEndOptions(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() + spanID := uint64(1234567890) startTime := time.Now() + duration := time.Second * 5 _, sp := tr.Start( ContextWithStartOptions(context.Background(), tracer.ResourceName("ctx_rsc"), tracer.ServiceName("ctx_srv"), tracer.StartTime(startTime), - tracer.WithSpanID(1234567890), + tracer.WithSpanID(spanID), ), "op_name") - - EndOptions(sp, tracer.FinishTime(startTime.Add(time.Second*5)), + EndOptions(sp, tracer.FinishTime(startTime.Add(duration)), tracer.WithError(errors.New("persisted_option"))) sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, "ctx_srv") - assert.Contains(p, "ctx_rsc") - assert.Contains(p, "1234567890") - assert.Contains(p, fmt.Sprint(startTime.UnixNano())) - assert.Contains(p, `"duration":5000000000,`) - assert.Contains(p, `persisted_option`) - assert.Contains(p, `"error":1`) + p := traces[0] + assert.Equal("ctx_srv", p[0]["service"]) + assert.Equal("ctx_rsc", p[0]["resource"]) + assert.Equal(1234567890.0, p[0]["span_id"]) + assert.Equal(float64(startTime.UnixNano()), p[0]["start"]) + assert.Equal(float64(duration.Nanoseconds()), p[0]["duration"]) + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, "persisted_option") + assert.Equal(1.0, p[0]["error"]) // this should be an error span } func TestSpanSetAttributes(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() - attributes := [][]string{{"k1", "v1_old"}, - {"k2", "v2"}, - {"k1", "v1_new"}, + toBeIgnored := map[string]string{"k1": "v1_old"} + attributes := map[string]string{ + "k2": "v2", + "k1": "v1_new", // maps to 'name' - {"operation.name", "ops"}, + "operation.name": "ops", // maps to 'service' - {"service.name", "srv"}, + "service.name": "srv", // maps to 'resource' - {"resource.name", "rsr"}, + "resource.name": "rsr", // maps to 'type' - {"span.type", "db"}, + "span.type": "db", } _, sp := tr.Start(context.Background(), "test") - for _, tag := range attributes { - sp.SetAttributes(attribute.String(tag[0], tag[1])) + for k, v := range toBeIgnored { + sp.SetAttributes(attribute.String(k, v)) + } + for k, v := range attributes { + sp.SetAttributes(attribute.String(k, v)) } // maps to '_dd1.sr.eausr' sp.SetAttributes(attribute.Int("analytics.event", 1)) sp.End() tracer.Flush() - payload, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(payload, `"k1":"v1_new"`) - assert.Contains(payload, `"k2":"v2"`) - assert.NotContains(payload, "v1_old") + p := traces[0] + meta := fmt.Sprintf("%v", p[0]["meta"]) + for k, v := range toBeIgnored { + assert.NotContains(meta, fmt.Sprintf("%s:%s", k, v)) + } + assert.Contains(meta, fmt.Sprintf("%s:%s", "k1", "v1_new")) + assert.Contains(meta, fmt.Sprintf("%s:%s", "k2", "v2")) // reserved attributes - assert.Contains(payload, `"name":"ops"`) - assert.Contains(payload, `"service":"srv"`) - assert.Contains(payload, `"resource":"rsr"`) - assert.Contains(payload, `"type":"db"`) - assert.Contains(payload, `"_dd1.sr.eausr":1`) + assert.NotContains(meta, fmt.Sprintf("%s:%s", "name", "ops")) + assert.NotContains(meta, fmt.Sprintf("%s:%s", "service", "srv")) + assert.NotContains(meta, fmt.Sprintf("%s:%s", "resource", "rsr")) + assert.NotContains(meta, fmt.Sprintf("%s:%s", "type", "db")) + assert.Equal("ops", p[0]["name"]) + assert.Equal("srv", p[0]["service"]) + assert.Equal("rsr", p[0]["resource"]) + assert.Equal("db", p[0]["type"]) + metrics := fmt.Sprintf("%v", p[0]["metrics"]) + assert.Contains(metrics, fmt.Sprintf("%s:%s", "_dd1.sr.eausr", "1")) } func TestSpanSetAttributesWithRemapping(t *testing.T) { @@ -443,17 +536,16 @@ func TestSpanSetAttributesWithRemapping(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, "graphql.server.request") + p := traces[0] + assert.Equal("graphql.server.request", p[0]["name"]) } func TestTracerStartOptions(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t, tracer.WithEnv("test_env"), tracer.WithService("test_serv")) tr := otel.Tracer("") @@ -462,12 +554,14 @@ func TestTracerStartOptions(t *testing.T) { _, sp := tr.Start(context.Background(), "test") sp.End() tracer.Flush() - payload, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(payload, "\"service\":\"test_serv\"") - assert.Contains(payload, "\"env\":\"test_env\"") + p := traces[0] + assert.Equal("test_serv", p[0]["service"]) + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, fmt.Sprintf("%s:%s", "env", "test_env")) } func TestOperationNameRemapping(t *testing.T) { @@ -483,13 +577,15 @@ func TestOperationNameRemapping(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, "graphql.server.request") + p := traces[0] + assert.Equal("graphql.server.request", p[0]["name"]) } func TestRemapName(t *testing.T) { + assert := assert.New(t) testCases := []struct { spanKind oteltrace.SpanKind in []attribute.KeyValue @@ -597,10 +693,6 @@ func TestRemapName(t *testing.T) { out: "internal", }, } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, payloads, cleanup := mockTracerProvider(t, tracer.WithEnv("test_env"), tracer.WithService("test_serv")) tr := otel.Tracer("") defer cleanup() @@ -612,18 +704,18 @@ func TestRemapName(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(t, p, test.out) + p := traces[0] + assert.Equal(test.out, p[0]["name"]) }) } } func TestRemapWithMultipleSetAttributes(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + assert := assert.New(t) _, payloads, cleanup := mockTracerProvider(t, tracer.WithEnv("test_env"), tracer.WithService("test_serv")) tr := otel.Tracer("") @@ -641,13 +733,15 @@ func TestRemapWithMultipleSetAttributes(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(t, p, `"name":"overriden.name"`) - assert.Contains(t, p, `"resource":"new.name"`) - assert.Contains(t, p, `"service":"new.service.name"`) - assert.Contains(t, p, `"type":"new.span.type"`) - assert.Contains(t, p, `"_dd1.sr.eausr":1`) + p := traces[0] + assert.Equal("overriden.name", p[0]["name"]) + assert.Equal("new.name", p[0]["resource"]) + assert.Equal("new.service.name", p[0]["service"]) + assert.Equal("new.span.type", p[0]["type"]) + metrics := fmt.Sprintf("%v", p[0]["metrics"]) + assert.Contains(metrics, fmt.Sprintf("%s:%s", "_dd1.sr.eausr", "1")) } diff --git a/ddtrace/opentelemetry/tracer.go b/ddtrace/opentelemetry/tracer.go index 296c1a47a4..bad2194409 100644 --- a/ddtrace/opentelemetry/tracer.go +++ b/ddtrace/opentelemetry/tracer.go @@ -62,6 +62,36 @@ func (t *oteltracer) Start(ctx context.Context, spanName string, opts ...oteltra o(&cfg) } } + // Add span links to underlying Datadog span + if len(ssConfig.Links()) > 0 { + links := make([]ddtrace.SpanLink, 0, len(ssConfig.Links())) + for _, otelLink := range ssConfig.Links() { + var link ddtrace.SpanLink + + traceIDbytes := otelLink.SpanContext.TraceID() + link.TraceIDHigh = binary.BigEndian.Uint64(traceIDbytes[:8]) + link.TraceID = binary.BigEndian.Uint64(traceIDbytes[8:]) + + spanIDbytes := otelLink.SpanContext.SpanID() + link.SpanID = binary.BigEndian.Uint64(spanIDbytes[:]) + + link.Tracestate = otelLink.SpanContext.TraceState().String() + + if otelLink.SpanContext.IsSampled() { + link.Flags = uint32(0x80000001) + } else { + link.Flags = uint32(0x80000000) + } + + link.Attributes = make(map[string]string) + for _, attr := range otelLink.Attributes { + link.Attributes[string(attr.Key)] = attr.Value.Emit() + } + + links = append(links, link) + } + ddopts = append(ddopts, tracer.WithSpanLinks(links)) + } // Since there is no way to see if and how the span operation name was set, // we have to record the attributes locally. // The span operation name will be calculated when it's ended. diff --git a/ddtrace/opentelemetry/tracer_test.go b/ddtrace/opentelemetry/tracer_test.go index c9a6a3f91a..ac3d685dbe 100644 --- a/ddtrace/opentelemetry/tracer_test.go +++ b/ddtrace/opentelemetry/tracer_test.go @@ -139,8 +139,6 @@ func TestForceFlush(t *testing.T) { } for _, tc := range testData { t.Run(fmt.Sprintf("Flush success: %t", tc.flushed), func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() tp, payloads, cleanup := mockTracerProvider(t) defer cleanup() @@ -156,10 +154,10 @@ func TestForceFlush(t *testing.T) { _, sp := tr.Start(context.Background(), "test_span") sp.End() tp.forceFlush(tc.timeOut, setFlushStatus, tc.flushFunc) - payload, err := waitForPayload(ctx, payloads) + p, err := waitForPayload(payloads) if tc.flushed { assert.NoError(err) - assert.Contains(payload, "test_span") + assert.Equal("test_span", p[0][0]["resource"]) assert.Equal(OK, flushStatus) } else { assert.Equal(ERROR, flushStatus) @@ -207,6 +205,25 @@ func TestSpanTelemetry(t *testing.T) { telemetryClient.AssertNumberOfCalls(t, "Count", 1) } +func TestConcurrentSetAttributes(_ *testing.T) { + tp := NewTracerProvider() + otel.SetTracerProvider(tp) + tr := otel.Tracer("") + + _, span := tr.Start(context.Background(), "test") + defer span.End() + + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + i := i + go func(val int) { + defer wg.Done() + span.SetAttributes(attribute.Float64("workerID", float64(i))) + }(i) + } +} + func BenchmarkOTelApiWithNoTags(b *testing.B) { testData := struct { env, srv, op string diff --git a/ddtrace/span_link_msgp.go b/ddtrace/span_link_msgp.go new file mode 100644 index 0000000000..c2b90ec516 --- /dev/null +++ b/ddtrace/span_link_msgp.go @@ -0,0 +1,221 @@ +package ddtrace + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *SpanLink) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "trace_id": + z.TraceID, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TraceID") + return + } + case "trace_id_high": + z.TraceIDHigh, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TraceIDHigh") + return + } + case "span_id": + z.SpanID, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "SpanID") + return + } + case "attributes": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Attributes") + return + } + if z.Attributes == nil { + z.Attributes = make(map[string]string, zb0002) + } else if len(z.Attributes) > 0 { + for key := range z.Attributes { + delete(z.Attributes, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0001 string + var za0002 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Attributes") + return + } + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Attributes", za0001) + return + } + z.Attributes[za0001] = za0002 + } + case "tracestate": + z.Tracestate, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Tracestate") + return + } + case "flags": + z.Flags, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *SpanLink) EncodeMsg(en *msgp.Writer) (err error) { + // omitempty: check for empty values + zb0001Len := uint32(6) + var zb0001Mask uint8 /* 6 bits */ + if z.TraceIDHigh == 0 { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.Attributes == nil { + zb0001Len-- + zb0001Mask |= 0x8 + } + if z.Tracestate == "" { + zb0001Len-- + zb0001Mask |= 0x10 + } + if z.Flags == 0 { + zb0001Len-- + zb0001Mask |= 0x20 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } + // write "trace_id" + err = en.Append(0xa8, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.TraceID) + if err != nil { + err = msgp.WrapError(err, "TraceID") + return + } + if (zb0001Mask & 0x2) == 0 { // if not empty + // write "trace_id_high" + err = en.Append(0xad, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x5f, 0x68, 0x69, 0x67, 0x68) + if err != nil { + return + } + err = en.WriteUint64(z.TraceIDHigh) + if err != nil { + err = msgp.WrapError(err, "TraceIDHigh") + return + } + } + // write "span_id" + err = en.Append(0xa7, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.SpanID) + if err != nil { + err = msgp.WrapError(err, "SpanID") + return + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // write "attributes" + err = en.Append(0xaa, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Attributes))) + if err != nil { + err = msgp.WrapError(err, "Attributes") + return + } + for za0001, za0002 := range z.Attributes { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Attributes") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Attributes", za0001) + return + } + } + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // write "tracestate" + err = en.Append(0xaa, 0x74, 0x72, 0x61, 0x63, 0x65, 0x73, 0x74, 0x61, 0x74, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Tracestate) + if err != nil { + err = msgp.WrapError(err, "Tracestate") + return + } + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // write "flags" + err = en.Append(0xa5, 0x66, 0x6c, 0x61, 0x67, 0x73) + if err != nil { + return + } + err = en.WriteUint32(z.Flags) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + } + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *SpanLink) Msgsize() (s int) { + s = 1 + 9 + msgp.Uint64Size + 14 + msgp.Uint64Size + 8 + msgp.Uint64Size + 11 + msgp.MapHeaderSize + if z.Attributes != nil { + for za0001, za0002 := range z.Attributes { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += 11 + msgp.StringPrefixSize + len(z.Tracestate) + 6 + msgp.Uint32Size + return +} diff --git a/ddtrace/tracer/context_test.go b/ddtrace/tracer/context_test.go index bfed735632..34af9bf704 100644 --- a/ddtrace/tracer/context_test.go +++ b/ddtrace/tracer/context_test.go @@ -9,7 +9,6 @@ import ( "context" "encoding/binary" "encoding/hex" - "os" "testing" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" @@ -116,35 +115,38 @@ func Test128(t *testing.T) { _, _, _, stop := startTestTracer(t) defer stop() - os.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") - span, _ := StartSpanFromContext(context.Background(), "http.request") - assert.NotZero(t, span.Context().TraceID()) - w3cCtx, ok := span.Context().(ddtrace.SpanContextW3C) - if !ok { - assert.Fail(t, "couldn't cast to ddtrace.SpanContextW3C") - } - id128 := w3cCtx.TraceID128() - assert.Len(t, id128, 32) // ensure there are enough leading zeros - idBytes, err := hex.DecodeString(id128) - assert.NoError(t, err) - assert.Equal(t, uint64(0), binary.BigEndian.Uint64(idBytes[:8])) // high 64 bits should be 0 - assert.Equal(t, span.Context().TraceID(), binary.BigEndian.Uint64(idBytes[8:])) - - // Enable 128 bit trace ids - os.Unsetenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") - span128, _ := StartSpanFromContext(context.Background(), "http.request") - assert.NotZero(t, span128.Context().TraceID()) - w3cCtx, ok = span128.Context().(ddtrace.SpanContextW3C) - if !ok { - assert.Fail(t, "couldn't cast to ddtrace.SpanContextW3C") - } - id128bit := w3cCtx.TraceID128() - assert.NotEmpty(t, id128bit) - assert.Len(t, id128bit, 32) - // Ensure that the lower order bits match the span's 64-bit trace id - b, err := hex.DecodeString(id128bit) - assert.NoError(t, err) - assert.Equal(t, span128.Context().TraceID(), binary.BigEndian.Uint64(b[8:])) + t.Run("disable 128 bit trace ids", func(t *testing.T) { + t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") + span, _ := StartSpanFromContext(context.Background(), "http.request") + assert.NotZero(t, span.Context().TraceID()) + w3cCtx, ok := span.Context().(ddtrace.SpanContextW3C) + if !ok { + assert.Fail(t, "couldn't cast to ddtrace.SpanContextW3C") + } + id128 := w3cCtx.TraceID128() + assert.Len(t, id128, 32) // ensure there are enough leading zeros + idBytes, err := hex.DecodeString(id128) + assert.NoError(t, err) + assert.Equal(t, uint64(0), binary.BigEndian.Uint64(idBytes[:8])) // high 64 bits should be 0 + assert.Equal(t, span.Context().TraceID(), binary.BigEndian.Uint64(idBytes[8:])) + }) + + t.Run("enable 128 bit trace ids", func(t *testing.T) { + // DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED is true by default + span128, _ := StartSpanFromContext(context.Background(), "http.request") + assert.NotZero(t, span128.Context().TraceID()) + w3cCtx, ok := span128.Context().(ddtrace.SpanContextW3C) + if !ok { + assert.Fail(t, "couldn't cast to ddtrace.SpanContextW3C") + } + id128bit := w3cCtx.TraceID128() + assert.NotEmpty(t, id128bit) + assert.Len(t, id128bit, 32) + // Ensure that the lower order bits match the span's 64-bit trace id + b, err := hex.DecodeString(id128bit) + assert.NoError(t, err) + assert.Equal(t, span128.Context().TraceID(), binary.BigEndian.Uint64(b[8:])) + }) } func TestStartSpanFromNilContext(t *testing.T) { diff --git a/ddtrace/tracer/data_streams.go b/ddtrace/tracer/data_streams.go index 32585cd41a..92f59f1c4f 100644 --- a/ddtrace/tracer/data_streams.go +++ b/ddtrace/tracer/data_streams.go @@ -62,3 +62,13 @@ func TrackKafkaProduceOffset(topic string, partition int32, offset int64) { } } } + +// TrackKafkaHighWatermarkOffset should be used in the producer, to track when it produces a message. +// if used together with TrackKafkaCommitOffset it can generate a Kafka lag in seconds metric. +func TrackKafkaHighWatermarkOffset(cluster string, topic string, partition int32, offset int64) { + if t, ok := internal.GetGlobalTracer().(dataStreamsContainer); ok { + if p := t.GetDataStreamsProcessor(); p != nil { + p.TrackKafkaHighWatermarkOffset(cluster, topic, partition, offset) + } + } +} diff --git a/ddtrace/tracer/log_test.go b/ddtrace/tracer/log_test.go index 6f4d945eb1..ab930f7698 100644 --- a/ddtrace/tracer/log_test.go +++ b/ddtrace/tracer/log_test.go @@ -8,7 +8,6 @@ package tracer import ( "fmt" "math" - "os" "testing" "gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig" @@ -39,8 +38,7 @@ func TestStartupLog(t *testing.T) { assert := assert.New(t) tp := new(log.RecordLogger) - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.123") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.123") tracer, _, _, stop := startTestTracer(t, WithLogger(tp), WithService("configured.service"), @@ -71,10 +69,8 @@ func TestStartupLog(t *testing.T) { t.Run("limit", func(t *testing.T) { assert := assert.New(t) tp := new(log.RecordLogger) - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.123") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") - os.Setenv("DD_TRACE_RATE_LIMIT", "1000.001") - defer os.Unsetenv("DD_TRACE_RATE_LIMIT") + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.123") + t.Setenv("DD_TRACE_RATE_LIMIT", "1000.001") tracer, _, _, stop := startTestTracer(t, WithLogger(tp), WithService("configured.service"), @@ -103,8 +99,7 @@ func TestStartupLog(t *testing.T) { t.Run("errors", func(t *testing.T) { assert := assert.New(t) tp := new(log.RecordLogger) - os.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service","sample_rate": 0.234}, {"service": "other.service"}]`) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") + t.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service","sample_rate": 0.234}, {"service": "other.service"}]`) tracer, _, _, stop := startTestTracer(t, WithLogger(tp)) defer stop() @@ -149,8 +144,7 @@ func TestLogSamplingRules(t *testing.T) { assert := assert.New(t) tp := new(log.RecordLogger) tp.Ignore("appsec: ", telemetry.LogPrefix) - os.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service", "sample_rate": 0.234}, {"service": "other.service"}, {"service": "last.service", "sample_rate": 0.56}, {"odd": "pairs"}, {"sample_rate": 9.10}]`) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") + t.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service", "sample_rate": 0.234}, {"service": "other.service"}, {"service": "last.service", "sample_rate": 0.56}, {"odd": "pairs"}, {"sample_rate": 9.10}]`) _, _, _, stop := startTestTracer(t, WithLogger(tp)) defer stop() diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 635abb9bde..e51c76ad44 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -1141,6 +1141,13 @@ func SpanType(name string) StartSpanOption { return Tag(ext.SpanType, name) } +// WithSpanLinks sets span links on the started span. +func WithSpanLinks(links []ddtrace.SpanLink) StartSpanOption { + return func(cfg *ddtrace.StartSpanConfig) { + cfg.SpanLinks = append(cfg.SpanLinks, links...) + } +} + var measuredTag = Tag(keyMeasured, 1) // Measured marks this span to be measured for metrics and stats calculations. diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index bf5307758d..767b25ef9b 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig" "gopkg.in/DataDog/dd-trace-go.v1/internal/namingschema" @@ -60,8 +61,7 @@ func testStatsd(t *testing.T, cfg *config, addr string) { } func TestStatsdUDPConnect(t *testing.T) { - defer func(old string) { os.Setenv("DD_DOGSTATSD_PORT", old) }(os.Getenv("DD_DOGSTATSD_PORT")) - os.Setenv("DD_DOGSTATSD_PORT", "8111") + t.Setenv("DD_DOGSTATSD_PORT", "8111") testStatsd(t, newConfig(), net.JoinHostPort(defaultHostname, "8111")) cfg := newConfig() addr := net.JoinHostPort(defaultHostname, "8111") @@ -144,8 +144,7 @@ func TestAutoDetectStatsd(t *testing.T) { }) t.Run("env", func(t *testing.T) { - defer func(old string) { os.Setenv("DD_DOGSTATSD_PORT", old) }(os.Getenv("DD_DOGSTATSD_PORT")) - os.Setenv("DD_DOGSTATSD_PORT", "8111") + t.Setenv("DD_DOGSTATSD_PORT", "8111") testStatsd(t, newConfig(), net.JoinHostPort(defaultHostname, "8111")) }) @@ -222,8 +221,7 @@ func TestLoadAgentFeatures(t *testing.T) { }) t.Run("discovery", func(t *testing.T) { - defer func(old string) { os.Setenv("DD_TRACE_FEATURES", old) }(os.Getenv("DD_TRACE_FEATURES")) - os.Setenv("DD_TRACE_FEATURES", "discovery") + t.Setenv("DD_TRACE_FEATURES", "discovery") srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte(`{"endpoints":["/v0.6/stats"],"client_drop_p0s":true,"statsd_port":8999}`)) })) @@ -410,16 +408,14 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env/on", func(t *testing.T) { - os.Setenv("DD_TRACE_ANALYTICS_ENABLED", "true") - defer os.Unsetenv("DD_TRACE_ANALYTICS_ENABLED") + t.Setenv("DD_TRACE_ANALYTICS_ENABLED", "true") defer globalconfig.SetAnalyticsRate(math.NaN()) newConfig() assert.Equal(t, 1.0, globalconfig.AnalyticsRate()) }) t.Run("env/off", func(t *testing.T) { - os.Setenv("DD_TRACE_ANALYTICS_ENABLED", "kj12") - defer os.Unsetenv("DD_TRACE_ANALYTICS_ENABLED") + t.Setenv("DD_TRACE_ANALYTICS_ENABLED", "kj12") defer globalconfig.SetAnalyticsRate(math.NaN()) newConfig() assert.True(t, math.IsNaN(globalconfig.AnalyticsRate())) @@ -435,8 +431,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-host", func(t *testing.T) { - os.Setenv("DD_AGENT_HOST", "my-host") - defer os.Unsetenv("DD_AGENT_HOST") + t.Setenv("DD_AGENT_HOST", "my-host") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -444,8 +439,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-port", func(t *testing.T) { - os.Setenv("DD_DOGSTATSD_PORT", "123") - defer os.Unsetenv("DD_DOGSTATSD_PORT") + t.Setenv("DD_DOGSTATSD_PORT", "123") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -453,10 +447,8 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-both", func(t *testing.T) { - os.Setenv("DD_AGENT_HOST", "my-host") - os.Setenv("DD_DOGSTATSD_PORT", "123") - defer os.Unsetenv("DD_AGENT_HOST") - defer os.Unsetenv("DD_DOGSTATSD_PORT") + t.Setenv("DD_AGENT_HOST", "my-host") + t.Setenv("DD_DOGSTATSD_PORT", "123") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -464,8 +456,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-env", func(t *testing.T) { - os.Setenv("DD_ENV", "testEnv") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_ENV", "testEnv") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -481,8 +472,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-agentAddr", func(t *testing.T) { - os.Setenv("DD_AGENT_HOST", "trace-agent") - defer os.Unsetenv("DD_AGENT_HOST") + t.Setenv("DD_AGENT_HOST", "trace-agent") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -518,8 +508,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("override", func(t *testing.T) { - os.Setenv("DD_ENV", "dev") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_ENV", "dev") assert := assert.New(t) env := "production" tracer := newTracer(WithEnv(env)) @@ -537,8 +526,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("override", func(t *testing.T) { - os.Setenv("DD_TRACE_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_ENABLED") + t.Setenv("DD_TRACE_ENABLED", "false") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -566,8 +554,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-tags", func(t *testing.T) { - os.Setenv("DD_TAGS", "env:test, aKey:aVal,bKey:bVal, cKey:") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "env:test, aKey:aVal,bKey:bVal, cKey:") assert := assert.New(t) c := newConfig() @@ -590,8 +577,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("override", func(t *testing.T) { - os.Setenv(traceprof.EndpointEnvVar, "false") - defer os.Unsetenv(traceprof.EndpointEnvVar) + t.Setenv(traceprof.EndpointEnvVar, "false") c := newConfig() assert.False(t, c.profilerEndpoints) }) @@ -604,16 +590,14 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("override", func(t *testing.T) { - os.Setenv(traceprof.CodeHotspotsEnvVar, "false") - defer os.Unsetenv(traceprof.CodeHotspotsEnvVar) + t.Setenv(traceprof.CodeHotspotsEnvVar, "false") c := newConfig() assert.False(t, c.profilerHotspots) }) }) t.Run("env-mapping", func(t *testing.T) { - os.Setenv("DD_SERVICE_MAPPING", "tracer.test:test2, svc:Newsvc,http.router:myRouter, noval:") - defer os.Unsetenv("DD_SERVICE_MAPPING") + t.Setenv("DD_SERVICE_MAPPING", "tracer.test:test2, svc:Newsvc,http.router:myRouter, noval:") assert := assert.New(t) c := newConfig() @@ -626,8 +610,7 @@ func TestTracerOptionsDefaults(t *testing.T) { t.Run("datadog-tags", func(t *testing.T) { t.Run("can-set-value", func(t *testing.T) { - os.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "200") - defer os.Unsetenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH") + t.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "200") assert := assert.New(t) c := newConfig() p := c.propagator.(*chainedPropagator).injectors[1].(*propagator) @@ -642,8 +625,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("clamped-to-zero", func(t *testing.T) { - os.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "-520") - defer os.Unsetenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH") + t.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "-520") assert := assert.New(t) c := newConfig() p := c.propagator.(*chainedPropagator).injectors[1].(*propagator) @@ -651,8 +633,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("upper-clamp", func(t *testing.T) { - os.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "1000") - defer os.Unsetenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH") + t.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "1000") assert := assert.New(t) c := newConfig() p := c.propagator.(*chainedPropagator).injectors[1].(*propagator) @@ -782,14 +763,12 @@ func TestDefaultDogstatsdAddr(t *testing.T) { }) t.Run("env", func(t *testing.T) { - defer func(old string) { os.Setenv("DD_DOGSTATSD_PORT", old) }(os.Getenv("DD_DOGSTATSD_PORT")) - os.Setenv("DD_DOGSTATSD_PORT", "8111") + t.Setenv("DD_DOGSTATSD_PORT", "8111") assert.Equal(t, defaultDogstatsdAddr(), "localhost:8111") }) t.Run("env+socket", func(t *testing.T) { - defer func(old string) { os.Setenv("DD_DOGSTATSD_PORT", old) }(os.Getenv("DD_DOGSTATSD_PORT")) - os.Setenv("DD_DOGSTATSD_PORT", "8111") + t.Setenv("DD_DOGSTATSD_PORT", "8111") assert.Equal(t, defaultDogstatsdAddr(), "localhost:8111") f, err := ioutil.TempFile("", "dsd.socket") if err != nil { @@ -847,8 +826,7 @@ func TestServiceName(t *testing.T) { t.Run("env", func(t *testing.T) { defer globalconfig.SetServiceName("") - os.Setenv("DD_SERVICE", "api-intake") - defer os.Unsetenv("DD_SERVICE") + t.Setenv("DD_SERVICE", "api-intake") assert := assert.New(t) c := newConfig() @@ -866,8 +844,7 @@ func TestServiceName(t *testing.T) { t.Run("DD_TAGS", func(t *testing.T) { defer globalconfig.SetServiceName("") - os.Setenv("DD_TAGS", "service:api-intake") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "service:api-intake") assert := assert.New(t) c := newConfig() @@ -882,8 +859,7 @@ func TestServiceName(t *testing.T) { assert.Equal(c.serviceName, filepath.Base(os.Args[0])) assert.Equal("", globalconfig.ServiceName()) - os.Setenv("DD_TAGS", "service:testService") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "service:testService") globalconfig.SetServiceName("") c = newConfig() assert.Equal(c.serviceName, "testService") @@ -894,8 +870,7 @@ func TestServiceName(t *testing.T) { assert.Equal(c.serviceName, "testService2") assert.Equal("testService2", globalconfig.ServiceName()) - os.Setenv("DD_SERVICE", "testService3") - defer os.Unsetenv("DD_SERVICE") + t.Setenv("DD_SERVICE", "testService3") globalconfig.SetServiceName("") c = newConfig(WithGlobalTag("service", "testService2")) assert.Equal(c.serviceName, "testService3") @@ -909,6 +884,19 @@ func TestServiceName(t *testing.T) { }) } +func TestStartWithLink(t *testing.T) { + assert := assert.New(t) + + links := []ddtrace.SpanLink{{TraceID: 1, SpanID: 2}, {TraceID: 3, SpanID: 4}} + span := newTracer().StartSpan("test.request", WithSpanLinks(links)).(*span) + + assert.Len(span.SpanLinks, 2) + assert.Equal(span.SpanLinks[0].TraceID, uint64(1)) + assert.Equal(span.SpanLinks[0].SpanID, uint64(2)) + assert.Equal(span.SpanLinks[1].TraceID, uint64(3)) + assert.Equal(span.SpanLinks[1].SpanID, uint64(4)) +} + func TestTagSeparators(t *testing.T) { assert := assert.New(t) @@ -993,8 +981,7 @@ func TestTagSeparators(t *testing.T) { }, } { t.Run("", func(t *testing.T) { - os.Setenv("DD_TAGS", tag.in) - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", tag.in) c := newConfig() globalTags := c.globalTags.get() for key, expected := range tag.out { @@ -1016,8 +1003,7 @@ func TestVersionConfig(t *testing.T) { }) t.Run("env", func(t *testing.T) { - os.Setenv("DD_VERSION", "1.2.3") - defer os.Unsetenv("DD_VERSION") + t.Setenv("DD_VERSION", "1.2.3") assert := assert.New(t) c := newConfig() @@ -1031,8 +1017,7 @@ func TestVersionConfig(t *testing.T) { }) t.Run("DD_TAGS", func(t *testing.T) { - os.Setenv("DD_TAGS", "version:1.2.3") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "version:1.2.3") assert := assert.New(t) c := newConfig() @@ -1044,16 +1029,14 @@ func TestVersionConfig(t *testing.T) { c := newConfig() assert.Equal(c.version, "") - os.Setenv("DD_TAGS", "version:1.1.1") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "version:1.1.1") c = newConfig() assert.Equal("1.1.1", c.version) c = newConfig(WithGlobalTag("version", "1.1.2")) assert.Equal("1.1.2", c.version) - os.Setenv("DD_VERSION", "1.1.3") - defer os.Unsetenv("DD_VERSION") + t.Setenv("DD_VERSION", "1.1.3") c = newConfig(WithGlobalTag("version", "1.1.2")) assert.Equal("1.1.3", c.version) @@ -1072,8 +1055,7 @@ func TestEnvConfig(t *testing.T) { }) t.Run("env", func(t *testing.T) { - os.Setenv("DD_ENV", "testing") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_ENV", "testing") assert := assert.New(t) c := newConfig() @@ -1087,8 +1069,7 @@ func TestEnvConfig(t *testing.T) { }) t.Run("DD_TAGS", func(t *testing.T) { - os.Setenv("DD_TAGS", "env:testing") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "env:testing") assert := assert.New(t) c := newConfig() @@ -1100,16 +1081,14 @@ func TestEnvConfig(t *testing.T) { c := newConfig() assert.Equal(c.env, "") - os.Setenv("DD_TAGS", "env:testing1") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "env:testing1") c = newConfig() assert.Equal("testing1", c.env) c = newConfig(WithGlobalTag("env", "testing2")) assert.Equal("testing2", c.env) - os.Setenv("DD_ENV", "testing3") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_ENV", "testing3") c = newConfig(WithGlobalTag("env", "testing2")) assert.Equal("testing3", c.env) @@ -1145,8 +1124,7 @@ func TestWithHostname(t *testing.T) { t.Run("env", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-env") - defer os.Unsetenv("DD_TRACE_SOURCE_HOSTNAME") + t.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-env") c := newConfig() assert.Equal("hostname-env", c.hostname) }) @@ -1154,8 +1132,7 @@ func TestWithHostname(t *testing.T) { t.Run("env-override", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-env") - defer os.Unsetenv("DD_TRACE_SOURCE_HOSTNAME") + t.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-env") c := newConfig(WithHostname("hostname-middleware")) assert.Equal("hostname-middleware", c.hostname) }) @@ -1170,16 +1147,14 @@ func TestWithTraceEnabled(t *testing.T) { t.Run("env", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_ENABLED") + t.Setenv("DD_TRACE_ENABLED", "false") c := newConfig() assert.False(c.enabled) }) t.Run("env-override", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_ENABLED") + t.Setenv("DD_TRACE_ENABLED", "false") c := newConfig(WithTraceEnabled(true)) assert.True(c.enabled) }) @@ -1238,8 +1213,7 @@ func TestWithHeaderTags(t *testing.T) { t.Run("envvar-only", func(t *testing.T) { defer globalconfig.ClearHeaderTags() - os.Setenv("DD_TRACE_HEADER_TAGS", " 1header:1tag,2.h.e.a.d.e.r ") - defer os.Unsetenv("DD_TRACE_HEADER_TAGS") + t.Setenv("DD_TRACE_HEADER_TAGS", " 1header:1tag,2.h.e.a.d.e.r ") assert := assert.New(t) newConfig() @@ -1251,8 +1225,7 @@ func TestWithHeaderTags(t *testing.T) { t.Run("env-override", func(t *testing.T) { defer globalconfig.ClearHeaderTags() assert := assert.New(t) - os.Setenv("DD_TRACE_HEADER_TAGS", "unexpected") - defer os.Unsetenv("DD_TRACE_HEADER_TAGS") + t.Setenv("DD_TRACE_HEADER_TAGS", "unexpected") newConfig(WithHeaderTags([]string{"expected"})) assert.Equal(ext.HTTPRequestHeaders+".expected", globalconfig.HeaderTag("Expected")) assert.Equal(1, globalconfig.HeaderTagsLen()) diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 0b8a7797f3..8dba524233 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "math" - "os" "regexp" "strings" "sync" @@ -189,7 +188,6 @@ func TestRateSamplerSetting(t *testing.T) { func TestRuleEnvVars(t *testing.T) { t.Run("sample-rate", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") for _, tt := range []struct { in string out float64 @@ -201,7 +199,7 @@ func TestRuleEnvVars(t *testing.T) { {in: "42.0", out: math.NaN()}, // default if out of range {in: "1point0", out: math.NaN()}, // default if invalid value } { - os.Setenv("DD_TRACE_SAMPLE_RATE", tt.in) + t.Setenv("DD_TRACE_SAMPLE_RATE", tt.in) res := globalSampleRate() if math.IsNaN(tt.out) { assert.True(math.IsNaN(res)) @@ -213,7 +211,6 @@ func TestRuleEnvVars(t *testing.T) { t.Run("rate-limit", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_TRACE_RATE_LIMIT") for _, tt := range []struct { in string out *rate.Limiter @@ -226,7 +223,7 @@ func TestRuleEnvVars(t *testing.T) { {in: "-1.0", out: rate.NewLimiter(100.0, 100)}, // default if out of range {in: "1point0", out: rate.NewLimiter(100.0, 100)}, // default if invalid value } { - os.Setenv("DD_TRACE_RATE_LIMIT", tt.in) + t.Setenv("DD_TRACE_RATE_LIMIT", tt.in) res := newRateLimiter() assert.Equal(tt.out, res.limiter) } @@ -234,7 +231,7 @@ func TestRuleEnvVars(t *testing.T) { t.Run("trace-sampling-rules", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") + tests := []struct { value string ruleN int @@ -283,7 +280,7 @@ func TestRuleEnvVars(t *testing.T) { } for i, test := range tests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - os.Setenv("DD_TRACE_SAMPLING_RULES", test.value) + t.Setenv("DD_TRACE_SAMPLING_RULES", test.value) rules, _, err := samplingRulesFromEnv() if test.errStr == "" { assert.NoError(err) @@ -297,7 +294,6 @@ func TestRuleEnvVars(t *testing.T) { t.Run("span-sampling-rules", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for i, tt := range []struct { value string @@ -340,7 +336,7 @@ func TestRuleEnvVars(t *testing.T) { }, } { t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", tt.value) + t.Setenv("DD_SPAN_SAMPLING_RULES", tt.value) _, rules, err := samplingRulesFromEnv() if tt.errStr == "" { assert.NoError(err) @@ -354,7 +350,6 @@ func TestRuleEnvVars(t *testing.T) { t.Run("span-sampling-rules-regex", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for i, tt := range []struct { rules string @@ -417,7 +412,7 @@ func TestRuleEnvVars(t *testing.T) { }, } { t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) + t.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) _, rules, err := samplingRulesFromEnv() assert.NoError(err) if tt.srvRegex == "" { @@ -468,7 +463,6 @@ func TestRulesSampler(t *testing.T) { }) t.Run("matching-trace-rules-env", func(t *testing.T) { - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") for _, tt := range []struct { rules string spanSrv string @@ -521,7 +515,7 @@ func TestRulesSampler(t *testing.T) { }, } { t.Run("", func(t *testing.T) { - os.Setenv("DD_TRACE_SAMPLING_RULES", tt.rules) + t.Setenv("DD_TRACE_SAMPLING_RULES", tt.rules) rules, _, err := samplingRulesFromEnv() assert.Nil(t, err) @@ -596,7 +590,6 @@ func TestRulesSampler(t *testing.T) { }) t.Run("matching-span-rules-from-env", func(t *testing.T) { - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for _, tt := range []struct { rules string spanSrv string @@ -629,7 +622,7 @@ func TestRulesSampler(t *testing.T) { }, } { t.Run("", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) + t.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) _, rules, err := samplingRulesFromEnv() assert.Nil(t, err) assert := assert.New(t) @@ -741,7 +734,6 @@ func TestRulesSampler(t *testing.T) { }) t.Run("not-matching-span-rules-from-env", func(t *testing.T) { - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for _, tt := range []struct { rules string spanSrv string @@ -787,7 +779,7 @@ func TestRulesSampler(t *testing.T) { }, } { t.Run("", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) + t.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) _, rules, _ := samplingRulesFromEnv() assert := assert.New(t) @@ -804,7 +796,6 @@ func TestRulesSampler(t *testing.T) { }) t.Run("not-matching-span-rules", func(t *testing.T) { - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for _, tt := range []struct { spanSrv string spanName string @@ -907,8 +898,7 @@ func TestRulesSampler(t *testing.T) { for _, rate := range sampleRates { t.Run("", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_SAMPLE_RATE", fmt.Sprint(rate)) - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLE_RATE", fmt.Sprint(rate)) rs := newRulesSampler(nil, rules, globalSampleRate()) span := makeSpan("http.request", "test-service") @@ -963,10 +953,8 @@ func TestRulesSampler(t *testing.T) { for _, test := range testEnvs { t.Run("", func(t *testing.T) { - os.Setenv("DD_TRACE_SAMPLING_RULES", test.rules) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", test.generalRate) - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLING_RULES", test.rules) + t.Setenv("DD_TRACE_SAMPLE_RATE", test.generalRate) _, _, _, stop := startTestTracer(t) defer stop() @@ -983,11 +971,9 @@ func TestRulesSampler(t *testing.T) { }) t.Run("locked-sampling-before-propagating-context", func(t *testing.T) { - os.Setenv("DD_TRACE_SAMPLING_RULES", + t.Setenv("DD_TRACE_SAMPLING_RULES", `[{"tags": {"tag2": "val2"}, "sample_rate": 0},{"tags": {"tag1": "val1"}, "sample_rate": 1},{"tags": {"tag0": "val*"}, "sample_rate": 0}]`) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", "0") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLE_RATE", "0") tr, _, _, stop := startTestTracer(t) defer stop() @@ -1134,7 +1120,6 @@ func TestSamplingLimiter(t *testing.T) { assert.Equal(now, sl.prevTime) assert.Equal(100.0, sl.seen) assert.Equal(42.0, sl.allowed) - }) t.Run("discards-rate", func(t *testing.T) { @@ -1352,8 +1337,7 @@ func BenchmarkGlobMatchSpan(b *testing.B) { } b.Run("no-regex", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv.name.ops.date", name:"name.ops.date?", sample_rate": 0.234}]`) - os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv.name.ops.date", "name": "name.ops.date?", "sample_rate": 0.234}]`) _, rules, err := samplingRulesFromEnv() assert.Nil(b, err) rs := newSingleSpanRulesSampler(rules) @@ -1366,8 +1350,7 @@ func BenchmarkGlobMatchSpan(b *testing.B) { }) b.Run("glob-match-?", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv?name?ops?date", name:"name*ops*date*", sample_rate": 0.234}]`) - os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv?name?ops?date", "name": "name*ops*date*", "sample_rate": 0.234}]`) _, rules, err := samplingRulesFromEnv() assert.Nil(b, err) rs := newSingleSpanRulesSampler(rules) @@ -1380,8 +1363,7 @@ func BenchmarkGlobMatchSpan(b *testing.B) { }) b.Run("glob-match-*", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv*name*ops*date", name:"name?ops?date?", sample_rate": 0.234}]`) - os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv*name*ops*date", "name": "name?ops?date?", "sample_rate": 0.234}]`) _, rules, err := samplingRulesFromEnv() assert.Nil(b, err) diff --git a/ddtrace/tracer/span.go b/ddtrace/tracer/span.go index 8ee8888765..b3aa4b2d7c 100644 --- a/ddtrace/tracer/span.go +++ b/ddtrace/tracer/span.go @@ -64,18 +64,19 @@ type errorConfig struct { type span struct { sync.RWMutex `msg:"-"` // all fields are protected by this RWMutex - Name string `msg:"name"` // operation name - Service string `msg:"service"` // service name (i.e. "grpc.server", "http.request") - Resource string `msg:"resource"` // resource name (i.e. "/user?id=123", "SELECT * FROM users") - Type string `msg:"type"` // protocol associated with the span (i.e. "web", "db", "cache") - Start int64 `msg:"start"` // span start time expressed in nanoseconds since epoch - Duration int64 `msg:"duration"` // duration of the span expressed in nanoseconds - Meta map[string]string `msg:"meta,omitempty"` // arbitrary map of metadata - Metrics map[string]float64 `msg:"metrics,omitempty"` // arbitrary map of numeric metrics - SpanID uint64 `msg:"span_id"` // identifier of this span - TraceID uint64 `msg:"trace_id"` // lower 64-bits of the root span identifier - ParentID uint64 `msg:"parent_id"` // identifier of the span's direct parent - Error int32 `msg:"error"` // error status of the span; 0 means no errors + Name string `msg:"name"` // operation name + Service string `msg:"service"` // service name (i.e. "grpc.server", "http.request") + Resource string `msg:"resource"` // resource name (i.e. "/user?id=123", "SELECT * FROM users") + Type string `msg:"type"` // protocol associated with the span (i.e. "web", "db", "cache") + Start int64 `msg:"start"` // span start time expressed in nanoseconds since epoch + Duration int64 `msg:"duration"` // duration of the span expressed in nanoseconds + Meta map[string]string `msg:"meta,omitempty"` // arbitrary map of metadata + Metrics map[string]float64 `msg:"metrics,omitempty"` // arbitrary map of numeric metrics + SpanID uint64 `msg:"span_id"` // identifier of this span + TraceID uint64 `msg:"trace_id"` // lower 64-bits of the root span identifier + ParentID uint64 `msg:"parent_id"` // identifier of the span's direct parent + Error int32 `msg:"error"` // error status of the span; 0 means no errors + SpanLinks []ddtrace.SpanLink `msg:"span_links"` // links to other spans goExecTraced bool `msg:"-"` noDebugStack bool `msg:"-"` // disables debug stack traces diff --git a/ddtrace/tracer/span_msgp.go b/ddtrace/tracer/span_msgp.go index 16bb758f8c..602ba239e9 100644 --- a/ddtrace/tracer/span_msgp.go +++ b/ddtrace/tracer/span_msgp.go @@ -1,18 +1,101 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016 Datadog, Inc. - package tracer -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT +// Code generated by github.com/tinylib/msgp DO NOT EDIT. import ( "github.com/tinylib/msgp/msgp" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" ) +// DecodeMsg implements msgp.Decodable +func (z *errorConfig) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "noDebugStack": + z.noDebugStack, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "noDebugStack") + return + } + case "stackFrames": + z.stackFrames, err = dc.ReadUint() + if err != nil { + err = msgp.WrapError(err, "stackFrames") + return + } + case "stackSkip": + z.stackSkip, err = dc.ReadUint() + if err != nil { + err = msgp.WrapError(err, "stackSkip") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z errorConfig) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "noDebugStack" + err = en.Append(0x83, 0xac, 0x6e, 0x6f, 0x44, 0x65, 0x62, 0x75, 0x67, 0x53, 0x74, 0x61, 0x63, 0x6b) + if err != nil { + return + } + err = en.WriteBool(z.noDebugStack) + if err != nil { + err = msgp.WrapError(err, "noDebugStack") + return + } + // write "stackFrames" + err = en.Append(0xab, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteUint(z.stackFrames) + if err != nil { + err = msgp.WrapError(err, "stackFrames") + return + } + // write "stackSkip" + err = en.Append(0xa9, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x53, 0x6b, 0x69, 0x70) + if err != nil { + return + } + err = en.WriteUint(z.stackSkip) + if err != nil { + err = msgp.WrapError(err, "stackSkip") + return + } + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z errorConfig) Msgsize() (s int) { + s = 1 + 13 + msgp.BoolSize + 12 + msgp.UintSize + 10 + msgp.UintSize + return +} + // DecodeMsg implements msgp.Decodable func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte @@ -20,52 +103,61 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var zb0001 uint32 zb0001, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err) return } for zb0001 > 0 { zb0001-- field, err = dc.ReadMapKeyPtr() if err != nil { + err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "name": z.Name, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Name") return } case "service": z.Service, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Service") return } case "resource": z.Resource, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Resource") return } case "type": z.Type, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Type") return } case "start": z.Start, err = dc.ReadInt64() if err != nil { + err = msgp.WrapError(err, "Start") return } case "duration": z.Duration, err = dc.ReadInt64() if err != nil { + err = msgp.WrapError(err, "Duration") return } case "meta": var zb0002 uint32 zb0002, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err, "Meta") return } - if z.Meta == nil && zb0002 > 0 { + if z.Meta == nil { z.Meta = make(map[string]string, zb0002) } else if len(z.Meta) > 0 { for key := range z.Meta { @@ -78,10 +170,12 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var za0002 string za0001, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Meta") return } za0002, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Meta", za0001) return } z.Meta[za0001] = za0002 @@ -90,9 +184,10 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var zb0003 uint32 zb0003, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err, "Metrics") return } - if z.Metrics == nil && zb0003 > 0 { + if z.Metrics == nil { z.Metrics = make(map[string]float64, zb0003) } else if len(z.Metrics) > 0 { for key := range z.Metrics { @@ -105,10 +200,12 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var za0004 float64 za0003, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Metrics") return } za0004, err = dc.ReadFloat64() if err != nil { + err = msgp.WrapError(err, "Metrics", za0003) return } z.Metrics[za0003] = za0004 @@ -116,26 +213,50 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { case "span_id": z.SpanID, err = dc.ReadUint64() if err != nil { + err = msgp.WrapError(err, "SpanID") return } case "trace_id": z.TraceID, err = dc.ReadUint64() if err != nil { + err = msgp.WrapError(err, "TraceID") return } case "parent_id": z.ParentID, err = dc.ReadUint64() if err != nil { + err = msgp.WrapError(err, "ParentID") return } case "error": z.Error, err = dc.ReadInt32() if err != nil { + err = msgp.WrapError(err, "Error") return } + case "span_links": + var zb0004 uint32 + zb0004, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "SpanLinks") + return + } + if cap(z.SpanLinks) >= int(zb0004) { + z.SpanLinks = (z.SpanLinks)[:zb0004] + } else { + z.SpanLinks = make([]ddtrace.SpanLink, zb0004) + } + for za0005 := range z.SpanLinks { + err = z.SpanLinks[za0005].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "SpanLinks", za0005) + return + } + } default: err = dc.Skip() if err != nil { + err = msgp.WrapError(err) return } } @@ -145,14 +266,33 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *span) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 12 + // omitempty: check for empty values + zb0001Len := uint32(13) + var zb0001Mask uint16 /* 13 bits */ + if z.Meta == nil { + zb0001Len-- + zb0001Mask |= 0x40 + } + if z.Metrics == nil { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } // write "name" - err = en.Append(0x8c, 0xa4, 0x6e, 0x61, 0x6d, 0x65) + err = en.Append(0xa4, 0x6e, 0x61, 0x6d, 0x65) if err != nil { return } err = en.WriteString(z.Name) if err != nil { + err = msgp.WrapError(err, "Name") return } // write "service" @@ -162,6 +302,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteString(z.Service) if err != nil { + err = msgp.WrapError(err, "Service") return } // write "resource" @@ -171,6 +312,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteString(z.Resource) if err != nil { + err = msgp.WrapError(err, "Resource") return } // write "type" @@ -180,6 +322,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteString(z.Type) if err != nil { + err = msgp.WrapError(err, "Type") return } // write "start" @@ -189,6 +332,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteInt64(z.Start) if err != nil { + err = msgp.WrapError(err, "Start") return } // write "duration" @@ -198,45 +342,56 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteInt64(z.Duration) if err != nil { + err = msgp.WrapError(err, "Duration") return } - // write "meta" - err = en.Append(0xa4, 0x6d, 0x65, 0x74, 0x61) - if err != nil { - return - } - err = en.WriteMapHeader(uint32(len(z.Meta))) - if err != nil { - return - } - for za0001, za0002 := range z.Meta { - err = en.WriteString(za0001) + if (zb0001Mask & 0x40) == 0 { // if not empty + // write "meta" + err = en.Append(0xa4, 0x6d, 0x65, 0x74, 0x61) if err != nil { return } - err = en.WriteString(za0002) + err = en.WriteMapHeader(uint32(len(z.Meta))) if err != nil { + err = msgp.WrapError(err, "Meta") return } + for za0001, za0002 := range z.Meta { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Meta", za0001) + return + } + } } - // write "metrics" - err = en.Append(0xa7, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) - if err != nil { - return - } - err = en.WriteMapHeader(uint32(len(z.Metrics))) - if err != nil { - return - } - for za0003, za0004 := range z.Metrics { - err = en.WriteString(za0003) + if (zb0001Mask & 0x80) == 0 { // if not empty + // write "metrics" + err = en.Append(0xa7, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) if err != nil { return } - err = en.WriteFloat64(za0004) + err = en.WriteMapHeader(uint32(len(z.Metrics))) if err != nil { + err = msgp.WrapError(err, "Metrics") return } + for za0003, za0004 := range z.Metrics { + err = en.WriteString(za0003) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + err = en.WriteFloat64(za0004) + if err != nil { + err = msgp.WrapError(err, "Metrics", za0003) + return + } + } } // write "span_id" err = en.Append(0xa7, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64) @@ -245,6 +400,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteUint64(z.SpanID) if err != nil { + err = msgp.WrapError(err, "SpanID") return } // write "trace_id" @@ -254,6 +410,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteUint64(z.TraceID) if err != nil { + err = msgp.WrapError(err, "TraceID") return } // write "parent_id" @@ -263,6 +420,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteUint64(z.ParentID) if err != nil { + err = msgp.WrapError(err, "ParentID") return } // write "error" @@ -272,8 +430,26 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteInt32(z.Error) if err != nil { + err = msgp.WrapError(err, "Error") + return + } + // write "span_links" + err = en.Append(0xaa, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x6c, 0x69, 0x6e, 0x6b, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.SpanLinks))) + if err != nil { + err = msgp.WrapError(err, "SpanLinks") return } + for za0005 := range z.SpanLinks { + err = z.SpanLinks[za0005].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "SpanLinks", za0005) + return + } + } return } @@ -293,7 +469,10 @@ func (z *span) Msgsize() (s int) { s += msgp.StringPrefixSize + len(za0003) + msgp.Float64Size } } - s += 8 + msgp.Uint64Size + 9 + msgp.Uint64Size + 10 + msgp.Uint64Size + 6 + msgp.Int32Size + s += 8 + msgp.Uint64Size + 9 + msgp.Uint64Size + 10 + msgp.Uint64Size + 6 + msgp.Int32Size + 11 + msgp.ArrayHeaderSize + for za0005 := range z.SpanLinks { + s += z.SpanLinks[za0005].Msgsize() + } return } @@ -302,6 +481,7 @@ func (z *spanList) DecodeMsg(dc *msgp.Reader) (err error) { var zb0002 uint32 zb0002, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err) return } if cap((*z)) >= int(zb0002) { @@ -313,6 +493,7 @@ func (z *spanList) DecodeMsg(dc *msgp.Reader) (err error) { if dc.IsNil() { err = dc.ReadNil() if err != nil { + err = msgp.WrapError(err, zb0001) return } (*z)[zb0001] = nil @@ -322,6 +503,7 @@ func (z *spanList) DecodeMsg(dc *msgp.Reader) (err error) { } err = (*z)[zb0001].DecodeMsg(dc) if err != nil { + err = msgp.WrapError(err, zb0001) return } } @@ -333,6 +515,7 @@ func (z *spanList) DecodeMsg(dc *msgp.Reader) (err error) { func (z spanList) EncodeMsg(en *msgp.Writer) (err error) { err = en.WriteArrayHeader(uint32(len(z))) if err != nil { + err = msgp.WrapError(err) return } for zb0003 := range z { @@ -344,6 +527,7 @@ func (z spanList) EncodeMsg(en *msgp.Writer) (err error) { } else { err = z[zb0003].EncodeMsg(en) if err != nil { + err = msgp.WrapError(err, zb0003) return } } @@ -369,6 +553,7 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { var zb0003 uint32 zb0003, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err) return } if cap((*z)) >= int(zb0003) { @@ -380,6 +565,7 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { var zb0004 uint32 zb0004, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err, zb0001) return } if cap((*z)[zb0001]) >= int(zb0004) { @@ -391,6 +577,7 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { if dc.IsNil() { err = dc.ReadNil() if err != nil { + err = msgp.WrapError(err, zb0001, zb0002) return } (*z)[zb0001][zb0002] = nil @@ -400,6 +587,7 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { } err = (*z)[zb0001][zb0002].DecodeMsg(dc) if err != nil { + err = msgp.WrapError(err, zb0001, zb0002) return } } @@ -412,11 +600,13 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { func (z spanLists) EncodeMsg(en *msgp.Writer) (err error) { err = en.WriteArrayHeader(uint32(len(z))) if err != nil { + err = msgp.WrapError(err) return } for zb0005 := range z { err = en.WriteArrayHeader(uint32(len(z[zb0005]))) if err != nil { + err = msgp.WrapError(err, zb0005) return } for zb0006 := range z[zb0005] { @@ -428,6 +618,7 @@ func (z spanLists) EncodeMsg(en *msgp.Writer) (err error) { } else { err = z[zb0005][zb0006].EncodeMsg(en) if err != nil { + err = msgp.WrapError(err, zb0005, zb0006) return } } diff --git a/ddtrace/tracer/span_test.go b/ddtrace/tracer/span_test.go index 946a34d87b..8f94dfb2f4 100644 --- a/ddtrace/tracer/span_test.go +++ b/ddtrace/tracer/span_test.go @@ -8,7 +8,6 @@ package tracer import ( "errors" "fmt" - "os" "runtime" "strings" "sync" @@ -567,7 +566,6 @@ func TestSpanProfilingTags(t *testing.T) { require.Equal(t, false, ok) }) } - } func TestSpanError(t *testing.T) { @@ -775,12 +773,9 @@ func TestSpanLog(t *testing.T) { }) t.Run("env", func(t *testing.T) { - os.Setenv("DD_SERVICE", "tracer.test") - defer os.Unsetenv("DD_SERVICE") - os.Setenv("DD_VERSION", "1.2.3") - defer os.Unsetenv("DD_VERSION") - os.Setenv("DD_ENV", "testenv") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_SERVICE", "tracer.test") + t.Setenv("DD_VERSION", "1.2.3") + t.Setenv("DD_ENV", "testenv") assert := assert.New(t) tracer, _, _, stop := startTestTracer(t) defer stop() @@ -809,12 +804,9 @@ func TestSpanLog(t *testing.T) { }) t.Run("notracer/env", func(t *testing.T) { - os.Setenv("DD_SERVICE", "tracer.test") - defer os.Unsetenv("DD_SERVICE") - os.Setenv("DD_VERSION", "1.2.3") - defer os.Unsetenv("DD_VERSION") - os.Setenv("DD_ENV", "testenv") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_SERVICE", "tracer.test") + t.Setenv("DD_VERSION", "1.2.3") + t.Setenv("DD_ENV", "testenv") assert := assert.New(t) tracer, _, _, stop := startTestTracer(t) span := tracer.StartSpan("test.request").(*span) diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 3a9ee6a59f..a71c099850 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -363,10 +363,11 @@ func (p *propagator) injectTextMap(spanCtx ddtrace.SpanContext, writer TextMapWr if ctx.origin != "" { writer.Set(originHeader, ctx.origin) } - // propagate OpenTracing baggage - for k, v := range ctx.baggage { + ctx.ForeachBaggageItem(func(k, v string) bool { + // Propagate OpenTracing baggage. writer.Set(p.cfg.BaggagePrefix+k, v) - } + return true + }) if p.cfg.MaxTagsHeaderLen <= 0 { return nil } diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index d73f7784f5..63fe93a3e2 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "net/http" - "os" "reflect" "regexp" "strconv" @@ -188,8 +187,7 @@ func TestTextMapExtractTracestatePropagation(t *testing.T) { t.Run(fmt.Sprintf("TestTextMapExtractTracestatePropagation-%s", tc.name), func(t *testing.T) { t.Setenv(headerPropagationStyle, tc.propagationStyle) if tc.onlyExtractFirst { - os.Setenv("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "true") - defer os.Unsetenv("DD_TRACE_PROPAGATION_EXTRACT_FIRST") + t.Setenv("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "true") } tracer := newTracer() assert := assert.New(t) @@ -517,8 +515,7 @@ func TestTextMapPropagator(t *testing.T) { }) t.Run("InjectExtract", func(t *testing.T) { - os.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "true") - defer os.Unsetenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") + t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "true") t.Setenv(headerPropagationStyleExtract, "datadog") t.Setenv(headerPropagationStyleInject, "datadog") propagator := NewPropagator(&PropagatorConfig{ diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 1c86293c40..1193de56bc 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -372,7 +372,7 @@ func (t *tracer) worker(tick <-chan time.Time) { t.statsd.Flush() t.stats.flushAndSend(time.Now(), withCurrentBucket) // TODO(x): In reality, the traceWriter.flush() call is not synchronous - // when using the agent traceWriter. However, this functionnality is used + // when using the agent traceWriter. However, this functionality is used // in Lambda so for that purpose this mechanism should suffice. done <- struct{}{} @@ -503,6 +503,10 @@ func (t *tracer) StartSpan(operationName string, options ...ddtrace.StartSpanOpt Start: startTime, noDebugStack: t.config.noDebugStack, } + for _, link := range opts.SpanLinks { + span.SpanLinks = append(span.SpanLinks, link) + } + if t.config.hostname != "" { span.setMeta(keyHostname, t.config.hostname) } diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 3049d38172..b3cc1b09ba 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -192,8 +192,7 @@ func TestTracerStart(t *testing.T) { }) t.Run("tracing_not_enabled", func(t *testing.T) { - os.Setenv("DD_TRACE_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_ENABLED") + t.Setenv("DD_TRACE_ENABLED", "false") Start() defer Stop() if _, ok := internal.GetGlobalTracer().(*tracer); ok { @@ -427,8 +426,7 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("client_dropped_with_single_spans:stats_enabled", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) // Stats are enabled, rules are available. Trace sample rate equals 0. // Span sample rate equals 1. The trace should be dropped. One single span is extracted. tracer, _, _, stop := startTestTracer(t) @@ -457,8 +455,7 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("client_dropped_with_single_spans:stats_disabled", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) // Stats are disabled, rules are available. Trace sample rate equals 0. // Span sample rate equals 1. The trace should be dropped. One span has single span tags set. tracer, _, _, stop := startTestTracer(t) @@ -485,8 +482,7 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("client_dropped_with_single_span_rules", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "match","name":"nothing", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "match","name":"nothing", "sample_rate": 1.0, "max_per_second": 15.0}]`) // Rules are available, but match nothing. Trace sample rate equals 0. // The trace should be dropped. No single spans extracted. tracer, _, _, stop := startTestTracer(t) @@ -513,8 +509,7 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("client_kept_with_single_spans", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*", "sample_rate": 1.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*", "sample_rate": 1.0}]`) // Rules are available. Trace sample rate equals 1. Span sample rate equals 1. // The trace should be kept. No single spans extracted. tracer, _, _, stop := startTestTracer(t) @@ -538,11 +533,9 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("single_spans_with_max_per_second:rate_1.0", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_*", "sample_rate": 1.0,"max_per_second":50}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") tracer, _, _, stop := startTestTracer(t) // Don't allow the rate limiter to reset while the test is running. current := time.Now() @@ -582,10 +575,8 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("single_spans_without_max_per_second:rate_1.0", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_*", "sample_rate": 1.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_*", "sample_rate": 1.0}]`) + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") tracer, _, _, stop := startTestTracer(t) defer stop() tracer.config.featureFlags = make(map[string]struct{}) @@ -618,10 +609,8 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("single_spans_without_max_per_second:rate_0.5", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_2", "sample_rate": 0.5}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_2", "sample_rate": 0.5}]`) + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") tracer, _, _, stop := startTestTracer(t) defer stop() tracer.config.featureFlags = make(map[string]struct{}) @@ -666,8 +655,7 @@ func TestTracerRuntimeMetrics(t *testing.T) { }) t.Run("env", func(t *testing.T) { - os.Setenv("DD_RUNTIME_METRICS_ENABLED", "true") - defer os.Unsetenv("DD_RUNTIME_METRICS_ENABLED") + t.Setenv("DD_RUNTIME_METRICS_ENABLED", "true") tp := new(log.RecordLogger) tp.Ignore("appsec: ", telemetry.LogPrefix) tracer := newTracer(WithLogger(tp), WithDebugMode(true)) @@ -676,8 +664,7 @@ func TestTracerRuntimeMetrics(t *testing.T) { }) t.Run("overrideEnv", func(t *testing.T) { - os.Setenv("DD_RUNTIME_METRICS_ENABLED", "false") - defer os.Unsetenv("DD_RUNTIME_METRICS_ENABLED") + t.Setenv("DD_RUNTIME_METRICS_ENABLED", "false") tp := new(log.RecordLogger) tp.Ignore("appsec: ", telemetry.LogPrefix) tracer := newTracer(WithRuntimeMetrics(), WithLogger(tp), WithDebugMode(true)) @@ -715,8 +702,7 @@ func TestTracerStartSpanOptions128(t *testing.T) { defer internal.SetGlobalTracer(&internal.NoopTracer{}) t.Run("64-bit-trace-id", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") + t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") opts := []StartSpanOption{ WithSpanID(987654), } @@ -936,6 +922,28 @@ func TestTracerBaggageImmutability(t *testing.T) { assert.Equal("changed!", childContext.baggage["key"]) } +func TestTracerInjectConcurrency(t *testing.T) { + tracer, _, _, stop := startTestTracer(t) + defer stop() + span, _ := StartSpanFromContext(context.Background(), "main") + defer span.Finish() + + var wg sync.WaitGroup + for i := 0; i < 500; i++ { + wg.Add(1) + i := i + go func(val int) { + defer wg.Done() + span.SetBaggageItem("val", fmt.Sprintf("%d", val)) + + traceContext := map[string]string{} + _ = tracer.Inject(span.Context(), TextMapCarrier(traceContext)) + }(i) + } + + wg.Wait() +} + func TestTracerSpanTags(t *testing.T) { tracer := newTracer() defer tracer.Stop() @@ -1044,10 +1052,9 @@ func TestNewSpanChild(t *testing.T) { } func testNewSpanChild(t *testing.T, is128 bool) { - t.Run(fmt.Sprintf("TestNewChildSpan(is128=%t)", is128), func(*testing.T) { + t.Run(fmt.Sprintf("TestNewChildSpan(is128=%t)", is128), func(t *testing.T) { if !is128 { - os.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") + t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") } assert := assert.New(t) @@ -1620,45 +1627,54 @@ func TestTracerFlush(t *testing.T) { func TestTracerReportsHostname(t *testing.T) { const hostname = "hostname-test" - t.Run("DD_TRACE_REPORT_HOSTNAME/set", func(t *testing.T) { - os.Setenv("DD_TRACE_REPORT_HOSTNAME", "true") - defer os.Unsetenv("DD_TRACE_REPORT_HOSTNAME") + testReportHostnameEnabled := func(t *testing.T, name string, withComputeStats bool) { + t.Run(name, func(t *testing.T) { + t.Setenv("DD_TRACE_REPORT_HOSTNAME", "true") + t.Setenv("DD_TRACE_COMPUTE_STATS", fmt.Sprintf("%t", withComputeStats)) - tracer, _, _, stop := startTestTracer(t) - defer stop() + tracer, _, _, stop := startTestTracer(t) + defer stop() - root := tracer.StartSpan("root").(*span) - child := tracer.StartSpan("child", ChildOf(root.Context())).(*span) - child.Finish() - root.Finish() + root := tracer.StartSpan("root").(*span) + child := tracer.StartSpan("child", ChildOf(root.Context())).(*span) + child.Finish() + root.Finish() - assert := assert.New(t) + assert := assert.New(t) - name, ok := root.Meta[keyHostname] - assert.True(ok) - assert.Equal(name, tracer.config.hostname) + name, ok := root.Meta[keyHostname] + assert.True(ok) + assert.Equal(name, tracer.config.hostname) - name, ok = child.Meta[keyHostname] - assert.True(ok) - assert.Equal(name, tracer.config.hostname) - }) - - t.Run("DD_TRACE_REPORT_HOSTNAME/unset", func(t *testing.T) { - tracer, _, _, stop := startTestTracer(t) - defer stop() - - root := tracer.StartSpan("root").(*span) - child := tracer.StartSpan("child", ChildOf(root.Context())).(*span) - child.Finish() - root.Finish() - - assert := assert.New(t) - - _, ok := root.Meta[keyHostname] - assert.False(ok) - _, ok = child.Meta[keyHostname] - assert.False(ok) - }) + name, ok = child.Meta[keyHostname] + assert.True(ok) + assert.Equal(name, tracer.config.hostname) + }) + } + testReportHostnameEnabled(t, "DD_TRACE_REPORT_HOSTNAME/set,DD_TRACE_COMPUTE_STATS/true", true) + testReportHostnameEnabled(t, "DD_TRACE_REPORT_HOSTNAME/set,DD_TRACE_COMPUTE_STATS/false", false) + + testReportHostnameDisabled := func(t *testing.T, name string, withComputeStats bool) { + t.Run(name, func(t *testing.T) { + t.Setenv("DD_TRACE_COMPUTE_STATS", fmt.Sprintf("%t", withComputeStats)) + tracer, _, _, stop := startTestTracer(t) + defer stop() + + root := tracer.StartSpan("root").(*span) + child := tracer.StartSpan("child", ChildOf(root.Context())).(*span) + child.Finish() + root.Finish() + + assert := assert.New(t) + + _, ok := root.Meta[keyHostname] + assert.False(ok) + _, ok = child.Meta[keyHostname] + assert.False(ok) + }) + } + testReportHostnameDisabled(t, "DD_TRACE_REPORT_HOSTNAME/unset,DD_TRACE_COMPUTE_STATS/true", true) + testReportHostnameDisabled(t, "DD_TRACE_REPORT_HOSTNAME/unset,DD_TRACE_COMPUTE_STATS/false", false) t.Run("WithHostname", func(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithHostname(hostname)) @@ -1681,8 +1697,7 @@ func TestTracerReportsHostname(t *testing.T) { }) t.Run("DD_TRACE_SOURCE_HOSTNAME/set", func(t *testing.T) { - os.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-test") - defer os.Unsetenv("DD_TRACE_SOURCE_HOSTNAME") + t.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-test") tracer, _, _, stop := startTestTracer(t) defer stop() @@ -2421,8 +2436,7 @@ func BenchmarkSingleSpanRetention(b *testing.B) { }) b.Run("with-rules/match-half", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) tracer, _, _, stop := startTestTracer(b) defer stop() tracer.config.agent.DropP0s = true @@ -2447,8 +2461,7 @@ func BenchmarkSingleSpanRetention(b *testing.B) { }) b.Run("with-rules/match-all", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) tracer, _, _, stop := startTestTracer(b) defer stop() tracer.config.agent.DropP0s = true diff --git a/ddtrace/tracer/transport_test.go b/ddtrace/tracer/transport_test.go index c359ae18f8..4163010410 100644 --- a/ddtrace/tracer/transport_test.go +++ b/ddtrace/tracer/transport_test.go @@ -96,12 +96,10 @@ func TestResolveAgentAddr(t *testing.T) { } { t.Run("", func(t *testing.T) { if tt.envHost != "" { - os.Setenv("DD_AGENT_HOST", tt.envHost) - defer os.Unsetenv("DD_AGENT_HOST") + t.Setenv("DD_AGENT_HOST", tt.envHost) } if tt.envPort != "" { - os.Setenv("DD_TRACE_AGENT_PORT", tt.envPort) - defer os.Unsetenv("DD_TRACE_AGENT_PORT") + t.Setenv("DD_TRACE_AGENT_PORT", tt.envPort) } c.agentURL = resolveAgentAddr() if tt.inOpt != nil { diff --git a/ddtrace/tracer/writer.go b/ddtrace/tracer/writer.go index a4c56e0c90..877c8ada20 100644 --- a/ddtrace/tracer/writer.go +++ b/ddtrace/tracer/writer.go @@ -106,7 +106,8 @@ func (h *agentTraceWriter) flush() { for attempt := 0; attempt <= h.config.sendRetries; attempt++ { size, count = p.size(), p.itemCount() log.Debug("Sending payload: size: %d traces: %d\n", size, count) - rc, err := h.config.transport.send(p) + var rc io.ReadCloser + rc, err = h.config.transport.send(p) if err == nil { log.Debug("sent traces after %d attempts", attempt+1) h.statsd.Count("datadog.tracer.flush_bytes", int64(size), nil, 1) diff --git a/ddtrace/tracer/writer_test.go b/ddtrace/tracer/writer_test.go index dd6cc61d0a..6c4f9de7aa 100644 --- a/ddtrace/tracer/writer_test.go +++ b/ddtrace/tracer/writer_test.go @@ -373,7 +373,7 @@ func TestTraceWriterFlushRetries(t *testing.T) { sentCounts := map[string]int64{ "datadog.tracer.decode_error": 1, - "datadog.tracer.flush_bytes": 172, + "datadog.tracer.flush_bytes": 184, "datadog.tracer.flush_traces": 1, } droppedCounts := map[string]int64{ diff --git a/go.mod b/go.mod index 54a7804cc1..1123a26521 100644 --- a/go.mod +++ b/go.mod @@ -146,7 +146,7 @@ require ( github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/ebitengine/purego v0.5.0 // indirect + github.com/ebitengine/purego v0.5.2 // indirect github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect diff --git a/go.sum b/go.sum index b0fb37eddc..7a706adda3 100644 --- a/go.sum +++ b/go.sum @@ -1063,8 +1063,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/ebitengine/purego v0.5.0 h1:JrMGKfRIAM4/QVKaesIIT7m/UVjTj5GYhRSQYwfVdpo= -github.com/ebitengine/purego v0.5.0/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= +github.com/ebitengine/purego v0.5.2 h1:r2MQEtkGzZ4LRtFZVAg5bjYKnUbxxloaeuGxH0t7qfs= +github.com/ebitengine/purego v0.5.2/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE= github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/go-elasticsearch/v6 v6.8.5 h1:U2HtkBseC1FNBmDr0TR2tKltL6FxoY+niDAlj5M8TK8= diff --git a/internal/apps/go.mod b/internal/apps/go.mod index 4f1daf5efe..3f1bd82ede 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -1,6 +1,6 @@ module github.com/DataDog/dd-trace-go/internal/apps -go 1.21 +go 1.19 require ( golang.org/x/sync v0.3.0 diff --git a/internal/apps/setup-smoke-test/Dockerfile b/internal/apps/setup-smoke-test/Dockerfile index a2e0774844..303cc60d21 100644 --- a/internal/apps/setup-smoke-test/Dockerfile +++ b/internal/apps/setup-smoke-test/Dockerfile @@ -42,6 +42,8 @@ RUN set -ex; if [ "$build_env" = "alpine" ] && [ "$build_with_cgo" = "1" ]; then apk update && apk add gcc libc-dev; \ fi +RUN go mod tidy + ARG build_with_vendoring RUN set -ex; if [ "$build_with_vendoring" = "y" ]; then \ go mod vendor; \ diff --git a/internal/datastreams/processor.go b/internal/datastreams/processor.go index c330d8e908..4bba1ae564 100644 --- a/internal/datastreams/processor.go +++ b/internal/datastreams/processor.go @@ -55,20 +55,22 @@ type statsGroup struct { } type bucket struct { - points map[uint64]statsGroup - latestCommitOffsets map[partitionConsumerKey]int64 - latestProduceOffsets map[partitionKey]int64 - start uint64 - duration uint64 + points map[uint64]statsGroup + latestCommitOffsets map[partitionConsumerKey]int64 + latestProduceOffsets map[partitionKey]int64 + latestHighWatermarkOffsets map[partitionKey]int64 + start uint64 + duration uint64 } func newBucket(start, duration uint64) bucket { return bucket{ - points: make(map[uint64]statsGroup), - latestCommitOffsets: make(map[partitionConsumerKey]int64), - latestProduceOffsets: make(map[partitionKey]int64), - start: start, - duration: duration, + points: make(map[uint64]statsGroup), + latestCommitOffsets: make(map[partitionConsumerKey]int64), + latestProduceOffsets: make(map[partitionKey]int64), + latestHighWatermarkOffsets: make(map[partitionKey]int64), + start: start, + duration: duration, } } @@ -105,7 +107,7 @@ func (b bucket) export(timestampType TimestampType) StatsBucket { Start: b.start, Duration: b.duration, Stats: stats, - Backlogs: make([]Backlog, 0, len(b.latestCommitOffsets)+len(b.latestProduceOffsets)), + Backlogs: make([]Backlog, 0, len(b.latestCommitOffsets)+len(b.latestProduceOffsets)+len(b.latestHighWatermarkOffsets)), } for key, offset := range b.latestProduceOffsets { exported.Backlogs = append(exported.Backlogs, Backlog{Tags: []string{fmt.Sprintf("partition:%d", key.partition), fmt.Sprintf("topic:%s", key.topic), "type:kafka_produce"}, Value: offset}) @@ -113,6 +115,9 @@ func (b bucket) export(timestampType TimestampType) StatsBucket { for key, offset := range b.latestCommitOffsets { exported.Backlogs = append(exported.Backlogs, Backlog{Tags: []string{fmt.Sprintf("consumer_group:%s", key.group), fmt.Sprintf("partition:%d", key.partition), fmt.Sprintf("topic:%s", key.topic), "type:kafka_commit"}, Value: offset}) } + for key, offset := range b.latestHighWatermarkOffsets { + exported.Backlogs = append(exported.Backlogs, Backlog{Tags: []string{fmt.Sprintf("partition:%d", key.partition), fmt.Sprintf("topic:%s", key.topic), "type:kafka_high_watermark"}, Value: offset}) + } return exported } @@ -154,6 +159,7 @@ type offsetType int const ( produceOffset offsetType = iota commitOffset + highWatermarkOffset ) type kafkaOffset struct { @@ -272,6 +278,13 @@ func (p *Processor) addKafkaOffset(o kafkaOffset) { }] = o.offset return } + if o.offsetType == highWatermarkOffset { + b.latestHighWatermarkOffsets[partitionKey{ + partition: o.partition, + topic: o.topic, + }] = o.offset + return + } b.latestCommitOffsets[partitionConsumerKey{ partition: o.partition, group: o.group, @@ -279,12 +292,32 @@ func (p *Processor) addKafkaOffset(o kafkaOffset) { }] = o.offset } +func (p *Processor) processInput(in *processorInput) { + atomic.AddInt64(&p.stats.payloadsIn, 1) + if in.typ == pointTypeStats { + p.add(in.point) + } else if in.typ == pointTypeKafkaOffset { + p.addKafkaOffset(in.kafkaOffset) + } +} + +func (p *Processor) flushInput() { + for { + in := p.in.pop() + if in == nil { + return + } + p.processInput(in) + } +} + func (p *Processor) run(tick <-chan time.Time) { for { select { case now := <-tick: p.sendToAgent(p.flush(now)) case done := <-p.flushRequest: + p.flushInput() p.sendToAgent(p.flush(time.Now().Add(bucketDuration * 10))) close(done) case <-p.stop: @@ -297,12 +330,7 @@ func (p *Processor) run(tick <-chan time.Time) { time.Sleep(time.Millisecond * 10) continue } - atomic.AddInt64(&p.stats.payloadsIn, 1) - if s.typ == pointTypeStats { - p.add(s.point) - } else if s.typ == pointTypeKafkaOffset { - p.addKafkaOffset(s.kafkaOffset) - } + p.processInput(s) } } } @@ -464,6 +492,21 @@ func (p *Processor) TrackKafkaProduceOffset(topic string, partition int32, offse } } +// TrackKafkaHighWatermarkOffset should be used in the consumer, to track the high watermark offsets of each partition. +// The first argument is the Kafka cluster ID, and will be used later. +func (p *Processor) TrackKafkaHighWatermarkOffset(_ string, topic string, partition int32, offset int64) { + dropped := p.in.push(&processorInput{typ: pointTypeKafkaOffset, kafkaOffset: kafkaOffset{ + offset: offset, + topic: topic, + partition: partition, + offsetType: highWatermarkOffset, + timestamp: p.time().UnixNano(), + }}) + if dropped { + atomic.AddInt64(&p.stats.dropped, 1) + } +} + func (p *Processor) runLoadAgentFeatures(tick <-chan time.Time) { for { select { diff --git a/internal/version/version.go b/internal/version/version.go index 1dbd37863e..e7e1512af3 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -13,7 +13,7 @@ import ( // Tag specifies the current release tag. It needs to be manually // updated. A test checks that the value of Tag never points to a // git tag that is older than HEAD. -const Tag = "v1.60.0-dev" +const Tag = "v1.61.0-dev" // Dissected version number. Filled during init() var ( diff --git a/profiler/internal/pprofutils/protobuf.go b/profiler/internal/pprofutils/protobuf.go index 9aed70915b..e49760dda6 100644 --- a/profiler/internal/pprofutils/protobuf.go +++ b/profiler/internal/pprofutils/protobuf.go @@ -33,8 +33,21 @@ func (p Protobuf) Convert(protobuf *profile.Profile, text io.Writer) error { } w.WriteString(strings.Join(sampleTypes, " ") + "\n") } - if err := protobuf.Aggregate(true, true, false, false, false); err != nil { - return err + // This is a workaround for a breaking change in the pprof library + // when it added columns as an additional attribute for aggregation. + if pb, ok := any(protobuf).(interface { + Aggregate(bool, bool, bool, bool, bool) error + }); ok { + if err := pb.Aggregate(true, true, false, false, false); err != nil { + return err + } + } + if pb, ok := any(protobuf).(interface { + Aggregate(bool, bool, bool, bool, bool, bool) error + }); ok { + if err := pb.Aggregate(true, true, false, false, false, false); err != nil { + return err + } } protobuf = protobuf.Compact() sort.Slice(protobuf.Sample, func(i, j int) bool { diff --git a/profiler/profiler_test.go b/profiler/profiler_test.go index 0a8fae8bdc..1ca48308c2 100644 --- a/profiler/profiler_test.go +++ b/profiler/profiler_test.go @@ -531,6 +531,8 @@ func TestExecutionTraceMisconfiguration(t *testing.T) { } func TestExecutionTraceRandom(t *testing.T) { + t.Skip("flaky test, see: https://github.com/DataDog/dd-trace-go/issues/2529") + collectTraces := func(t *testing.T, profilePeriod, tracePeriod time.Duration, count int) int { t.Setenv("DD_PROFILING_EXECUTION_TRACE_ENABLED", "true") t.Setenv("DD_PROFILING_EXECUTION_TRACE_PERIOD", tracePeriod.String())