From 9b6783beee722463c1c9bcd0362c0bc6597ad759 Mon Sep 17 00:00:00 2001 From: Nick Ripley Date: Thu, 18 Jan 2024 07:32:28 -0500 Subject: [PATCH 01/13] profiler/internal/pprofutils: work around breaking pprof change (#2515) The Google pprof library introduced a breaking change, adding an extra argument to a function we use. We currently have "smoke tests" which assert that we can unconditionally upgrade every dependency to the latest version and still compile. This commit is meant to satisify that test. We don't actually *need* a newer version of the pprof library. But there is a possibility (however unlikely) that a user wants to handle pprofs separately of our library using that library. Either we upgrade our dependency, breaking them, or they upgrade their dependency, breaking us. To avoid either issue, use an interface assertion to support either version of the API. --- profiler/internal/pprofutils/protobuf.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/profiler/internal/pprofutils/protobuf.go b/profiler/internal/pprofutils/protobuf.go index 9aed70915b..e49760dda6 100644 --- a/profiler/internal/pprofutils/protobuf.go +++ b/profiler/internal/pprofutils/protobuf.go @@ -33,8 +33,21 @@ func (p Protobuf) Convert(protobuf *profile.Profile, text io.Writer) error { } w.WriteString(strings.Join(sampleTypes, " ") + "\n") } - if err := protobuf.Aggregate(true, true, false, false, false); err != nil { - return err + // This is a workaround for a breaking change in the pprof library + // when it added columns as an additional attribute for aggregation. + if pb, ok := any(protobuf).(interface { + Aggregate(bool, bool, bool, bool, bool) error + }); ok { + if err := pb.Aggregate(true, true, false, false, false); err != nil { + return err + } + } + if pb, ok := any(protobuf).(interface { + Aggregate(bool, bool, bool, bool, bool, bool) error + }); ok { + if err := pb.Aggregate(true, true, false, false, false, false); err != nil { + return err + } } protobuf = protobuf.Compact() sort.Slice(protobuf.Sample, func(i, j int) bool { From 882cddbb17c27e723807f4fa0f3bb0f5be6ccd9b Mon Sep 17 00:00:00 2001 From: Katie Hockman Date: Thu, 18 Jan 2024 10:00:05 -0500 Subject: [PATCH 02/13] interna/version: bump version.go to v1.61.0-dev (#2501) --- internal/version/version.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/version/version.go b/internal/version/version.go index 1dbd37863e..e7e1512af3 100644 --- a/internal/version/version.go +++ b/internal/version/version.go @@ -13,7 +13,7 @@ import ( // Tag specifies the current release tag. It needs to be manually // updated. A test checks that the value of Tag never points to a // git tag that is older than HEAD. -const Tag = "v1.60.0-dev" +const Tag = "v1.61.0-dev" // Dissected version number. Filled during init() var ( From b5e522535606f88fcbeed6a58c78414dfa52b5ba Mon Sep 17 00:00:00 2001 From: Eliott Bouhana <47679741+eliottness@users.noreply.github.com> Date: Mon, 22 Jan 2024 16:44:45 +0100 Subject: [PATCH 03/13] go.mod: upgrade purego v0.5.0 => upgrade purego v0.5.2 (#2524) Signed-off-by: Eliott Bouhana --- go.mod | 2 +- go.sum | 4 ++-- internal/apps/go.mod | 2 +- internal/apps/setup-smoke-test/Dockerfile | 2 ++ 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 54a7804cc1..1123a26521 100644 --- a/go.mod +++ b/go.mod @@ -146,7 +146,7 @@ require ( github.com/eapache/go-resiliency v1.4.0 // indirect github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect github.com/eapache/queue v1.1.0 // indirect - github.com/ebitengine/purego v0.5.0 // indirect + github.com/ebitengine/purego v0.5.2 // indirect github.com/elastic/elastic-transport-go/v8 v8.1.0 // indirect github.com/fatih/color v1.15.0 // indirect github.com/felixge/httpsnoop v1.0.3 // indirect diff --git a/go.sum b/go.sum index b0fb37eddc..7a706adda3 100644 --- a/go.sum +++ b/go.sum @@ -1063,8 +1063,8 @@ github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4A github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= -github.com/ebitengine/purego v0.5.0 h1:JrMGKfRIAM4/QVKaesIIT7m/UVjTj5GYhRSQYwfVdpo= -github.com/ebitengine/purego v0.5.0/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= +github.com/ebitengine/purego v0.5.2 h1:r2MQEtkGzZ4LRtFZVAg5bjYKnUbxxloaeuGxH0t7qfs= +github.com/ebitengine/purego v0.5.2/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= github.com/elastic/elastic-transport-go/v8 v8.1.0 h1:NeqEz1ty4RQz+TVbUrpSU7pZ48XkzGWQj02k5koahIE= github.com/elastic/elastic-transport-go/v8 v8.1.0/go.mod h1:87Tcz8IVNe6rVSLdBux1o/PEItLtyabHU3naC7IoqKI= github.com/elastic/go-elasticsearch/v6 v6.8.5 h1:U2HtkBseC1FNBmDr0TR2tKltL6FxoY+niDAlj5M8TK8= diff --git a/internal/apps/go.mod b/internal/apps/go.mod index 4f1daf5efe..3f1bd82ede 100644 --- a/internal/apps/go.mod +++ b/internal/apps/go.mod @@ -1,6 +1,6 @@ module github.com/DataDog/dd-trace-go/internal/apps -go 1.21 +go 1.19 require ( golang.org/x/sync v0.3.0 diff --git a/internal/apps/setup-smoke-test/Dockerfile b/internal/apps/setup-smoke-test/Dockerfile index a2e0774844..303cc60d21 100644 --- a/internal/apps/setup-smoke-test/Dockerfile +++ b/internal/apps/setup-smoke-test/Dockerfile @@ -42,6 +42,8 @@ RUN set -ex; if [ "$build_env" = "alpine" ] && [ "$build_with_cgo" = "1" ]; then apk update && apk add gcc libc-dev; \ fi +RUN go mod tidy + ARG build_with_vendoring RUN set -ex; if [ "$build_with_vendoring" = "y" ]; then \ go mod vendor; \ From f63e028cdc79c2b580ad382b3dcce47182bfc9de Mon Sep 17 00:00:00 2001 From: Katie Hockman Date: Mon, 22 Jan 2024 15:16:46 -0500 Subject: [PATCH 04/13] .github/workflows: fixes apm:ecosystem label for issues and PRs (#2525) --- .../workflows/ecosystems-label-issue copy.yml | 17 +++++++++++++++++ ...ystems-label.yml => ecosystems-label-pr.yml} | 9 +-------- 2 files changed, 18 insertions(+), 8 deletions(-) create mode 100644 .github/workflows/ecosystems-label-issue copy.yml rename .github/workflows/{ecosystems-label.yml => ecosystems-label-pr.yml} (73%) diff --git a/.github/workflows/ecosystems-label-issue copy.yml b/.github/workflows/ecosystems-label-issue copy.yml new file mode 100644 index 0000000000..f63226c003 --- /dev/null +++ b/.github/workflows/ecosystems-label-issue copy.yml @@ -0,0 +1,17 @@ +name: Label APM Ecosystems issues +on: + issues: + types: + - reopened + - opened + - edited +jobs: + label_issues: + if: contains(github.event.issue.title, 'contrib') + runs-on: ubuntu-latest + steps: + # https://github.com/marketplace/actions/actions-ecosystem-add-labels + - name: add label + uses: actions-ecosystem/action-add-labels@v1 + with: + labels: apm:ecosystem diff --git a/.github/workflows/ecosystems-label.yml b/.github/workflows/ecosystems-label-pr.yml similarity index 73% rename from .github/workflows/ecosystems-label.yml rename to .github/workflows/ecosystems-label-pr.yml index 909c1db005..36f35b5422 100644 --- a/.github/workflows/ecosystems-label.yml +++ b/.github/workflows/ecosystems-label-pr.yml @@ -1,12 +1,5 @@ -name: Label APM Ecosystems issues +name: Label APM Ecosystems Pull Requests on: - issues: - paths: - - "contrib/**" - types: - - reopened - - opened - - edited pull_request: paths: - "contrib/**" From c03c56b47042238f18d16f623b4f6918a9a14e9c Mon Sep 17 00:00:00 2001 From: Eng Zer Jun Date: Thu, 25 Jan 2024 23:01:04 +0800 Subject: [PATCH 05/13] test: use `t.Setenv` to set env vars in tests (#2437) --- contrib/internal/httptrace/config_test.go | 19 +--- ddtrace/tracer/context_test.go | 62 +++++------ ddtrace/tracer/log_test.go | 16 +-- ddtrace/tracer/option_test.go | 123 ++++++++-------------- ddtrace/tracer/sampler_test.go | 52 +++------ ddtrace/tracer/span_test.go | 20 ++-- ddtrace/tracer/textmap_test.go | 7 +- ddtrace/tracer/tracer_test.go | 59 ++++------- ddtrace/tracer/transport_test.go | 6 +- 9 files changed, 126 insertions(+), 238 deletions(-) diff --git a/contrib/internal/httptrace/config_test.go b/contrib/internal/httptrace/config_test.go index c052292346..6b6856d01c 100644 --- a/contrib/internal/httptrace/config_test.go +++ b/contrib/internal/httptrace/config_test.go @@ -6,7 +6,6 @@ package httptrace import ( - "os" "testing" "github.com/stretchr/testify/require" @@ -50,9 +49,8 @@ func TestConfig(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - defer cleanEnv()() for k, v := range tc.env { - os.Setenv(k, v) + t.Setenv(k, v) } c := newConfig() require.Equal(t, tc.cfg.queryStringRegexp, c.queryStringRegexp) @@ -60,18 +58,3 @@ func TestConfig(t *testing.T) { }) } } - -func cleanEnv() func() { - env := map[string]string{ - envQueryStringDisabled: os.Getenv(envQueryStringDisabled), - envQueryStringRegexp: os.Getenv(envQueryStringRegexp), - } - for k := range env { - os.Unsetenv(k) - } - return func() { - for k, v := range env { - os.Setenv(k, v) - } - } -} diff --git a/ddtrace/tracer/context_test.go b/ddtrace/tracer/context_test.go index bfed735632..34af9bf704 100644 --- a/ddtrace/tracer/context_test.go +++ b/ddtrace/tracer/context_test.go @@ -9,7 +9,6 @@ import ( "context" "encoding/binary" "encoding/hex" - "os" "testing" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" @@ -116,35 +115,38 @@ func Test128(t *testing.T) { _, _, _, stop := startTestTracer(t) defer stop() - os.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") - span, _ := StartSpanFromContext(context.Background(), "http.request") - assert.NotZero(t, span.Context().TraceID()) - w3cCtx, ok := span.Context().(ddtrace.SpanContextW3C) - if !ok { - assert.Fail(t, "couldn't cast to ddtrace.SpanContextW3C") - } - id128 := w3cCtx.TraceID128() - assert.Len(t, id128, 32) // ensure there are enough leading zeros - idBytes, err := hex.DecodeString(id128) - assert.NoError(t, err) - assert.Equal(t, uint64(0), binary.BigEndian.Uint64(idBytes[:8])) // high 64 bits should be 0 - assert.Equal(t, span.Context().TraceID(), binary.BigEndian.Uint64(idBytes[8:])) - - // Enable 128 bit trace ids - os.Unsetenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") - span128, _ := StartSpanFromContext(context.Background(), "http.request") - assert.NotZero(t, span128.Context().TraceID()) - w3cCtx, ok = span128.Context().(ddtrace.SpanContextW3C) - if !ok { - assert.Fail(t, "couldn't cast to ddtrace.SpanContextW3C") - } - id128bit := w3cCtx.TraceID128() - assert.NotEmpty(t, id128bit) - assert.Len(t, id128bit, 32) - // Ensure that the lower order bits match the span's 64-bit trace id - b, err := hex.DecodeString(id128bit) - assert.NoError(t, err) - assert.Equal(t, span128.Context().TraceID(), binary.BigEndian.Uint64(b[8:])) + t.Run("disable 128 bit trace ids", func(t *testing.T) { + t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") + span, _ := StartSpanFromContext(context.Background(), "http.request") + assert.NotZero(t, span.Context().TraceID()) + w3cCtx, ok := span.Context().(ddtrace.SpanContextW3C) + if !ok { + assert.Fail(t, "couldn't cast to ddtrace.SpanContextW3C") + } + id128 := w3cCtx.TraceID128() + assert.Len(t, id128, 32) // ensure there are enough leading zeros + idBytes, err := hex.DecodeString(id128) + assert.NoError(t, err) + assert.Equal(t, uint64(0), binary.BigEndian.Uint64(idBytes[:8])) // high 64 bits should be 0 + assert.Equal(t, span.Context().TraceID(), binary.BigEndian.Uint64(idBytes[8:])) + }) + + t.Run("enable 128 bit trace ids", func(t *testing.T) { + // DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED is true by default + span128, _ := StartSpanFromContext(context.Background(), "http.request") + assert.NotZero(t, span128.Context().TraceID()) + w3cCtx, ok := span128.Context().(ddtrace.SpanContextW3C) + if !ok { + assert.Fail(t, "couldn't cast to ddtrace.SpanContextW3C") + } + id128bit := w3cCtx.TraceID128() + assert.NotEmpty(t, id128bit) + assert.Len(t, id128bit, 32) + // Ensure that the lower order bits match the span's 64-bit trace id + b, err := hex.DecodeString(id128bit) + assert.NoError(t, err) + assert.Equal(t, span128.Context().TraceID(), binary.BigEndian.Uint64(b[8:])) + }) } func TestStartSpanFromNilContext(t *testing.T) { diff --git a/ddtrace/tracer/log_test.go b/ddtrace/tracer/log_test.go index 6f4d945eb1..ab930f7698 100644 --- a/ddtrace/tracer/log_test.go +++ b/ddtrace/tracer/log_test.go @@ -8,7 +8,6 @@ package tracer import ( "fmt" "math" - "os" "testing" "gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig" @@ -39,8 +38,7 @@ func TestStartupLog(t *testing.T) { assert := assert.New(t) tp := new(log.RecordLogger) - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.123") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.123") tracer, _, _, stop := startTestTracer(t, WithLogger(tp), WithService("configured.service"), @@ -71,10 +69,8 @@ func TestStartupLog(t *testing.T) { t.Run("limit", func(t *testing.T) { assert := assert.New(t) tp := new(log.RecordLogger) - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.123") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") - os.Setenv("DD_TRACE_RATE_LIMIT", "1000.001") - defer os.Unsetenv("DD_TRACE_RATE_LIMIT") + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.123") + t.Setenv("DD_TRACE_RATE_LIMIT", "1000.001") tracer, _, _, stop := startTestTracer(t, WithLogger(tp), WithService("configured.service"), @@ -103,8 +99,7 @@ func TestStartupLog(t *testing.T) { t.Run("errors", func(t *testing.T) { assert := assert.New(t) tp := new(log.RecordLogger) - os.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service","sample_rate": 0.234}, {"service": "other.service"}]`) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") + t.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service","sample_rate": 0.234}, {"service": "other.service"}]`) tracer, _, _, stop := startTestTracer(t, WithLogger(tp)) defer stop() @@ -149,8 +144,7 @@ func TestLogSamplingRules(t *testing.T) { assert := assert.New(t) tp := new(log.RecordLogger) tp.Ignore("appsec: ", telemetry.LogPrefix) - os.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service", "sample_rate": 0.234}, {"service": "other.service"}, {"service": "last.service", "sample_rate": 0.56}, {"odd": "pairs"}, {"sample_rate": 9.10}]`) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") + t.Setenv("DD_TRACE_SAMPLING_RULES", `[{"service": "some.service", "sample_rate": 0.234}, {"service": "other.service"}, {"service": "last.service", "sample_rate": 0.56}, {"odd": "pairs"}, {"sample_rate": 9.10}]`) _, _, _, stop := startTestTracer(t, WithLogger(tp)) defer stop() diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index bf5307758d..77e52858c8 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -60,8 +60,7 @@ func testStatsd(t *testing.T, cfg *config, addr string) { } func TestStatsdUDPConnect(t *testing.T) { - defer func(old string) { os.Setenv("DD_DOGSTATSD_PORT", old) }(os.Getenv("DD_DOGSTATSD_PORT")) - os.Setenv("DD_DOGSTATSD_PORT", "8111") + t.Setenv("DD_DOGSTATSD_PORT", "8111") testStatsd(t, newConfig(), net.JoinHostPort(defaultHostname, "8111")) cfg := newConfig() addr := net.JoinHostPort(defaultHostname, "8111") @@ -144,8 +143,7 @@ func TestAutoDetectStatsd(t *testing.T) { }) t.Run("env", func(t *testing.T) { - defer func(old string) { os.Setenv("DD_DOGSTATSD_PORT", old) }(os.Getenv("DD_DOGSTATSD_PORT")) - os.Setenv("DD_DOGSTATSD_PORT", "8111") + t.Setenv("DD_DOGSTATSD_PORT", "8111") testStatsd(t, newConfig(), net.JoinHostPort(defaultHostname, "8111")) }) @@ -222,8 +220,7 @@ func TestLoadAgentFeatures(t *testing.T) { }) t.Run("discovery", func(t *testing.T) { - defer func(old string) { os.Setenv("DD_TRACE_FEATURES", old) }(os.Getenv("DD_TRACE_FEATURES")) - os.Setenv("DD_TRACE_FEATURES", "discovery") + t.Setenv("DD_TRACE_FEATURES", "discovery") srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { w.Write([]byte(`{"endpoints":["/v0.6/stats"],"client_drop_p0s":true,"statsd_port":8999}`)) })) @@ -410,16 +407,14 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env/on", func(t *testing.T) { - os.Setenv("DD_TRACE_ANALYTICS_ENABLED", "true") - defer os.Unsetenv("DD_TRACE_ANALYTICS_ENABLED") + t.Setenv("DD_TRACE_ANALYTICS_ENABLED", "true") defer globalconfig.SetAnalyticsRate(math.NaN()) newConfig() assert.Equal(t, 1.0, globalconfig.AnalyticsRate()) }) t.Run("env/off", func(t *testing.T) { - os.Setenv("DD_TRACE_ANALYTICS_ENABLED", "kj12") - defer os.Unsetenv("DD_TRACE_ANALYTICS_ENABLED") + t.Setenv("DD_TRACE_ANALYTICS_ENABLED", "kj12") defer globalconfig.SetAnalyticsRate(math.NaN()) newConfig() assert.True(t, math.IsNaN(globalconfig.AnalyticsRate())) @@ -435,8 +430,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-host", func(t *testing.T) { - os.Setenv("DD_AGENT_HOST", "my-host") - defer os.Unsetenv("DD_AGENT_HOST") + t.Setenv("DD_AGENT_HOST", "my-host") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -444,8 +438,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-port", func(t *testing.T) { - os.Setenv("DD_DOGSTATSD_PORT", "123") - defer os.Unsetenv("DD_DOGSTATSD_PORT") + t.Setenv("DD_DOGSTATSD_PORT", "123") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -453,10 +446,8 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-both", func(t *testing.T) { - os.Setenv("DD_AGENT_HOST", "my-host") - os.Setenv("DD_DOGSTATSD_PORT", "123") - defer os.Unsetenv("DD_AGENT_HOST") - defer os.Unsetenv("DD_DOGSTATSD_PORT") + t.Setenv("DD_AGENT_HOST", "my-host") + t.Setenv("DD_DOGSTATSD_PORT", "123") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -464,8 +455,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-env", func(t *testing.T) { - os.Setenv("DD_ENV", "testEnv") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_ENV", "testEnv") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -481,8 +471,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-agentAddr", func(t *testing.T) { - os.Setenv("DD_AGENT_HOST", "trace-agent") - defer os.Unsetenv("DD_AGENT_HOST") + t.Setenv("DD_AGENT_HOST", "trace-agent") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -518,8 +507,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("override", func(t *testing.T) { - os.Setenv("DD_ENV", "dev") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_ENV", "dev") assert := assert.New(t) env := "production" tracer := newTracer(WithEnv(env)) @@ -537,8 +525,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("override", func(t *testing.T) { - os.Setenv("DD_TRACE_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_ENABLED") + t.Setenv("DD_TRACE_ENABLED", "false") tracer := newTracer() defer tracer.Stop() c := tracer.config @@ -566,8 +553,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("env-tags", func(t *testing.T) { - os.Setenv("DD_TAGS", "env:test, aKey:aVal,bKey:bVal, cKey:") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "env:test, aKey:aVal,bKey:bVal, cKey:") assert := assert.New(t) c := newConfig() @@ -590,8 +576,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("override", func(t *testing.T) { - os.Setenv(traceprof.EndpointEnvVar, "false") - defer os.Unsetenv(traceprof.EndpointEnvVar) + t.Setenv(traceprof.EndpointEnvVar, "false") c := newConfig() assert.False(t, c.profilerEndpoints) }) @@ -604,16 +589,14 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("override", func(t *testing.T) { - os.Setenv(traceprof.CodeHotspotsEnvVar, "false") - defer os.Unsetenv(traceprof.CodeHotspotsEnvVar) + t.Setenv(traceprof.CodeHotspotsEnvVar, "false") c := newConfig() assert.False(t, c.profilerHotspots) }) }) t.Run("env-mapping", func(t *testing.T) { - os.Setenv("DD_SERVICE_MAPPING", "tracer.test:test2, svc:Newsvc,http.router:myRouter, noval:") - defer os.Unsetenv("DD_SERVICE_MAPPING") + t.Setenv("DD_SERVICE_MAPPING", "tracer.test:test2, svc:Newsvc,http.router:myRouter, noval:") assert := assert.New(t) c := newConfig() @@ -626,8 +609,7 @@ func TestTracerOptionsDefaults(t *testing.T) { t.Run("datadog-tags", func(t *testing.T) { t.Run("can-set-value", func(t *testing.T) { - os.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "200") - defer os.Unsetenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH") + t.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "200") assert := assert.New(t) c := newConfig() p := c.propagator.(*chainedPropagator).injectors[1].(*propagator) @@ -642,8 +624,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("clamped-to-zero", func(t *testing.T) { - os.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "-520") - defer os.Unsetenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH") + t.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "-520") assert := assert.New(t) c := newConfig() p := c.propagator.(*chainedPropagator).injectors[1].(*propagator) @@ -651,8 +632,7 @@ func TestTracerOptionsDefaults(t *testing.T) { }) t.Run("upper-clamp", func(t *testing.T) { - os.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "1000") - defer os.Unsetenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH") + t.Setenv("DD_TRACE_X_DATADOG_TAGS_MAX_LENGTH", "1000") assert := assert.New(t) c := newConfig() p := c.propagator.(*chainedPropagator).injectors[1].(*propagator) @@ -782,14 +762,12 @@ func TestDefaultDogstatsdAddr(t *testing.T) { }) t.Run("env", func(t *testing.T) { - defer func(old string) { os.Setenv("DD_DOGSTATSD_PORT", old) }(os.Getenv("DD_DOGSTATSD_PORT")) - os.Setenv("DD_DOGSTATSD_PORT", "8111") + t.Setenv("DD_DOGSTATSD_PORT", "8111") assert.Equal(t, defaultDogstatsdAddr(), "localhost:8111") }) t.Run("env+socket", func(t *testing.T) { - defer func(old string) { os.Setenv("DD_DOGSTATSD_PORT", old) }(os.Getenv("DD_DOGSTATSD_PORT")) - os.Setenv("DD_DOGSTATSD_PORT", "8111") + t.Setenv("DD_DOGSTATSD_PORT", "8111") assert.Equal(t, defaultDogstatsdAddr(), "localhost:8111") f, err := ioutil.TempFile("", "dsd.socket") if err != nil { @@ -847,8 +825,7 @@ func TestServiceName(t *testing.T) { t.Run("env", func(t *testing.T) { defer globalconfig.SetServiceName("") - os.Setenv("DD_SERVICE", "api-intake") - defer os.Unsetenv("DD_SERVICE") + t.Setenv("DD_SERVICE", "api-intake") assert := assert.New(t) c := newConfig() @@ -866,8 +843,7 @@ func TestServiceName(t *testing.T) { t.Run("DD_TAGS", func(t *testing.T) { defer globalconfig.SetServiceName("") - os.Setenv("DD_TAGS", "service:api-intake") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "service:api-intake") assert := assert.New(t) c := newConfig() @@ -882,8 +858,7 @@ func TestServiceName(t *testing.T) { assert.Equal(c.serviceName, filepath.Base(os.Args[0])) assert.Equal("", globalconfig.ServiceName()) - os.Setenv("DD_TAGS", "service:testService") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "service:testService") globalconfig.SetServiceName("") c = newConfig() assert.Equal(c.serviceName, "testService") @@ -894,8 +869,7 @@ func TestServiceName(t *testing.T) { assert.Equal(c.serviceName, "testService2") assert.Equal("testService2", globalconfig.ServiceName()) - os.Setenv("DD_SERVICE", "testService3") - defer os.Unsetenv("DD_SERVICE") + t.Setenv("DD_SERVICE", "testService3") globalconfig.SetServiceName("") c = newConfig(WithGlobalTag("service", "testService2")) assert.Equal(c.serviceName, "testService3") @@ -993,8 +967,7 @@ func TestTagSeparators(t *testing.T) { }, } { t.Run("", func(t *testing.T) { - os.Setenv("DD_TAGS", tag.in) - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", tag.in) c := newConfig() globalTags := c.globalTags.get() for key, expected := range tag.out { @@ -1016,8 +989,7 @@ func TestVersionConfig(t *testing.T) { }) t.Run("env", func(t *testing.T) { - os.Setenv("DD_VERSION", "1.2.3") - defer os.Unsetenv("DD_VERSION") + t.Setenv("DD_VERSION", "1.2.3") assert := assert.New(t) c := newConfig() @@ -1031,8 +1003,7 @@ func TestVersionConfig(t *testing.T) { }) t.Run("DD_TAGS", func(t *testing.T) { - os.Setenv("DD_TAGS", "version:1.2.3") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "version:1.2.3") assert := assert.New(t) c := newConfig() @@ -1044,16 +1015,14 @@ func TestVersionConfig(t *testing.T) { c := newConfig() assert.Equal(c.version, "") - os.Setenv("DD_TAGS", "version:1.1.1") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "version:1.1.1") c = newConfig() assert.Equal("1.1.1", c.version) c = newConfig(WithGlobalTag("version", "1.1.2")) assert.Equal("1.1.2", c.version) - os.Setenv("DD_VERSION", "1.1.3") - defer os.Unsetenv("DD_VERSION") + t.Setenv("DD_VERSION", "1.1.3") c = newConfig(WithGlobalTag("version", "1.1.2")) assert.Equal("1.1.3", c.version) @@ -1072,8 +1041,7 @@ func TestEnvConfig(t *testing.T) { }) t.Run("env", func(t *testing.T) { - os.Setenv("DD_ENV", "testing") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_ENV", "testing") assert := assert.New(t) c := newConfig() @@ -1087,8 +1055,7 @@ func TestEnvConfig(t *testing.T) { }) t.Run("DD_TAGS", func(t *testing.T) { - os.Setenv("DD_TAGS", "env:testing") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "env:testing") assert := assert.New(t) c := newConfig() @@ -1100,16 +1067,14 @@ func TestEnvConfig(t *testing.T) { c := newConfig() assert.Equal(c.env, "") - os.Setenv("DD_TAGS", "env:testing1") - defer os.Unsetenv("DD_TAGS") + t.Setenv("DD_TAGS", "env:testing1") c = newConfig() assert.Equal("testing1", c.env) c = newConfig(WithGlobalTag("env", "testing2")) assert.Equal("testing2", c.env) - os.Setenv("DD_ENV", "testing3") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_ENV", "testing3") c = newConfig(WithGlobalTag("env", "testing2")) assert.Equal("testing3", c.env) @@ -1145,8 +1110,7 @@ func TestWithHostname(t *testing.T) { t.Run("env", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-env") - defer os.Unsetenv("DD_TRACE_SOURCE_HOSTNAME") + t.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-env") c := newConfig() assert.Equal("hostname-env", c.hostname) }) @@ -1154,8 +1118,7 @@ func TestWithHostname(t *testing.T) { t.Run("env-override", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-env") - defer os.Unsetenv("DD_TRACE_SOURCE_HOSTNAME") + t.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-env") c := newConfig(WithHostname("hostname-middleware")) assert.Equal("hostname-middleware", c.hostname) }) @@ -1170,16 +1133,14 @@ func TestWithTraceEnabled(t *testing.T) { t.Run("env", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_ENABLED") + t.Setenv("DD_TRACE_ENABLED", "false") c := newConfig() assert.False(c.enabled) }) t.Run("env-override", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_ENABLED") + t.Setenv("DD_TRACE_ENABLED", "false") c := newConfig(WithTraceEnabled(true)) assert.True(c.enabled) }) @@ -1238,8 +1199,7 @@ func TestWithHeaderTags(t *testing.T) { t.Run("envvar-only", func(t *testing.T) { defer globalconfig.ClearHeaderTags() - os.Setenv("DD_TRACE_HEADER_TAGS", " 1header:1tag,2.h.e.a.d.e.r ") - defer os.Unsetenv("DD_TRACE_HEADER_TAGS") + t.Setenv("DD_TRACE_HEADER_TAGS", " 1header:1tag,2.h.e.a.d.e.r ") assert := assert.New(t) newConfig() @@ -1251,8 +1211,7 @@ func TestWithHeaderTags(t *testing.T) { t.Run("env-override", func(t *testing.T) { defer globalconfig.ClearHeaderTags() assert := assert.New(t) - os.Setenv("DD_TRACE_HEADER_TAGS", "unexpected") - defer os.Unsetenv("DD_TRACE_HEADER_TAGS") + t.Setenv("DD_TRACE_HEADER_TAGS", "unexpected") newConfig(WithHeaderTags([]string{"expected"})) assert.Equal(ext.HTTPRequestHeaders+".expected", globalconfig.HeaderTag("Expected")) assert.Equal(1, globalconfig.HeaderTagsLen()) diff --git a/ddtrace/tracer/sampler_test.go b/ddtrace/tracer/sampler_test.go index 0b8a7797f3..8dba524233 100644 --- a/ddtrace/tracer/sampler_test.go +++ b/ddtrace/tracer/sampler_test.go @@ -10,7 +10,6 @@ import ( "fmt" "io" "math" - "os" "regexp" "strings" "sync" @@ -189,7 +188,6 @@ func TestRateSamplerSetting(t *testing.T) { func TestRuleEnvVars(t *testing.T) { t.Run("sample-rate", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") for _, tt := range []struct { in string out float64 @@ -201,7 +199,7 @@ func TestRuleEnvVars(t *testing.T) { {in: "42.0", out: math.NaN()}, // default if out of range {in: "1point0", out: math.NaN()}, // default if invalid value } { - os.Setenv("DD_TRACE_SAMPLE_RATE", tt.in) + t.Setenv("DD_TRACE_SAMPLE_RATE", tt.in) res := globalSampleRate() if math.IsNaN(tt.out) { assert.True(math.IsNaN(res)) @@ -213,7 +211,6 @@ func TestRuleEnvVars(t *testing.T) { t.Run("rate-limit", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_TRACE_RATE_LIMIT") for _, tt := range []struct { in string out *rate.Limiter @@ -226,7 +223,7 @@ func TestRuleEnvVars(t *testing.T) { {in: "-1.0", out: rate.NewLimiter(100.0, 100)}, // default if out of range {in: "1point0", out: rate.NewLimiter(100.0, 100)}, // default if invalid value } { - os.Setenv("DD_TRACE_RATE_LIMIT", tt.in) + t.Setenv("DD_TRACE_RATE_LIMIT", tt.in) res := newRateLimiter() assert.Equal(tt.out, res.limiter) } @@ -234,7 +231,7 @@ func TestRuleEnvVars(t *testing.T) { t.Run("trace-sampling-rules", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") + tests := []struct { value string ruleN int @@ -283,7 +280,7 @@ func TestRuleEnvVars(t *testing.T) { } for i, test := range tests { t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { - os.Setenv("DD_TRACE_SAMPLING_RULES", test.value) + t.Setenv("DD_TRACE_SAMPLING_RULES", test.value) rules, _, err := samplingRulesFromEnv() if test.errStr == "" { assert.NoError(err) @@ -297,7 +294,6 @@ func TestRuleEnvVars(t *testing.T) { t.Run("span-sampling-rules", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for i, tt := range []struct { value string @@ -340,7 +336,7 @@ func TestRuleEnvVars(t *testing.T) { }, } { t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", tt.value) + t.Setenv("DD_SPAN_SAMPLING_RULES", tt.value) _, rules, err := samplingRulesFromEnv() if tt.errStr == "" { assert.NoError(err) @@ -354,7 +350,6 @@ func TestRuleEnvVars(t *testing.T) { t.Run("span-sampling-rules-regex", func(t *testing.T) { assert := assert.New(t) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for i, tt := range []struct { rules string @@ -417,7 +412,7 @@ func TestRuleEnvVars(t *testing.T) { }, } { t.Run(fmt.Sprintf("%v", i), func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) + t.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) _, rules, err := samplingRulesFromEnv() assert.NoError(err) if tt.srvRegex == "" { @@ -468,7 +463,6 @@ func TestRulesSampler(t *testing.T) { }) t.Run("matching-trace-rules-env", func(t *testing.T) { - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") for _, tt := range []struct { rules string spanSrv string @@ -521,7 +515,7 @@ func TestRulesSampler(t *testing.T) { }, } { t.Run("", func(t *testing.T) { - os.Setenv("DD_TRACE_SAMPLING_RULES", tt.rules) + t.Setenv("DD_TRACE_SAMPLING_RULES", tt.rules) rules, _, err := samplingRulesFromEnv() assert.Nil(t, err) @@ -596,7 +590,6 @@ func TestRulesSampler(t *testing.T) { }) t.Run("matching-span-rules-from-env", func(t *testing.T) { - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for _, tt := range []struct { rules string spanSrv string @@ -629,7 +622,7 @@ func TestRulesSampler(t *testing.T) { }, } { t.Run("", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) + t.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) _, rules, err := samplingRulesFromEnv() assert.Nil(t, err) assert := assert.New(t) @@ -741,7 +734,6 @@ func TestRulesSampler(t *testing.T) { }) t.Run("not-matching-span-rules-from-env", func(t *testing.T) { - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for _, tt := range []struct { rules string spanSrv string @@ -787,7 +779,7 @@ func TestRulesSampler(t *testing.T) { }, } { t.Run("", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) + t.Setenv("DD_SPAN_SAMPLING_RULES", tt.rules) _, rules, _ := samplingRulesFromEnv() assert := assert.New(t) @@ -804,7 +796,6 @@ func TestRulesSampler(t *testing.T) { }) t.Run("not-matching-span-rules", func(t *testing.T) { - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") for _, tt := range []struct { spanSrv string spanName string @@ -907,8 +898,7 @@ func TestRulesSampler(t *testing.T) { for _, rate := range sampleRates { t.Run("", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_SAMPLE_RATE", fmt.Sprint(rate)) - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLE_RATE", fmt.Sprint(rate)) rs := newRulesSampler(nil, rules, globalSampleRate()) span := makeSpan("http.request", "test-service") @@ -963,10 +953,8 @@ func TestRulesSampler(t *testing.T) { for _, test := range testEnvs { t.Run("", func(t *testing.T) { - os.Setenv("DD_TRACE_SAMPLING_RULES", test.rules) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", test.generalRate) - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLING_RULES", test.rules) + t.Setenv("DD_TRACE_SAMPLE_RATE", test.generalRate) _, _, _, stop := startTestTracer(t) defer stop() @@ -983,11 +971,9 @@ func TestRulesSampler(t *testing.T) { }) t.Run("locked-sampling-before-propagating-context", func(t *testing.T) { - os.Setenv("DD_TRACE_SAMPLING_RULES", + t.Setenv("DD_TRACE_SAMPLING_RULES", `[{"tags": {"tag2": "val2"}, "sample_rate": 0},{"tags": {"tag1": "val1"}, "sample_rate": 1},{"tags": {"tag0": "val*"}, "sample_rate": 0}]`) - defer os.Unsetenv("DD_TRACE_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", "0") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLE_RATE", "0") tr, _, _, stop := startTestTracer(t) defer stop() @@ -1134,7 +1120,6 @@ func TestSamplingLimiter(t *testing.T) { assert.Equal(now, sl.prevTime) assert.Equal(100.0, sl.seen) assert.Equal(42.0, sl.allowed) - }) t.Run("discards-rate", func(t *testing.T) { @@ -1352,8 +1337,7 @@ func BenchmarkGlobMatchSpan(b *testing.B) { } b.Run("no-regex", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv.name.ops.date", name:"name.ops.date?", sample_rate": 0.234}]`) - os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv.name.ops.date", "name": "name.ops.date?", "sample_rate": 0.234}]`) _, rules, err := samplingRulesFromEnv() assert.Nil(b, err) rs := newSingleSpanRulesSampler(rules) @@ -1366,8 +1350,7 @@ func BenchmarkGlobMatchSpan(b *testing.B) { }) b.Run("glob-match-?", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv?name?ops?date", name:"name*ops*date*", sample_rate": 0.234}]`) - os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv?name?ops?date", "name": "name*ops*date*", "sample_rate": 0.234}]`) _, rules, err := samplingRulesFromEnv() assert.Nil(b, err) rs := newSingleSpanRulesSampler(rules) @@ -1380,8 +1363,7 @@ func BenchmarkGlobMatchSpan(b *testing.B) { }) b.Run("glob-match-*", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv*name*ops*date", name:"name?ops?date?", sample_rate": 0.234}]`) - os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "srv*name*ops*date", "name": "name?ops?date?", "sample_rate": 0.234}]`) _, rules, err := samplingRulesFromEnv() assert.Nil(b, err) diff --git a/ddtrace/tracer/span_test.go b/ddtrace/tracer/span_test.go index 946a34d87b..8f94dfb2f4 100644 --- a/ddtrace/tracer/span_test.go +++ b/ddtrace/tracer/span_test.go @@ -8,7 +8,6 @@ package tracer import ( "errors" "fmt" - "os" "runtime" "strings" "sync" @@ -567,7 +566,6 @@ func TestSpanProfilingTags(t *testing.T) { require.Equal(t, false, ok) }) } - } func TestSpanError(t *testing.T) { @@ -775,12 +773,9 @@ func TestSpanLog(t *testing.T) { }) t.Run("env", func(t *testing.T) { - os.Setenv("DD_SERVICE", "tracer.test") - defer os.Unsetenv("DD_SERVICE") - os.Setenv("DD_VERSION", "1.2.3") - defer os.Unsetenv("DD_VERSION") - os.Setenv("DD_ENV", "testenv") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_SERVICE", "tracer.test") + t.Setenv("DD_VERSION", "1.2.3") + t.Setenv("DD_ENV", "testenv") assert := assert.New(t) tracer, _, _, stop := startTestTracer(t) defer stop() @@ -809,12 +804,9 @@ func TestSpanLog(t *testing.T) { }) t.Run("notracer/env", func(t *testing.T) { - os.Setenv("DD_SERVICE", "tracer.test") - defer os.Unsetenv("DD_SERVICE") - os.Setenv("DD_VERSION", "1.2.3") - defer os.Unsetenv("DD_VERSION") - os.Setenv("DD_ENV", "testenv") - defer os.Unsetenv("DD_ENV") + t.Setenv("DD_SERVICE", "tracer.test") + t.Setenv("DD_VERSION", "1.2.3") + t.Setenv("DD_ENV", "testenv") assert := assert.New(t) tracer, _, _, stop := startTestTracer(t) span := tracer.StartSpan("test.request").(*span) diff --git a/ddtrace/tracer/textmap_test.go b/ddtrace/tracer/textmap_test.go index d73f7784f5..63fe93a3e2 100644 --- a/ddtrace/tracer/textmap_test.go +++ b/ddtrace/tracer/textmap_test.go @@ -9,7 +9,6 @@ import ( "errors" "fmt" "net/http" - "os" "reflect" "regexp" "strconv" @@ -188,8 +187,7 @@ func TestTextMapExtractTracestatePropagation(t *testing.T) { t.Run(fmt.Sprintf("TestTextMapExtractTracestatePropagation-%s", tc.name), func(t *testing.T) { t.Setenv(headerPropagationStyle, tc.propagationStyle) if tc.onlyExtractFirst { - os.Setenv("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "true") - defer os.Unsetenv("DD_TRACE_PROPAGATION_EXTRACT_FIRST") + t.Setenv("DD_TRACE_PROPAGATION_EXTRACT_FIRST", "true") } tracer := newTracer() assert := assert.New(t) @@ -517,8 +515,7 @@ func TestTextMapPropagator(t *testing.T) { }) t.Run("InjectExtract", func(t *testing.T) { - os.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "true") - defer os.Unsetenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") + t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "true") t.Setenv(headerPropagationStyleExtract, "datadog") t.Setenv(headerPropagationStyleInject, "datadog") propagator := NewPropagator(&PropagatorConfig{ diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 3049d38172..9914a40dbe 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -192,8 +192,7 @@ func TestTracerStart(t *testing.T) { }) t.Run("tracing_not_enabled", func(t *testing.T) { - os.Setenv("DD_TRACE_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_ENABLED") + t.Setenv("DD_TRACE_ENABLED", "false") Start() defer Stop() if _, ok := internal.GetGlobalTracer().(*tracer); ok { @@ -427,8 +426,7 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("client_dropped_with_single_spans:stats_enabled", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) // Stats are enabled, rules are available. Trace sample rate equals 0. // Span sample rate equals 1. The trace should be dropped. One single span is extracted. tracer, _, _, stop := startTestTracer(t) @@ -457,8 +455,7 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("client_dropped_with_single_spans:stats_disabled", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) // Stats are disabled, rules are available. Trace sample rate equals 0. // Span sample rate equals 1. The trace should be dropped. One span has single span tags set. tracer, _, _, stop := startTestTracer(t) @@ -485,8 +482,7 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("client_dropped_with_single_span_rules", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "match","name":"nothing", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "match","name":"nothing", "sample_rate": 1.0, "max_per_second": 15.0}]`) // Rules are available, but match nothing. Trace sample rate equals 0. // The trace should be dropped. No single spans extracted. tracer, _, _, stop := startTestTracer(t) @@ -513,8 +509,7 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("client_kept_with_single_spans", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*", "sample_rate": 1.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*", "sample_rate": 1.0}]`) // Rules are available. Trace sample rate equals 1. Span sample rate equals 1. // The trace should be kept. No single spans extracted. tracer, _, _, stop := startTestTracer(t) @@ -538,11 +533,9 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("single_spans_with_max_per_second:rate_1.0", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_*", "sample_rate": 1.0,"max_per_second":50}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") tracer, _, _, stop := startTestTracer(t) // Don't allow the rate limiter to reset while the test is running. current := time.Now() @@ -582,10 +575,8 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("single_spans_without_max_per_second:rate_1.0", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_*", "sample_rate": 1.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_*", "sample_rate": 1.0}]`) + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") tracer, _, _, stop := startTestTracer(t) defer stop() tracer.config.featureFlags = make(map[string]struct{}) @@ -618,10 +609,8 @@ func TestSamplingDecision(t *testing.T) { }) t.Run("single_spans_without_max_per_second:rate_0.5", func(t *testing.T) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_2", "sample_rate": 0.5}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") - os.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") - defer os.Unsetenv("DD_TRACE_SAMPLE_RATE") + t.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"name_2", "sample_rate": 0.5}]`) + t.Setenv("DD_TRACE_SAMPLE_RATE", "0.8") tracer, _, _, stop := startTestTracer(t) defer stop() tracer.config.featureFlags = make(map[string]struct{}) @@ -666,8 +655,7 @@ func TestTracerRuntimeMetrics(t *testing.T) { }) t.Run("env", func(t *testing.T) { - os.Setenv("DD_RUNTIME_METRICS_ENABLED", "true") - defer os.Unsetenv("DD_RUNTIME_METRICS_ENABLED") + t.Setenv("DD_RUNTIME_METRICS_ENABLED", "true") tp := new(log.RecordLogger) tp.Ignore("appsec: ", telemetry.LogPrefix) tracer := newTracer(WithLogger(tp), WithDebugMode(true)) @@ -676,8 +664,7 @@ func TestTracerRuntimeMetrics(t *testing.T) { }) t.Run("overrideEnv", func(t *testing.T) { - os.Setenv("DD_RUNTIME_METRICS_ENABLED", "false") - defer os.Unsetenv("DD_RUNTIME_METRICS_ENABLED") + t.Setenv("DD_RUNTIME_METRICS_ENABLED", "false") tp := new(log.RecordLogger) tp.Ignore("appsec: ", telemetry.LogPrefix) tracer := newTracer(WithRuntimeMetrics(), WithLogger(tp), WithDebugMode(true)) @@ -715,8 +702,7 @@ func TestTracerStartSpanOptions128(t *testing.T) { defer internal.SetGlobalTracer(&internal.NoopTracer{}) t.Run("64-bit-trace-id", func(t *testing.T) { assert := assert.New(t) - os.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") + t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") opts := []StartSpanOption{ WithSpanID(987654), } @@ -1044,10 +1030,9 @@ func TestNewSpanChild(t *testing.T) { } func testNewSpanChild(t *testing.T, is128 bool) { - t.Run(fmt.Sprintf("TestNewChildSpan(is128=%t)", is128), func(*testing.T) { + t.Run(fmt.Sprintf("TestNewChildSpan(is128=%t)", is128), func(t *testing.T) { if !is128 { - os.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") - defer os.Unsetenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED") + t.Setenv("DD_TRACE_128_BIT_TRACEID_GENERATION_ENABLED", "false") } assert := assert.New(t) @@ -1621,8 +1606,7 @@ func TestTracerReportsHostname(t *testing.T) { const hostname = "hostname-test" t.Run("DD_TRACE_REPORT_HOSTNAME/set", func(t *testing.T) { - os.Setenv("DD_TRACE_REPORT_HOSTNAME", "true") - defer os.Unsetenv("DD_TRACE_REPORT_HOSTNAME") + t.Setenv("DD_TRACE_REPORT_HOSTNAME", "true") tracer, _, _, stop := startTestTracer(t) defer stop() @@ -1681,8 +1665,7 @@ func TestTracerReportsHostname(t *testing.T) { }) t.Run("DD_TRACE_SOURCE_HOSTNAME/set", func(t *testing.T) { - os.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-test") - defer os.Unsetenv("DD_TRACE_SOURCE_HOSTNAME") + t.Setenv("DD_TRACE_SOURCE_HOSTNAME", "hostname-test") tracer, _, _, stop := startTestTracer(t) defer stop() @@ -2421,8 +2404,7 @@ func BenchmarkSingleSpanRetention(b *testing.B) { }) b.Run("with-rules/match-half", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) tracer, _, _, stop := startTestTracer(b) defer stop() tracer.config.agent.DropP0s = true @@ -2447,8 +2429,7 @@ func BenchmarkSingleSpanRetention(b *testing.B) { }) b.Run("with-rules/match-all", func(b *testing.B) { - os.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) - defer os.Unsetenv("DD_SPAN_SAMPLING_RULES") + b.Setenv("DD_SPAN_SAMPLING_RULES", `[{"service": "test_*","name":"*_1", "sample_rate": 1.0, "max_per_second": 15.0}]`) tracer, _, _, stop := startTestTracer(b) defer stop() tracer.config.agent.DropP0s = true diff --git a/ddtrace/tracer/transport_test.go b/ddtrace/tracer/transport_test.go index c359ae18f8..4163010410 100644 --- a/ddtrace/tracer/transport_test.go +++ b/ddtrace/tracer/transport_test.go @@ -96,12 +96,10 @@ func TestResolveAgentAddr(t *testing.T) { } { t.Run("", func(t *testing.T) { if tt.envHost != "" { - os.Setenv("DD_AGENT_HOST", tt.envHost) - defer os.Unsetenv("DD_AGENT_HOST") + t.Setenv("DD_AGENT_HOST", tt.envHost) } if tt.envPort != "" { - os.Setenv("DD_TRACE_AGENT_PORT", tt.envPort) - defer os.Unsetenv("DD_TRACE_AGENT_PORT") + t.Setenv("DD_TRACE_AGENT_PORT", tt.envPort) } c.agentURL = resolveAgentAddr() if tt.inOpt != nil { From a3176e0728b5ed781a99ca0b146263dccfbb5d56 Mon Sep 17 00:00:00 2001 From: Piotr WOLSKI Date: Thu, 25 Jan 2024 16:37:57 -0500 Subject: [PATCH 06/13] [data streams] Track high watermark offsets (#2511) --- .../confluent-kafka-go/kafka.v2/kafka.go | 16 ++++ .../confluent-kafka-go/kafka.v2/kafka_test.go | 19 +++++ .../confluent-kafka-go/kafka/kafka.go | 16 ++++ .../confluent-kafka-go/kafka/kafka_test.go | 19 +++++ ddtrace/mocktracer/data_streams.go | 45 +++++++++++ ddtrace/mocktracer/mocktracer.go | 36 ++++----- ddtrace/tracer/data_streams.go | 10 +++ internal/datastreams/processor.go | 77 +++++++++++++++---- 8 files changed, 201 insertions(+), 37 deletions(-) create mode 100644 ddtrace/mocktracer/data_streams.go diff --git a/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka.go b/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka.go index 8c7ae41542..21b89369f2 100644 --- a/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka.go +++ b/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka.go @@ -91,6 +91,7 @@ func (c *Consumer) traceEventsChannel(in chan kafka.Event) chan kafka.Event { setConsumeCheckpoint(c.cfg.dataStreamsEnabled, c.cfg.groupID, msg) } else if offset, ok := evt.(kafka.OffsetsCommitted); ok { commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, offset.Offsets, offset.Error) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, offset.Offsets) } out <- evt @@ -176,10 +177,22 @@ func (c *Consumer) Poll(timeoutMS int) (event kafka.Event) { c.prev = c.startSpan(msg) } else if offset, ok := evt.(kafka.OffsetsCommitted); ok { commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, offset.Offsets, offset.Error) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, offset.Offsets) } return evt } +func (c *Consumer) trackHighWatermark(dataStreamsEnabled bool, offsets []kafka.TopicPartition) { + if !dataStreamsEnabled { + return + } + for _, tp := range offsets { + if _, high, err := c.Consumer.GetWatermarkOffsets(*tp.Topic, tp.Partition); err == nil { + tracer.TrackKafkaHighWatermarkOffset("", *tp.Topic, tp.Partition, high) + } + } +} + // ReadMessage polls the consumer for a message. Message will be traced. func (c *Consumer) ReadMessage(timeout time.Duration) (*kafka.Message, error) { if c.prev != nil { @@ -199,6 +212,7 @@ func (c *Consumer) ReadMessage(timeout time.Duration) (*kafka.Message, error) { func (c *Consumer) Commit() ([]kafka.TopicPartition, error) { tps, err := c.Consumer.Commit() commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } @@ -206,6 +220,7 @@ func (c *Consumer) Commit() ([]kafka.TopicPartition, error) { func (c *Consumer) CommitMessage(msg *kafka.Message) ([]kafka.TopicPartition, error) { tps, err := c.Consumer.CommitMessage(msg) commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } @@ -213,6 +228,7 @@ func (c *Consumer) CommitMessage(msg *kafka.Message) ([]kafka.TopicPartition, er func (c *Consumer) CommitOffsets(offsets []kafka.TopicPartition) ([]kafka.TopicPartition, error) { tps, err := c.Consumer.CommitOffsets(offsets) commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } diff --git a/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka_test.go b/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka_test.go index 32dfa1e9eb..8efa05fc58 100644 --- a/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka_test.go +++ b/contrib/confluentinc/confluent-kafka-go/kafka.v2/kafka_test.go @@ -9,6 +9,7 @@ import ( "context" "errors" "os" + "strings" "testing" "time" @@ -17,6 +18,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + internaldsm "gopkg.in/DataDog/dd-trace-go.v1/internal/datastreams" "github.com/confluentinc/confluent-kafka-go/v2/kafka" "github.com/stretchr/testify/assert" @@ -76,6 +78,8 @@ func produceThenConsume(t *testing.T, consumerAction consumerActionFn, producerO msg2, err := consumerAction(c) require.NoError(t, err) + _, err = c.CommitMessage(msg2) + require.NoError(t, err) assert.Equal(t, msg1.String(), msg2.String()) err = c.Close() require.NoError(t, err) @@ -84,6 +88,21 @@ func produceThenConsume(t *testing.T, consumerAction consumerActionFn, producerO require.Len(t, spans, 2) // they should be linked via headers assert.Equal(t, spans[0].TraceID(), spans[1].TraceID()) + + if c.cfg.dataStreamsEnabled { + backlogs := mt.SentDSMBacklogs() + toMap := func(b []internaldsm.Backlog) map[string]struct{} { + m := make(map[string]struct{}) + for _, b := range backlogs { + m[strings.Join(b.Tags, "")] = struct{}{} + } + return m + } + backlogsMap := toMap(backlogs) + require.Contains(t, backlogsMap, "consumer_group:"+testGroupID+"partition:0"+"topic:"+testTopic+"type:kafka_commit") + require.Contains(t, backlogsMap, "partition:0"+"topic:"+testTopic+"type:kafka_high_watermark") + require.Contains(t, backlogsMap, "partition:0"+"topic:"+testTopic+"type:kafka_produce") + } return spans, msg2 } diff --git a/contrib/confluentinc/confluent-kafka-go/kafka/kafka.go b/contrib/confluentinc/confluent-kafka-go/kafka/kafka.go index c0c9a91c29..4de425ca56 100644 --- a/contrib/confluentinc/confluent-kafka-go/kafka/kafka.go +++ b/contrib/confluentinc/confluent-kafka-go/kafka/kafka.go @@ -91,6 +91,7 @@ func (c *Consumer) traceEventsChannel(in chan kafka.Event) chan kafka.Event { setConsumeCheckpoint(c.cfg.dataStreamsEnabled, c.cfg.groupID, msg) } else if offset, ok := evt.(kafka.OffsetsCommitted); ok { commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, offset.Offsets, offset.Error) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, offset.Offsets) } out <- evt @@ -176,10 +177,22 @@ func (c *Consumer) Poll(timeoutMS int) (event kafka.Event) { c.prev = c.startSpan(msg) } else if offset, ok := evt.(kafka.OffsetsCommitted); ok { commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, offset.Offsets, offset.Error) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, offset.Offsets) } return evt } +func (c *Consumer) trackHighWatermark(dataStreamsEnabled bool, offsets []kafka.TopicPartition) { + if !dataStreamsEnabled { + return + } + for _, tp := range offsets { + if _, high, err := c.Consumer.GetWatermarkOffsets(*tp.Topic, tp.Partition); err == nil { + tracer.TrackKafkaHighWatermarkOffset("", *tp.Topic, tp.Partition, high) + } + } +} + // ReadMessage polls the consumer for a message. Message will be traced. func (c *Consumer) ReadMessage(timeout time.Duration) (*kafka.Message, error) { if c.prev != nil { @@ -199,6 +212,7 @@ func (c *Consumer) ReadMessage(timeout time.Duration) (*kafka.Message, error) { func (c *Consumer) Commit() ([]kafka.TopicPartition, error) { tps, err := c.Consumer.Commit() commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } @@ -206,6 +220,7 @@ func (c *Consumer) Commit() ([]kafka.TopicPartition, error) { func (c *Consumer) CommitMessage(msg *kafka.Message) ([]kafka.TopicPartition, error) { tps, err := c.Consumer.CommitMessage(msg) commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } @@ -213,6 +228,7 @@ func (c *Consumer) CommitMessage(msg *kafka.Message) ([]kafka.TopicPartition, er func (c *Consumer) CommitOffsets(offsets []kafka.TopicPartition) ([]kafka.TopicPartition, error) { tps, err := c.Consumer.CommitOffsets(offsets) commitOffsets(c.cfg.dataStreamsEnabled, c.cfg.groupID, tps, err) + c.trackHighWatermark(c.cfg.dataStreamsEnabled, tps) return tps, err } diff --git a/contrib/confluentinc/confluent-kafka-go/kafka/kafka_test.go b/contrib/confluentinc/confluent-kafka-go/kafka/kafka_test.go index 2196beda41..4707a1e5ae 100644 --- a/contrib/confluentinc/confluent-kafka-go/kafka/kafka_test.go +++ b/contrib/confluentinc/confluent-kafka-go/kafka/kafka_test.go @@ -9,6 +9,7 @@ import ( "context" "errors" "os" + "strings" "testing" "time" @@ -17,6 +18,7 @@ import ( "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/mocktracer" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" + internaldsm "gopkg.in/DataDog/dd-trace-go.v1/internal/datastreams" "github.com/confluentinc/confluent-kafka-go/kafka" "github.com/stretchr/testify/assert" @@ -76,6 +78,8 @@ func produceThenConsume(t *testing.T, consumerAction consumerActionFn, producerO msg2, err := consumerAction(c) require.NoError(t, err) + _, err = c.CommitMessage(msg2) + require.NoError(t, err) assert.Equal(t, msg1.String(), msg2.String()) err = c.Close() require.NoError(t, err) @@ -84,6 +88,21 @@ func produceThenConsume(t *testing.T, consumerAction consumerActionFn, producerO require.Len(t, spans, 2) // they should be linked via headers assert.Equal(t, spans[0].TraceID(), spans[1].TraceID()) + + if c.cfg.dataStreamsEnabled { + backlogs := mt.SentDSMBacklogs() + toMap := func(b []internaldsm.Backlog) map[string]struct{} { + m := make(map[string]struct{}) + for _, b := range backlogs { + m[strings.Join(b.Tags, "")] = struct{}{} + } + return m + } + backlogsMap := toMap(backlogs) + require.Contains(t, backlogsMap, "consumer_group:"+testGroupID+"partition:0"+"topic:"+testTopic+"type:kafka_commit") + require.Contains(t, backlogsMap, "partition:0"+"topic:"+testTopic+"type:kafka_high_watermark") + require.Contains(t, backlogsMap, "partition:0"+"topic:"+testTopic+"type:kafka_produce") + } return spans, msg2 } diff --git a/ddtrace/mocktracer/data_streams.go b/ddtrace/mocktracer/data_streams.go new file mode 100644 index 0000000000..dcb7191e79 --- /dev/null +++ b/ddtrace/mocktracer/data_streams.go @@ -0,0 +1,45 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016 Datadog, Inc. + +package mocktracer + +import ( + "compress/gzip" + "net/http" + + "github.com/tinylib/msgp/msgp" + + "gopkg.in/DataDog/dd-trace-go.v1/internal/datastreams" +) + +type mockDSMTransport struct { + backlogs []datastreams.Backlog +} + +// RoundTrip does nothing and returns a dummy response. +func (t *mockDSMTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // You can customize the dummy response if needed. + gzipReader, err := gzip.NewReader(req.Body) + if err != nil { + return nil, err + } + var p datastreams.StatsPayload + err = msgp.Decode(gzipReader, &p) + if err != nil { + return nil, err + } + for _, bucket := range p.Stats { + t.backlogs = append(t.backlogs, bucket.Backlogs...) + } + return &http.Response{ + StatusCode: 200, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + Request: req, + ContentLength: -1, + Body: http.NoBody, + }, nil +} diff --git a/ddtrace/mocktracer/mocktracer.go b/ddtrace/mocktracer/mocktracer.go index 3b748f5c65..0a4252bfd1 100644 --- a/ddtrace/mocktracer/mocktracer.go +++ b/ddtrace/mocktracer/mocktracer.go @@ -37,6 +37,7 @@ type Tracer interface { // FinishedSpans returns the set of finished spans. FinishedSpans() []Span + SentDSMBacklogs() []datastreams.Backlog // Reset resets the spans and services recorded in the tracer. This is // especially useful when running tests in a loop, where a clean start @@ -63,11 +64,25 @@ type mocktracer struct { sync.RWMutex // guards below spans finishedSpans []Span openSpans map[uint64]Span + dsmTransport *mockDSMTransport + dsmProcessor *datastreams.Processor +} + +func (t *mocktracer) SentDSMBacklogs() []datastreams.Backlog { + t.dsmProcessor.Flush() + return t.dsmTransport.backlogs } func newMockTracer() *mocktracer { var t mocktracer t.openSpans = make(map[uint64]Span) + t.dsmTransport = &mockDSMTransport{} + client := &http.Client{ + Transport: t.dsmTransport, + } + t.dsmProcessor = datastreams.NewProcessor(&statsd.NoOpClient{}, "env", "service", "v1", &url.URL{Scheme: "http", Host: "agent-address"}, client, func() bool { return true }) + t.dsmProcessor.Start() + t.dsmProcessor.Flush() return &t } @@ -91,27 +106,8 @@ func (t *mocktracer) StartSpan(operationName string, opts ...ddtrace.StartSpanOp return span } -type noOpTransport struct{} - -// RoundTrip does nothing and returns a dummy response. -func (t *noOpTransport) RoundTrip(req *http.Request) (*http.Response, error) { - // You can customize the dummy response if needed. - return &http.Response{ - StatusCode: 200, - Proto: "HTTP/1.1", - ProtoMajor: 1, - ProtoMinor: 1, - Request: req, - ContentLength: -1, - Body: http.NoBody, - }, nil -} - func (t *mocktracer) GetDataStreamsProcessor() *datastreams.Processor { - client := &http.Client{ - Transport: &noOpTransport{}, - } - return datastreams.NewProcessor(&statsd.NoOpClient{}, "env", "service", "v1", &url.URL{Scheme: "http", Host: "agent-address"}, client, func() bool { return true }) + return t.dsmProcessor } func (t *mocktracer) OpenSpans() []Span { diff --git a/ddtrace/tracer/data_streams.go b/ddtrace/tracer/data_streams.go index 32585cd41a..92f59f1c4f 100644 --- a/ddtrace/tracer/data_streams.go +++ b/ddtrace/tracer/data_streams.go @@ -62,3 +62,13 @@ func TrackKafkaProduceOffset(topic string, partition int32, offset int64) { } } } + +// TrackKafkaHighWatermarkOffset should be used in the producer, to track when it produces a message. +// if used together with TrackKafkaCommitOffset it can generate a Kafka lag in seconds metric. +func TrackKafkaHighWatermarkOffset(cluster string, topic string, partition int32, offset int64) { + if t, ok := internal.GetGlobalTracer().(dataStreamsContainer); ok { + if p := t.GetDataStreamsProcessor(); p != nil { + p.TrackKafkaHighWatermarkOffset(cluster, topic, partition, offset) + } + } +} diff --git a/internal/datastreams/processor.go b/internal/datastreams/processor.go index c330d8e908..4bba1ae564 100644 --- a/internal/datastreams/processor.go +++ b/internal/datastreams/processor.go @@ -55,20 +55,22 @@ type statsGroup struct { } type bucket struct { - points map[uint64]statsGroup - latestCommitOffsets map[partitionConsumerKey]int64 - latestProduceOffsets map[partitionKey]int64 - start uint64 - duration uint64 + points map[uint64]statsGroup + latestCommitOffsets map[partitionConsumerKey]int64 + latestProduceOffsets map[partitionKey]int64 + latestHighWatermarkOffsets map[partitionKey]int64 + start uint64 + duration uint64 } func newBucket(start, duration uint64) bucket { return bucket{ - points: make(map[uint64]statsGroup), - latestCommitOffsets: make(map[partitionConsumerKey]int64), - latestProduceOffsets: make(map[partitionKey]int64), - start: start, - duration: duration, + points: make(map[uint64]statsGroup), + latestCommitOffsets: make(map[partitionConsumerKey]int64), + latestProduceOffsets: make(map[partitionKey]int64), + latestHighWatermarkOffsets: make(map[partitionKey]int64), + start: start, + duration: duration, } } @@ -105,7 +107,7 @@ func (b bucket) export(timestampType TimestampType) StatsBucket { Start: b.start, Duration: b.duration, Stats: stats, - Backlogs: make([]Backlog, 0, len(b.latestCommitOffsets)+len(b.latestProduceOffsets)), + Backlogs: make([]Backlog, 0, len(b.latestCommitOffsets)+len(b.latestProduceOffsets)+len(b.latestHighWatermarkOffsets)), } for key, offset := range b.latestProduceOffsets { exported.Backlogs = append(exported.Backlogs, Backlog{Tags: []string{fmt.Sprintf("partition:%d", key.partition), fmt.Sprintf("topic:%s", key.topic), "type:kafka_produce"}, Value: offset}) @@ -113,6 +115,9 @@ func (b bucket) export(timestampType TimestampType) StatsBucket { for key, offset := range b.latestCommitOffsets { exported.Backlogs = append(exported.Backlogs, Backlog{Tags: []string{fmt.Sprintf("consumer_group:%s", key.group), fmt.Sprintf("partition:%d", key.partition), fmt.Sprintf("topic:%s", key.topic), "type:kafka_commit"}, Value: offset}) } + for key, offset := range b.latestHighWatermarkOffsets { + exported.Backlogs = append(exported.Backlogs, Backlog{Tags: []string{fmt.Sprintf("partition:%d", key.partition), fmt.Sprintf("topic:%s", key.topic), "type:kafka_high_watermark"}, Value: offset}) + } return exported } @@ -154,6 +159,7 @@ type offsetType int const ( produceOffset offsetType = iota commitOffset + highWatermarkOffset ) type kafkaOffset struct { @@ -272,6 +278,13 @@ func (p *Processor) addKafkaOffset(o kafkaOffset) { }] = o.offset return } + if o.offsetType == highWatermarkOffset { + b.latestHighWatermarkOffsets[partitionKey{ + partition: o.partition, + topic: o.topic, + }] = o.offset + return + } b.latestCommitOffsets[partitionConsumerKey{ partition: o.partition, group: o.group, @@ -279,12 +292,32 @@ func (p *Processor) addKafkaOffset(o kafkaOffset) { }] = o.offset } +func (p *Processor) processInput(in *processorInput) { + atomic.AddInt64(&p.stats.payloadsIn, 1) + if in.typ == pointTypeStats { + p.add(in.point) + } else if in.typ == pointTypeKafkaOffset { + p.addKafkaOffset(in.kafkaOffset) + } +} + +func (p *Processor) flushInput() { + for { + in := p.in.pop() + if in == nil { + return + } + p.processInput(in) + } +} + func (p *Processor) run(tick <-chan time.Time) { for { select { case now := <-tick: p.sendToAgent(p.flush(now)) case done := <-p.flushRequest: + p.flushInput() p.sendToAgent(p.flush(time.Now().Add(bucketDuration * 10))) close(done) case <-p.stop: @@ -297,12 +330,7 @@ func (p *Processor) run(tick <-chan time.Time) { time.Sleep(time.Millisecond * 10) continue } - atomic.AddInt64(&p.stats.payloadsIn, 1) - if s.typ == pointTypeStats { - p.add(s.point) - } else if s.typ == pointTypeKafkaOffset { - p.addKafkaOffset(s.kafkaOffset) - } + p.processInput(s) } } } @@ -464,6 +492,21 @@ func (p *Processor) TrackKafkaProduceOffset(topic string, partition int32, offse } } +// TrackKafkaHighWatermarkOffset should be used in the consumer, to track the high watermark offsets of each partition. +// The first argument is the Kafka cluster ID, and will be used later. +func (p *Processor) TrackKafkaHighWatermarkOffset(_ string, topic string, partition int32, offset int64) { + dropped := p.in.push(&processorInput{typ: pointTypeKafkaOffset, kafkaOffset: kafkaOffset{ + offset: offset, + topic: topic, + partition: partition, + offsetType: highWatermarkOffset, + timestamp: p.time().UnixNano(), + }}) + if dropped { + atomic.AddInt64(&p.stats.dropped, 1) + } +} + func (p *Processor) runLoadAgentFeatures(tick <-chan time.Time) { for { select { From e34ca9b942959f4b162b107994b4432132f2a93c Mon Sep 17 00:00:00 2001 From: Nick Ripley Date: Mon, 29 Jan 2024 16:19:47 -0500 Subject: [PATCH 07/13] profiler: skip flaky TestExecutionTraceRandom (#2531) --- profiler/profiler_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/profiler/profiler_test.go b/profiler/profiler_test.go index 0a8fae8bdc..1ca48308c2 100644 --- a/profiler/profiler_test.go +++ b/profiler/profiler_test.go @@ -531,6 +531,8 @@ func TestExecutionTraceMisconfiguration(t *testing.T) { } func TestExecutionTraceRandom(t *testing.T) { + t.Skip("flaky test, see: https://github.com/DataDog/dd-trace-go/issues/2529") + collectTraces := func(t *testing.T, profilePeriod, tracePeriod time.Duration, count int) int { t.Setenv("DD_PROFILING_EXECUTION_TRACE_ENABLED", "true") t.Setenv("DD_PROFILING_EXECUTION_TRACE_PERIOD", tracePeriod.String()) From d7ed3ea75ae8cc691d34f405debe585874778df1 Mon Sep 17 00:00:00 2001 From: Jason Lui <34436336+mrkagelui@users.noreply.github.com> Date: Tue, 30 Jan 2024 05:36:42 +0800 Subject: [PATCH 08/13] contrib/labstack/echo.v4: add option to ignore errors (#1567) --- contrib/labstack/echo.v4/echotrace.go | 18 ++- contrib/labstack/echo.v4/echotrace_test.go | 122 +++++++++++++++++++++ contrib/labstack/echo.v4/option.go | 9 ++ 3 files changed, 146 insertions(+), 3 deletions(-) diff --git a/contrib/labstack/echo.v4/echotrace.go b/contrib/labstack/echo.v4/echotrace.go index 3911036d60..ddbf53da08 100644 --- a/contrib/labstack/echo.v4/echotrace.go +++ b/contrib/labstack/echo.v4/echotrace.go @@ -89,7 +89,7 @@ func Middleware(opts ...Option) echo.MiddlewareFunc { } // serve the request to the next middleware err := next(c) - if err != nil { + if err != nil && !shouldIgnoreError(cfg, err) { // invokes the registered HTTP error handler c.Error(err) @@ -109,12 +109,16 @@ func Middleware(opts ...Option) echo.MiddlewareFunc { } } else if status := c.Response().Status; status > 0 { if cfg.isStatusError(status) { - finishOpts = append(finishOpts, tracer.WithError(fmt.Errorf("%d: %s", status, http.StatusText(status)))) + if statusErr := errorFromStatusCode(status); !shouldIgnoreError(cfg, statusErr) { + finishOpts = append(finishOpts, tracer.WithError(statusErr)) + } } span.SetTag(ext.HTTPCode, strconv.Itoa(status)) } else { if cfg.isStatusError(200) { - finishOpts = append(finishOpts, tracer.WithError(fmt.Errorf("%d: %s", 200, http.StatusText(200)))) + if statusErr := errorFromStatusCode(200); !shouldIgnoreError(cfg, statusErr) { + finishOpts = append(finishOpts, tracer.WithError(statusErr)) + } } span.SetTag(ext.HTTPCode, "200") } @@ -122,3 +126,11 @@ func Middleware(opts ...Option) echo.MiddlewareFunc { } } } + +func errorFromStatusCode(statusCode int) error { + return fmt.Errorf("%d: %s", statusCode, http.StatusText(statusCode)) +} + +func shouldIgnoreError(cfg *config, err error) bool { + return cfg.errCheck != nil && !cfg.errCheck(err) +} diff --git a/contrib/labstack/echo.v4/echotrace_test.go b/contrib/labstack/echo.v4/echotrace_test.go index 01eb5f873e..50af8802d1 100644 --- a/contrib/labstack/echo.v4/echotrace_test.go +++ b/contrib/labstack/echo.v4/echotrace_test.go @@ -589,6 +589,128 @@ func TestWithHeaderTags(t *testing.T) { }) } +func TestWithErrorCheck(t *testing.T) { + tests := []struct { + name string + err error + opts []Option + wantErr error + }{ + { + name: "ignore-4xx-404-error", + err: &echo.HTTPError{ + Code: http.StatusNotFound, + Message: "not found", + Internal: errors.New("not found"), + }, + opts: []Option{ + WithErrorCheck(func(err error) bool { + var he *echo.HTTPError + if errors.As(err, &he) { + // do not tag 4xx errors + return !(he.Code < 500 && he.Code >= 400) + } + return true + }), + }, + wantErr: nil, // 404 is returned, hence not tagged + }, + { + name: "ignore-4xx-500-error", + err: &echo.HTTPError{ + Code: http.StatusInternalServerError, + Message: "internal error", + Internal: errors.New("internal error"), + }, + opts: []Option{ + WithErrorCheck(func(err error) bool { + var he *echo.HTTPError + if errors.As(err, &he) { + // do not tag 4xx errors + return !(he.Code < 500 && he.Code >= 400) + } + return true + }), + }, + wantErr: &echo.HTTPError{ + Code: http.StatusInternalServerError, + Message: "internal error", + Internal: errors.New("internal error"), + }, // this is 500, tagged + }, + { + name: "ignore-none", + err: errors.New("any error"), + opts: []Option{ + WithErrorCheck(func(err error) bool { + return true + }), + }, + wantErr: errors.New("any error"), + }, + { + name: "ignore-all", + err: errors.New("any error"), + opts: []Option{ + WithErrorCheck(func(err error) bool { + return false + }), + }, + wantErr: nil, + }, + { + // withErrorCheck also runs for the errors created from the WithStatusCheck option. + name: "ignore-errors-from-status-check", + err: &echo.HTTPError{ + Code: http.StatusNotFound, + Message: "internal error", + Internal: errors.New("internal error"), + }, + opts: []Option{ + WithStatusCheck(func(statusCode int) bool { + return statusCode == http.StatusNotFound + }), + WithErrorCheck(func(err error) bool { + return false + }), + }, + wantErr: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mt := mocktracer.Start() + defer mt.Stop() + + router := echo.New() + router.Use(Middleware(tt.opts...)) + var called, traced bool + + // always return the specified error + router.GET("/err", func(c echo.Context) error { + _, traced = tracer.SpanFromContext(c.Request().Context()) + called = true + return tt.err + }) + r := httptest.NewRequest(http.MethodGet, "/err", nil) + w := httptest.NewRecorder() + router.ServeHTTP(w, r) + + assert.True(t, called) + assert.True(t, traced) + spans := mt.FinishedSpans() + require.Len(t, spans, 1) // fail at once if there is no span + + span := spans[0] + if tt.wantErr == nil { + assert.NotContains(t, span.Tags(), ext.Error) + return + } + assert.Equal(t, tt.wantErr, span.Tag(ext.Error)) + }) + } +} + func TestWithCustomTags(t *testing.T) { assert := assert.New(t) mt := mocktracer.Start() diff --git a/contrib/labstack/echo.v4/option.go b/contrib/labstack/echo.v4/option.go index 9ce8263890..2994d42c61 100644 --- a/contrib/labstack/echo.v4/option.go +++ b/contrib/labstack/echo.v4/option.go @@ -27,6 +27,7 @@ type config struct { isStatusError func(statusCode int) bool translateError func(err error) (*echo.HTTPError, bool) headerTags *internal.LockMap + errCheck func(error) bool tags map[string]interface{} } @@ -129,6 +130,14 @@ func WithHeaderTags(headers []string) Option { } } +// WithErrorCheck sets the func which determines if err would be ignored (if it returns true, the error is not tagged). +// This function also checks the errors created from the WithStatusCheck option. +func WithErrorCheck(errCheck func(error) bool) Option { + return func(cfg *config) { + cfg.errCheck = errCheck + } +} + // WithCustomTag will attach the value to the span tagged by the key. Standard // span tags cannot be replaced. func WithCustomTag(key string, value interface{}) Option { From c527cccd7386a26ad6144d92f2d741277d570159 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dario=20Casta=C3=B1=C3=A9?= Date: Tue, 30 Jan 2024 16:28:20 +0100 Subject: [PATCH 09/13] ddtrace/opentelemetry: add RWMutex to handle concurrent calls to setters (#2521) --- ddtrace/opentelemetry/span.go | 12 +++++++++++- ddtrace/opentelemetry/tracer_test.go | 19 +++++++++++++++++++ ddtrace/tracer/textmap.go | 7 ++++--- ddtrace/tracer/tracer_test.go | 22 ++++++++++++++++++++++ 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/ddtrace/opentelemetry/span.go b/ddtrace/opentelemetry/span.go index 466a0fa7b3..61c197434a 100644 --- a/ddtrace/opentelemetry/span.go +++ b/ddtrace/opentelemetry/span.go @@ -10,6 +10,7 @@ import ( "errors" "strconv" "strings" + "sync" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" @@ -25,7 +26,8 @@ import ( var _ oteltrace.Span = (*span)(nil) type span struct { - noop.Span // https://pkg.go.dev/go.opentelemetry.io/otel/trace#hdr-API_Implementations + noop.Span // https://pkg.go.dev/go.opentelemetry.io/otel/trace#hdr-API_Implementations + mu sync.RWMutex `msg:"-"` // all fields are protected by this RWMutex DD tracer.Span finished bool attributes map[string]interface{} @@ -38,10 +40,14 @@ type span struct { func (s *span) TracerProvider() oteltrace.TracerProvider { return s.oteltracer.provider } func (s *span) SetName(name string) { + s.mu.Lock() + defer s.mu.Unlock() s.attributes[ext.SpanName] = strings.ToLower(name) } func (s *span) End(options ...oteltrace.SpanEndOption) { + s.mu.Lock() + defer s.mu.Unlock() if s.finished { return } @@ -157,6 +163,8 @@ type statusInfo struct { // value before (OK > Error > Unset), the code will not be changed. // The code and description are set once when the span is finished. func (s *span) SetStatus(code otelcodes.Code, description string) { + s.mu.Lock() + defer s.mu.Unlock() if code >= s.statusInfo.code { s.statusInfo = statusInfo{code, description} } @@ -175,6 +183,8 @@ func (s *span) SetStatus(code otelcodes.Code, description string) { // The list of reserved tags might be extended in the future. // Any other non-reserved tags will be set as provided. func (s *span) SetAttributes(kv ...attribute.KeyValue) { + s.mu.Lock() + defer s.mu.Unlock() for _, kv := range kv { if k, v := toReservedAttributes(string(kv.Key), kv.Value); k != "" { s.attributes[k] = v diff --git a/ddtrace/opentelemetry/tracer_test.go b/ddtrace/opentelemetry/tracer_test.go index c9a6a3f91a..c09c291292 100644 --- a/ddtrace/opentelemetry/tracer_test.go +++ b/ddtrace/opentelemetry/tracer_test.go @@ -207,6 +207,25 @@ func TestSpanTelemetry(t *testing.T) { telemetryClient.AssertNumberOfCalls(t, "Count", 1) } +func TestConcurrentSetAttributes(_ *testing.T) { + tp := NewTracerProvider() + otel.SetTracerProvider(tp) + tr := otel.Tracer("") + + _, span := tr.Start(context.Background(), "test") + defer span.End() + + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + i := i + go func(val int) { + defer wg.Done() + span.SetAttributes(attribute.Float64("workerID", float64(i))) + }(i) + } +} + func BenchmarkOTelApiWithNoTags(b *testing.B) { testData := struct { env, srv, op string diff --git a/ddtrace/tracer/textmap.go b/ddtrace/tracer/textmap.go index 3a9ee6a59f..a71c099850 100644 --- a/ddtrace/tracer/textmap.go +++ b/ddtrace/tracer/textmap.go @@ -363,10 +363,11 @@ func (p *propagator) injectTextMap(spanCtx ddtrace.SpanContext, writer TextMapWr if ctx.origin != "" { writer.Set(originHeader, ctx.origin) } - // propagate OpenTracing baggage - for k, v := range ctx.baggage { + ctx.ForeachBaggageItem(func(k, v string) bool { + // Propagate OpenTracing baggage. writer.Set(p.cfg.BaggagePrefix+k, v) - } + return true + }) if p.cfg.MaxTagsHeaderLen <= 0 { return nil } diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 9914a40dbe..756a652d60 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -922,6 +922,28 @@ func TestTracerBaggageImmutability(t *testing.T) { assert.Equal("changed!", childContext.baggage["key"]) } +func TestTracerInjectConcurrency(t *testing.T) { + tracer, _, _, stop := startTestTracer(t) + defer stop() + span, _ := StartSpanFromContext(context.Background(), "main") + defer span.Finish() + + var wg sync.WaitGroup + for i := 0; i < 500; i++ { + wg.Add(1) + i := i + go func(val int) { + defer wg.Done() + span.SetBaggageItem("val", fmt.Sprintf("%d", val)) + + traceContext := map[string]string{} + _ = tracer.Inject(span.Context(), TextMapCarrier(traceContext)) + }(i) + } + + wg.Wait() +} + func TestTracerSpanTags(t *testing.T) { tracer := newTracer() defer tracer.Stop() From 8c4834f24951876c7f50a98fa640a96541cc3c04 Mon Sep 17 00:00:00 2001 From: Romain Marcadier Date: Tue, 30 Jan 2024 19:46:19 +0100 Subject: [PATCH 10/13] ddtrace/opentelemetry: refactor OTel API tests (#2503) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Katie Hockman Co-authored-by: Dario Castañé --- ddtrace/opentelemetry/otel_test.go | 20 +- ddtrace/opentelemetry/span_test.go | 280 +++++++++++++++------------ ddtrace/opentelemetry/tracer_test.go | 6 +- ddtrace/tracer/tracer.go | 2 +- ddtrace/tracer/writer.go | 3 +- 5 files changed, 172 insertions(+), 139 deletions(-) diff --git a/ddtrace/opentelemetry/otel_test.go b/ddtrace/opentelemetry/otel_test.go index 81ea64d62a..78c17670f9 100644 --- a/ddtrace/opentelemetry/otel_test.go +++ b/ddtrace/opentelemetry/otel_test.go @@ -11,7 +11,6 @@ import ( "context" "net/http" "net/http/httptest" - "strings" "testing" "github.com/stretchr/testify/assert" @@ -23,6 +22,7 @@ import ( ) func TestHttpDistributedTrace(t *testing.T) { + assert := assert.New(t) tp, payloads, cleanup := mockTracerProvider(t) defer cleanup() otel.SetTracerProvider(tp) @@ -33,11 +33,10 @@ func TestHttpDistributedTrace(t *testing.T) { w := otelhttp.NewHandler(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { receivedSpan := oteltrace.SpanFromContext(r.Context()) - assert.Equal(t, rootSpan.SpanContext().TraceID(), receivedSpan.SpanContext().TraceID()) + assert.Equal(rootSpan.SpanContext().TraceID(), receivedSpan.SpanContext().TraceID()) }), "testOperation") testServer := httptest.NewServer(w) defer testServer.Close() - c := http.Client{Transport: otelhttp.NewTransport(nil)} req, err := http.NewRequestWithContext(sctx, http.MethodGet, testServer.URL, nil) require.NoError(t, err) @@ -47,12 +46,11 @@ func TestHttpDistributedTrace(t *testing.T) { rootSpan.End() p := <-payloads - numSpans := strings.Count(p, "\"span_id\"") - assert.Equal(t, 3, numSpans) - assert.Contains(t, p, `"name":"internal"`) - assert.Contains(t, p, `"name":"server.request`) - assert.Contains(t, p, `"name":"client.request"`) - assert.Contains(t, p, `"resource":"testRootSpan"`) - assert.Contains(t, p, `"resource":"testOperation"`) - assert.Contains(t, p, `"resource":"HTTP GET"`) + assert.Len(p, 2) + assert.Equal("server.request", p[0][0]["name"]) + assert.Equal("internal", p[1][0]["name"]) + assert.Equal("client.request", p[1][1]["name"]) + assert.Equal("testOperation", p[0][0]["resource"]) + assert.Equal("testRootSpan", p[1][0]["resource"]) + assert.Equal("HTTP GET", p[1][1]["resource"]) } diff --git a/ddtrace/opentelemetry/span_test.go b/ddtrace/opentelemetry/span_test.go index f60a0a5f27..127f23fce7 100644 --- a/ddtrace/opentelemetry/span_test.go +++ b/ddtrace/opentelemetry/span_test.go @@ -8,6 +8,7 @@ package opentelemetry import ( "bytes" "context" + "encoding/json" "errors" "fmt" "io" @@ -28,21 +29,41 @@ import ( oteltrace "go.opentelemetry.io/otel/trace" ) -func mockTracerProvider(t *testing.T, opts ...tracer.StartOption) (tp *TracerProvider, payloads chan string, cleanup func()) { - payloads = make(chan string) +type traces [][]map[string]interface{} + +func mockTracerProvider(t *testing.T, opts ...tracer.StartOption) (tp *TracerProvider, payloads chan traces, cleanup func()) { + payloads = make(chan traces) s, c := httpmem.ServerAndClient(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/v0.4/traces": if h := r.Header.Get("X-Datadog-Trace-Count"); h == "0" { return } - buf, err := io.ReadAll(r.Body) + req := r.Clone(context.Background()) + defer req.Body.Close() + buf, err := io.ReadAll(req.Body) if err != nil || len(buf) == 0 { - t.Fatalf("Test agent: Error receiving traces") + t.Fatalf("Test agent: Error receiving traces: %v", err) + } + var payload bytes.Buffer + _, err = msgp.UnmarshalAsJSON(&payload, buf) + if err != nil { + t.Fatalf("Failed to unmarshal payload bytes as JSON: %v", err) + } + var tr [][]map[string]interface{} + err = json.Unmarshal(payload.Bytes(), &tr) + if err != nil || len(tr) == 0 { + t.Fatalf("Failed to unmarshal payload bytes as trace: %v", err) + } + payloads <- tr + default: + if r.Method == "GET" { + // Write an empty JSON object to the output, to avoid spurious decoding + // errors to be reported in the logs, which may lead someone + // investigating a test failure into the wrong direction. + w.Write([]byte("{}")) + return } - var js bytes.Buffer - msgp.UnmarshalAsJSON(&js, buf) - payloads <- js.String() } w.WriteHeader(200) })) @@ -50,24 +71,27 @@ func mockTracerProvider(t *testing.T, opts ...tracer.StartOption) (tp *TracerPro tp = NewTracerProvider(opts...) otel.SetTracerProvider(tp) return tp, payloads, func() { - s.Close() - tp.Shutdown() + if err := s.Close(); err != nil { + t.Fatalf("Test Agent server Close failure: %v", err) + } + if err := tp.Shutdown(); err != nil { + t.Fatalf("Tracer Provider shutdown failure: %v", err) + } } } -func waitForPayload(ctx context.Context, payloads chan string) (string, error) { +func waitForPayload(payloads chan traces) (traces, error) { select { - case <-ctx.Done(): - return "", fmt.Errorf("Timed out waiting for traces") case p := <-payloads: return p, nil + case <-time.After(10 * time.Second): + return nil, fmt.Errorf("Timed out waiting for traces") } } func TestSpanResourceNameDefault(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + ctx := context.Background() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") @@ -77,39 +101,39 @@ func TestSpanResourceNameDefault(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, `"name":"internal"`) - assert.Contains(p, `"resource":"OperationName"`) + p := traces[0] + assert.Equal("internal", p[0]["name"]) + assert.Equal("OperationName", p[0]["resource"]) } func TestSpanSetName(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() _, sp := tr.Start(ctx, "OldName") sp.SetName("NewName") sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, strings.ToLower("NewName")) + p := traces[0] + assert.Equal(strings.ToLower("NewName"), p[0]["name"]) } func TestSpanEnd(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() @@ -139,22 +163,22 @@ func TestSpanEnd(t *testing.T) { } tracer.Flush() - payload, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } + p := traces[0] - assert.Contains(payload, name) - assert.NotContains(payload, ignoredName) - assert.Contains(payload, msg) - assert.NotContains(payload, ignoredMsg) - assert.Contains(payload, `"error":1`) // this should be an error span - + assert.Equal(name, p[0]["resource"]) + assert.Equal(ext.SpanKindInternal, p[0]["name"]) // default + assert.Equal(1.0, p[0]["error"]) // this should be an error span + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, msg) for k, v := range attributes { - assert.Contains(payload, fmt.Sprintf("\"%s\":\"%s\"", k, v)) + assert.Contains(meta, fmt.Sprintf("%s:%s", k, v)) } for k, v := range ignoredAttributes { - assert.NotContains(payload, fmt.Sprintf("\"%s\":\"%s\"", k, v)) + assert.NotContains(meta, fmt.Sprintf("%s:%s", k, v)) } } @@ -193,26 +217,25 @@ func TestSpanSetStatus(t *testing.T) { for _, test := range testData { t.Run(fmt.Sprintf("Setting Code: %d", test.code), func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - var sp oteltrace.Span testStatus := func() { sp.End() tracer.Flush() - payload, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } + p := traces[0] // An error description is set IFF the span has an error // status code value. Messages related to any other status code // are ignored. + meta := fmt.Sprintf("%v", p[0]["meta"]) if test.code == codes.Error { - assert.Contains(payload, test.msg) + assert.Contains(meta, test.msg) } else { - assert.NotContains(payload, test.msg) + assert.NotContains(meta, test.msg) } - assert.NotContains(payload, test.ignoredCode) + assert.NotContains(meta, test.ignoredCode) } _, sp = tr.Start(context.Background(), "test") sp.SetStatus(test.code, test.msg) @@ -229,8 +252,6 @@ func TestSpanSetStatus(t *testing.T) { func TestSpanContextWithStartOptions(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() @@ -262,27 +283,28 @@ func TestSpanContextWithStartOptions(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - if strings.Count(p, "span_id") != 2 { - t.Fatalf("payload does not contain two spans\n%s", p) - } - assert.Contains(p, `"service":"persisted_srv"`) - assert.Contains(p, `"resource":"persisted_ctx_rsc"`) - assert.Contains(p, `"span.kind":"producer"`) - assert.Contains(p, fmt.Sprint(spanID)) - assert.Contains(p, fmt.Sprint(startTime.UnixNano())) - assert.Contains(p, fmt.Sprint(duration.Nanoseconds())) + p := traces[0] + t.Logf("%v", p[0]) + assert.Len(p, 2) + assert.Equal("persisted_srv", p[0]["service"]) + assert.Equal("persisted_ctx_rsc", p[0]["resource"]) + assert.Equal(1234567890.0, p[0]["span_id"]) + assert.Equal("producer", p[0]["name"]) + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, "producer") + assert.Equal(float64(startTime.UnixNano()), p[0]["start"]) + assert.Equal(float64(duration.Nanoseconds()), p[0]["duration"]) assert.NotContains(p, "discarded") - assert.Equal(1, strings.Count(p, `"span_id":1234567890`)) + assert.NotEqual(1234567890.0, p[1]["span_id"]) } func TestSpanContextWithStartOptionsPriorityOrder(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() @@ -299,20 +321,21 @@ func TestSpanContextWithStartOptionsPriorityOrder(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, "persisted_ctx_rsc") - assert.Contains(p, "persisted_srv") - assert.Contains(p, `"span.kind":"producer"`) + p := traces[0] + assert.Equal("persisted_srv", p[0]["service"]) + assert.Equal("persisted_ctx_rsc", p[0]["resource"]) + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, "producer") assert.NotContains(p, "discarded") } func TestSpanEndOptionsPriorityOrder(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() @@ -331,100 +354,112 @@ func TestSpanEndOptionsPriorityOrder(t *testing.T) { EndOptions(sp, tracer.FinishTime(startTime.Add(time.Second*5))) // EndOptions timestamp should prevail sp.End(oteltrace.WithTimestamp(startTime.Add(time.Second * 3))) + duration := time.Second * 5 // making sure end options don't have effect after the span has returned - EndOptions(sp, tracer.FinishTime(startTime.Add(time.Second*2))) + EndOptions(sp, tracer.FinishTime(startTime.Add(duration))) sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, `"duration":5000000000,`) - assert.NotContains(p, `"duration":2000000000,`) - assert.NotContains(p, `"duration":1000000000,`) - assert.NotContains(p, `"duration":3000000000,`) + p := traces[0] + assert.Equal(float64(duration.Nanoseconds()), p[0]["duration"]) } func TestSpanEndOptions(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() + spanID := uint64(1234567890) startTime := time.Now() + duration := time.Second * 5 _, sp := tr.Start( ContextWithStartOptions(context.Background(), tracer.ResourceName("ctx_rsc"), tracer.ServiceName("ctx_srv"), tracer.StartTime(startTime), - tracer.WithSpanID(1234567890), + tracer.WithSpanID(spanID), ), "op_name") - - EndOptions(sp, tracer.FinishTime(startTime.Add(time.Second*5)), + EndOptions(sp, tracer.FinishTime(startTime.Add(duration)), tracer.WithError(errors.New("persisted_option"))) sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, "ctx_srv") - assert.Contains(p, "ctx_rsc") - assert.Contains(p, "1234567890") - assert.Contains(p, fmt.Sprint(startTime.UnixNano())) - assert.Contains(p, `"duration":5000000000,`) - assert.Contains(p, `persisted_option`) - assert.Contains(p, `"error":1`) + p := traces[0] + assert.Equal("ctx_srv", p[0]["service"]) + assert.Equal("ctx_rsc", p[0]["resource"]) + assert.Equal(1234567890.0, p[0]["span_id"]) + assert.Equal(float64(startTime.UnixNano()), p[0]["start"]) + assert.Equal(float64(duration.Nanoseconds()), p[0]["duration"]) + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, "persisted_option") + assert.Equal(1.0, p[0]["error"]) // this should be an error span } func TestSpanSetAttributes(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t) tr := otel.Tracer("") defer cleanup() - attributes := [][]string{{"k1", "v1_old"}, - {"k2", "v2"}, - {"k1", "v1_new"}, + toBeIgnored := map[string]string{"k1": "v1_old"} + attributes := map[string]string{ + "k2": "v2", + "k1": "v1_new", // maps to 'name' - {"operation.name", "ops"}, + "operation.name": "ops", // maps to 'service' - {"service.name", "srv"}, + "service.name": "srv", // maps to 'resource' - {"resource.name", "rsr"}, + "resource.name": "rsr", // maps to 'type' - {"span.type", "db"}, + "span.type": "db", } _, sp := tr.Start(context.Background(), "test") - for _, tag := range attributes { - sp.SetAttributes(attribute.String(tag[0], tag[1])) + for k, v := range toBeIgnored { + sp.SetAttributes(attribute.String(k, v)) + } + for k, v := range attributes { + sp.SetAttributes(attribute.String(k, v)) } // maps to '_dd1.sr.eausr' sp.SetAttributes(attribute.Int("analytics.event", 1)) sp.End() tracer.Flush() - payload, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(payload, `"k1":"v1_new"`) - assert.Contains(payload, `"k2":"v2"`) - assert.NotContains(payload, "v1_old") + p := traces[0] + meta := fmt.Sprintf("%v", p[0]["meta"]) + for k, v := range toBeIgnored { + assert.NotContains(meta, fmt.Sprintf("%s:%s", k, v)) + } + assert.Contains(meta, fmt.Sprintf("%s:%s", "k1", "v1_new")) + assert.Contains(meta, fmt.Sprintf("%s:%s", "k2", "v2")) // reserved attributes - assert.Contains(payload, `"name":"ops"`) - assert.Contains(payload, `"service":"srv"`) - assert.Contains(payload, `"resource":"rsr"`) - assert.Contains(payload, `"type":"db"`) - assert.Contains(payload, `"_dd1.sr.eausr":1`) + assert.NotContains(meta, fmt.Sprintf("%s:%s", "name", "ops")) + assert.NotContains(meta, fmt.Sprintf("%s:%s", "service", "srv")) + assert.NotContains(meta, fmt.Sprintf("%s:%s", "resource", "rsr")) + assert.NotContains(meta, fmt.Sprintf("%s:%s", "type", "db")) + assert.Equal("ops", p[0]["name"]) + assert.Equal("srv", p[0]["service"]) + assert.Equal("rsr", p[0]["resource"]) + assert.Equal("db", p[0]["type"]) + metrics := fmt.Sprintf("%v", p[0]["metrics"]) + assert.Contains(metrics, fmt.Sprintf("%s:%s", "_dd1.sr.eausr", "1")) } func TestSpanSetAttributesWithRemapping(t *testing.T) { @@ -443,17 +478,16 @@ func TestSpanSetAttributesWithRemapping(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, "graphql.server.request") + p := traces[0] + assert.Equal("graphql.server.request", p[0]["name"]) } func TestTracerStartOptions(t *testing.T) { assert := assert.New(t) - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() _, payloads, cleanup := mockTracerProvider(t, tracer.WithEnv("test_env"), tracer.WithService("test_serv")) tr := otel.Tracer("") @@ -462,12 +496,14 @@ func TestTracerStartOptions(t *testing.T) { _, sp := tr.Start(context.Background(), "test") sp.End() tracer.Flush() - payload, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(payload, "\"service\":\"test_serv\"") - assert.Contains(payload, "\"env\":\"test_env\"") + p := traces[0] + assert.Equal("test_serv", p[0]["service"]) + meta := fmt.Sprintf("%v", p[0]["meta"]) + assert.Contains(meta, fmt.Sprintf("%s:%s", "env", "test_env")) } func TestOperationNameRemapping(t *testing.T) { @@ -483,13 +519,15 @@ func TestOperationNameRemapping(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(p, "graphql.server.request") + p := traces[0] + assert.Equal("graphql.server.request", p[0]["name"]) } func TestRemapName(t *testing.T) { + assert := assert.New(t) testCases := []struct { spanKind oteltrace.SpanKind in []attribute.KeyValue @@ -597,10 +635,6 @@ func TestRemapName(t *testing.T) { out: "internal", }, } - - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() - _, payloads, cleanup := mockTracerProvider(t, tracer.WithEnv("test_env"), tracer.WithService("test_serv")) tr := otel.Tracer("") defer cleanup() @@ -612,18 +646,18 @@ func TestRemapName(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(t, p, test.out) + p := traces[0] + assert.Equal(test.out, p[0]["name"]) }) } } func TestRemapWithMultipleSetAttributes(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - defer cancel() + assert := assert.New(t) _, payloads, cleanup := mockTracerProvider(t, tracer.WithEnv("test_env"), tracer.WithService("test_serv")) tr := otel.Tracer("") @@ -641,13 +675,15 @@ func TestRemapWithMultipleSetAttributes(t *testing.T) { sp.End() tracer.Flush() - p, err := waitForPayload(ctx, payloads) + traces, err := waitForPayload(payloads) if err != nil { t.Fatalf(err.Error()) } - assert.Contains(t, p, `"name":"overriden.name"`) - assert.Contains(t, p, `"resource":"new.name"`) - assert.Contains(t, p, `"service":"new.service.name"`) - assert.Contains(t, p, `"type":"new.span.type"`) - assert.Contains(t, p, `"_dd1.sr.eausr":1`) + p := traces[0] + assert.Equal("overriden.name", p[0]["name"]) + assert.Equal("new.name", p[0]["resource"]) + assert.Equal("new.service.name", p[0]["service"]) + assert.Equal("new.span.type", p[0]["type"]) + metrics := fmt.Sprintf("%v", p[0]["metrics"]) + assert.Contains(metrics, fmt.Sprintf("%s:%s", "_dd1.sr.eausr", "1")) } diff --git a/ddtrace/opentelemetry/tracer_test.go b/ddtrace/opentelemetry/tracer_test.go index c09c291292..ac3d685dbe 100644 --- a/ddtrace/opentelemetry/tracer_test.go +++ b/ddtrace/opentelemetry/tracer_test.go @@ -139,8 +139,6 @@ func TestForceFlush(t *testing.T) { } for _, tc := range testData { t.Run(fmt.Sprintf("Flush success: %t", tc.flushed), func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() tp, payloads, cleanup := mockTracerProvider(t) defer cleanup() @@ -156,10 +154,10 @@ func TestForceFlush(t *testing.T) { _, sp := tr.Start(context.Background(), "test_span") sp.End() tp.forceFlush(tc.timeOut, setFlushStatus, tc.flushFunc) - payload, err := waitForPayload(ctx, payloads) + p, err := waitForPayload(payloads) if tc.flushed { assert.NoError(err) - assert.Contains(payload, "test_span") + assert.Equal("test_span", p[0][0]["resource"]) assert.Equal(OK, flushStatus) } else { assert.Equal(ERROR, flushStatus) diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index 1c86293c40..ddef5ffa6f 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -372,7 +372,7 @@ func (t *tracer) worker(tick <-chan time.Time) { t.statsd.Flush() t.stats.flushAndSend(time.Now(), withCurrentBucket) // TODO(x): In reality, the traceWriter.flush() call is not synchronous - // when using the agent traceWriter. However, this functionnality is used + // when using the agent traceWriter. However, this functionality is used // in Lambda so for that purpose this mechanism should suffice. done <- struct{}{} diff --git a/ddtrace/tracer/writer.go b/ddtrace/tracer/writer.go index a4c56e0c90..877c8ada20 100644 --- a/ddtrace/tracer/writer.go +++ b/ddtrace/tracer/writer.go @@ -106,7 +106,8 @@ func (h *agentTraceWriter) flush() { for attempt := 0; attempt <= h.config.sendRetries; attempt++ { size, count = p.size(), p.itemCount() log.Debug("Sending payload: size: %d traces: %d\n", size, count) - rc, err := h.config.transport.send(p) + var rc io.ReadCloser + rc, err = h.config.transport.send(p) if err == nil { log.Debug("sent traces after %d attempts", attempt+1) h.statsd.Count("datadog.tracer.flush_bytes", int64(size), nil, 1) From e707cc94fdd4e663fccdb6ab3fb3258539444bc0 Mon Sep 17 00:00:00 2001 From: Katie Hockman Date: Wed, 31 Jan 2024 10:58:51 -0500 Subject: [PATCH 11/13] tracer: verify that hostname reporting is honored regardless of stats calculation (#2533) --- ddtrace/tracer/tracer_test.go | 76 ++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/ddtrace/tracer/tracer_test.go b/ddtrace/tracer/tracer_test.go index 756a652d60..b3cc1b09ba 100644 --- a/ddtrace/tracer/tracer_test.go +++ b/ddtrace/tracer/tracer_test.go @@ -1627,44 +1627,54 @@ func TestTracerFlush(t *testing.T) { func TestTracerReportsHostname(t *testing.T) { const hostname = "hostname-test" - t.Run("DD_TRACE_REPORT_HOSTNAME/set", func(t *testing.T) { - t.Setenv("DD_TRACE_REPORT_HOSTNAME", "true") + testReportHostnameEnabled := func(t *testing.T, name string, withComputeStats bool) { + t.Run(name, func(t *testing.T) { + t.Setenv("DD_TRACE_REPORT_HOSTNAME", "true") + t.Setenv("DD_TRACE_COMPUTE_STATS", fmt.Sprintf("%t", withComputeStats)) - tracer, _, _, stop := startTestTracer(t) - defer stop() - - root := tracer.StartSpan("root").(*span) - child := tracer.StartSpan("child", ChildOf(root.Context())).(*span) - child.Finish() - root.Finish() - - assert := assert.New(t) - - name, ok := root.Meta[keyHostname] - assert.True(ok) - assert.Equal(name, tracer.config.hostname) - - name, ok = child.Meta[keyHostname] - assert.True(ok) - assert.Equal(name, tracer.config.hostname) - }) + tracer, _, _, stop := startTestTracer(t) + defer stop() - t.Run("DD_TRACE_REPORT_HOSTNAME/unset", func(t *testing.T) { - tracer, _, _, stop := startTestTracer(t) - defer stop() + root := tracer.StartSpan("root").(*span) + child := tracer.StartSpan("child", ChildOf(root.Context())).(*span) + child.Finish() + root.Finish() - root := tracer.StartSpan("root").(*span) - child := tracer.StartSpan("child", ChildOf(root.Context())).(*span) - child.Finish() - root.Finish() + assert := assert.New(t) - assert := assert.New(t) + name, ok := root.Meta[keyHostname] + assert.True(ok) + assert.Equal(name, tracer.config.hostname) - _, ok := root.Meta[keyHostname] - assert.False(ok) - _, ok = child.Meta[keyHostname] - assert.False(ok) - }) + name, ok = child.Meta[keyHostname] + assert.True(ok) + assert.Equal(name, tracer.config.hostname) + }) + } + testReportHostnameEnabled(t, "DD_TRACE_REPORT_HOSTNAME/set,DD_TRACE_COMPUTE_STATS/true", true) + testReportHostnameEnabled(t, "DD_TRACE_REPORT_HOSTNAME/set,DD_TRACE_COMPUTE_STATS/false", false) + + testReportHostnameDisabled := func(t *testing.T, name string, withComputeStats bool) { + t.Run(name, func(t *testing.T) { + t.Setenv("DD_TRACE_COMPUTE_STATS", fmt.Sprintf("%t", withComputeStats)) + tracer, _, _, stop := startTestTracer(t) + defer stop() + + root := tracer.StartSpan("root").(*span) + child := tracer.StartSpan("child", ChildOf(root.Context())).(*span) + child.Finish() + root.Finish() + + assert := assert.New(t) + + _, ok := root.Meta[keyHostname] + assert.False(ok) + _, ok = child.Meta[keyHostname] + assert.False(ok) + }) + } + testReportHostnameDisabled(t, "DD_TRACE_REPORT_HOSTNAME/unset,DD_TRACE_COMPUTE_STATS/true", true) + testReportHostnameDisabled(t, "DD_TRACE_REPORT_HOSTNAME/unset,DD_TRACE_COMPUTE_STATS/false", false) t.Run("WithHostname", func(t *testing.T) { tracer, _, _, stop := startTestTracer(t, WithHostname(hostname)) From bbbbd7a66fdefbd42ebed3711e5df6bdd18e7db4 Mon Sep 17 00:00:00 2001 From: Munir Abdinur Date: Thu, 1 Feb 2024 15:01:25 -0500 Subject: [PATCH 12/13] tracing: Adds support for Span Links (#2502) --- ddtrace/ddtrace.go | 22 +++ ddtrace/mocktracer/mockspan.go | 1 + ddtrace/opentelemetry/span_test.go | 58 +++++++ ddtrace/opentelemetry/tracer.go | 30 ++++ ddtrace/span_link_msgp.go | 221 ++++++++++++++++++++++++ ddtrace/tracer/option.go | 7 + ddtrace/tracer/option_test.go | 14 ++ ddtrace/tracer/span.go | 25 +-- ddtrace/tracer/span_msgp.go | 265 +++++++++++++++++++++++++---- ddtrace/tracer/tracer.go | 4 + ddtrace/tracer/writer_test.go | 2 +- 11 files changed, 599 insertions(+), 50 deletions(-) create mode 100644 ddtrace/span_link_msgp.go diff --git a/ddtrace/ddtrace.go b/ddtrace/ddtrace.go index c4d106458c..e311b5ff25 100644 --- a/ddtrace/ddtrace.go +++ b/ddtrace/ddtrace.go @@ -94,6 +94,25 @@ type SpanContext interface { ForeachBaggageItem(handler func(k, v string) bool) } +// SpanLink represents a reference to a span that exists outside of the trace. +// +//go:generate msgp -unexported -marshal=false -o=span_link_msgp.go -tests=false + +type SpanLink struct { + // TraceID represents the low 64 bits of the linked span's trace id. This field is required. + TraceID uint64 `msg:"trace_id" json:"trace_id"` + // TraceIDHigh represents the high 64 bits of the linked span's trace id. This field is only set if the linked span's trace id is 128 bits. + TraceIDHigh uint64 `msg:"trace_id_high,omitempty" json:"trace_id_high"` + // SpanID represents the linked span's span id. + SpanID uint64 `msg:"span_id" json:"span_id"` + // Attributes is a mapping of keys to string values. These values are used to add additional context to the span link. + Attributes map[string]string `msg:"attributes,omitempty" json:"attributes"` + // Tracestate is the tracestate of the linked span. This field is optional. + Tracestate string `msg:"tracestate,omitempty" json:"tracestate"` + // Flags represents the W3C trace flags of the linked span. This field is optional. + Flags uint32 `msg:"flags,omitempty" json:"flags"` +} + // StartSpanOption is a configuration option that can be used with a Tracer's StartSpan method. type StartSpanOption func(cfg *StartSpanConfig) @@ -144,6 +163,9 @@ type StartSpanConfig struct { // Context is the parent context where the span should be stored. Context context.Context + + // SpanLink represents a causal relationship between two spans. A span can have multiple links. + SpanLinks []SpanLink } // Logger implementations are able to log given messages that the tracer or profiler might output. diff --git a/ddtrace/mocktracer/mockspan.go b/ddtrace/mocktracer/mockspan.go index 506cb1145b..7609fa30c7 100644 --- a/ddtrace/mocktracer/mockspan.go +++ b/ddtrace/mocktracer/mockspan.go @@ -107,6 +107,7 @@ type mockspan struct { parentID uint64 context *spanContext tracer *mocktracer + links []ddtrace.SpanLink } // SetTag sets a given tag on the span. diff --git a/ddtrace/opentelemetry/span_test.go b/ddtrace/opentelemetry/span_test.go index 127f23fce7..221bce8be6 100644 --- a/ddtrace/opentelemetry/span_test.go +++ b/ddtrace/opentelemetry/span_test.go @@ -17,6 +17,7 @@ import ( "testing" "time" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/tracer" "gopkg.in/DataDog/dd-trace-go.v1/internal/httpmem" @@ -132,6 +133,63 @@ func TestSpanSetName(t *testing.T) { assert.Equal(strings.ToLower("NewName"), p[0]["name"]) } +func TestSpanLink(t *testing.T) { + assert := assert.New(t) + + _, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Use traceID, spanID, and traceflags that can be unmarshalled from unint64 to float64 without loss of precision + traceID, _ := oteltrace.TraceIDFromHex("00000000000001c8000000000000007b") + spanID, _ := oteltrace.SpanIDFromHex("000000000000000f") + traceState, _ := oteltrace.ParseTraceState("dd_origin=ci") + remoteSpanContext := oteltrace.NewSpanContext( + oteltrace.SpanContextConfig{ + TraceID: traceID, + SpanID: spanID, + TraceFlags: 0x0, + TraceState: traceState, + Remote: true, + }, + ) + + _, payloads, cleanup := mockTracerProvider(t) + tr := otel.Tracer("") + defer cleanup() + + // Create a span with a link to a remote span + _, decoratedSpan := tr.Start(context.Background(), "span_with_link", + oteltrace.WithLinks(oteltrace.Link{ + SpanContext: remoteSpanContext, + Attributes: []attribute.KeyValue{attribute.String("link.name", "alpha_transaction")}, + })) + decoratedSpan.End() + + // Flush span with link and unmarshal the payload to a map + tracer.Flush() + payload, err := waitForPayload(payloads) + if err != nil { + t.Fatalf(err.Error()) + } + assert.NotNil(payload) + // Ensure payload contains one trace and one span + assert.Len(payload[0], 1) + + // Convert the span_links field from type []map[string]interface{} to a struct + var spanLinks []ddtrace.SpanLink + spanLinkBytes, _ := json.Marshal(payload[0][0]["span_links"]) + json.Unmarshal(spanLinkBytes, &spanLinks) + assert.Len(spanLinks, 1) + + // Ensure the span link has the correct values + assert.Equal(spanLinks[0].TraceID, uint64(123)) + assert.Equal(spanLinks[0].TraceIDHigh, uint64(456)) + assert.Equal(spanLinks[0].SpanID, uint64(15)) + assert.Equal(spanLinks[0].Attributes, map[string]string{"link.name": "alpha_transaction"}) + assert.Equal(spanLinks[0].Tracestate, "dd_origin=ci") + assert.Equal(spanLinks[0].Flags, uint32(0x80000000)) +} + func TestSpanEnd(t *testing.T) { assert := assert.New(t) _, payloads, cleanup := mockTracerProvider(t) diff --git a/ddtrace/opentelemetry/tracer.go b/ddtrace/opentelemetry/tracer.go index 296c1a47a4..bad2194409 100644 --- a/ddtrace/opentelemetry/tracer.go +++ b/ddtrace/opentelemetry/tracer.go @@ -62,6 +62,36 @@ func (t *oteltracer) Start(ctx context.Context, spanName string, opts ...oteltra o(&cfg) } } + // Add span links to underlying Datadog span + if len(ssConfig.Links()) > 0 { + links := make([]ddtrace.SpanLink, 0, len(ssConfig.Links())) + for _, otelLink := range ssConfig.Links() { + var link ddtrace.SpanLink + + traceIDbytes := otelLink.SpanContext.TraceID() + link.TraceIDHigh = binary.BigEndian.Uint64(traceIDbytes[:8]) + link.TraceID = binary.BigEndian.Uint64(traceIDbytes[8:]) + + spanIDbytes := otelLink.SpanContext.SpanID() + link.SpanID = binary.BigEndian.Uint64(spanIDbytes[:]) + + link.Tracestate = otelLink.SpanContext.TraceState().String() + + if otelLink.SpanContext.IsSampled() { + link.Flags = uint32(0x80000001) + } else { + link.Flags = uint32(0x80000000) + } + + link.Attributes = make(map[string]string) + for _, attr := range otelLink.Attributes { + link.Attributes[string(attr.Key)] = attr.Value.Emit() + } + + links = append(links, link) + } + ddopts = append(ddopts, tracer.WithSpanLinks(links)) + } // Since there is no way to see if and how the span operation name was set, // we have to record the attributes locally. // The span operation name will be calculated when it's ended. diff --git a/ddtrace/span_link_msgp.go b/ddtrace/span_link_msgp.go new file mode 100644 index 0000000000..c2b90ec516 --- /dev/null +++ b/ddtrace/span_link_msgp.go @@ -0,0 +1,221 @@ +package ddtrace + +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *SpanLink) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "trace_id": + z.TraceID, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TraceID") + return + } + case "trace_id_high": + z.TraceIDHigh, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TraceIDHigh") + return + } + case "span_id": + z.SpanID, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "SpanID") + return + } + case "attributes": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Attributes") + return + } + if z.Attributes == nil { + z.Attributes = make(map[string]string, zb0002) + } else if len(z.Attributes) > 0 { + for key := range z.Attributes { + delete(z.Attributes, key) + } + } + for zb0002 > 0 { + zb0002-- + var za0001 string + var za0002 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Attributes") + return + } + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Attributes", za0001) + return + } + z.Attributes[za0001] = za0002 + } + case "tracestate": + z.Tracestate, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Tracestate") + return + } + case "flags": + z.Flags, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *SpanLink) EncodeMsg(en *msgp.Writer) (err error) { + // omitempty: check for empty values + zb0001Len := uint32(6) + var zb0001Mask uint8 /* 6 bits */ + if z.TraceIDHigh == 0 { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.Attributes == nil { + zb0001Len-- + zb0001Mask |= 0x8 + } + if z.Tracestate == "" { + zb0001Len-- + zb0001Mask |= 0x10 + } + if z.Flags == 0 { + zb0001Len-- + zb0001Mask |= 0x20 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } + // write "trace_id" + err = en.Append(0xa8, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.TraceID) + if err != nil { + err = msgp.WrapError(err, "TraceID") + return + } + if (zb0001Mask & 0x2) == 0 { // if not empty + // write "trace_id_high" + err = en.Append(0xad, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x5f, 0x68, 0x69, 0x67, 0x68) + if err != nil { + return + } + err = en.WriteUint64(z.TraceIDHigh) + if err != nil { + err = msgp.WrapError(err, "TraceIDHigh") + return + } + } + // write "span_id" + err = en.Append(0xa7, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.SpanID) + if err != nil { + err = msgp.WrapError(err, "SpanID") + return + } + if (zb0001Mask & 0x8) == 0 { // if not empty + // write "attributes" + err = en.Append(0xaa, 0x61, 0x74, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Attributes))) + if err != nil { + err = msgp.WrapError(err, "Attributes") + return + } + for za0001, za0002 := range z.Attributes { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Attributes") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Attributes", za0001) + return + } + } + } + if (zb0001Mask & 0x10) == 0 { // if not empty + // write "tracestate" + err = en.Append(0xaa, 0x74, 0x72, 0x61, 0x63, 0x65, 0x73, 0x74, 0x61, 0x74, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Tracestate) + if err != nil { + err = msgp.WrapError(err, "Tracestate") + return + } + } + if (zb0001Mask & 0x20) == 0 { // if not empty + // write "flags" + err = en.Append(0xa5, 0x66, 0x6c, 0x61, 0x67, 0x73) + if err != nil { + return + } + err = en.WriteUint32(z.Flags) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + } + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *SpanLink) Msgsize() (s int) { + s = 1 + 9 + msgp.Uint64Size + 14 + msgp.Uint64Size + 8 + msgp.Uint64Size + 11 + msgp.MapHeaderSize + if z.Attributes != nil { + for za0001, za0002 := range z.Attributes { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += 11 + msgp.StringPrefixSize + len(z.Tracestate) + 6 + msgp.Uint32Size + return +} diff --git a/ddtrace/tracer/option.go b/ddtrace/tracer/option.go index 635abb9bde..e51c76ad44 100644 --- a/ddtrace/tracer/option.go +++ b/ddtrace/tracer/option.go @@ -1141,6 +1141,13 @@ func SpanType(name string) StartSpanOption { return Tag(ext.SpanType, name) } +// WithSpanLinks sets span links on the started span. +func WithSpanLinks(links []ddtrace.SpanLink) StartSpanOption { + return func(cfg *ddtrace.StartSpanConfig) { + cfg.SpanLinks = append(cfg.SpanLinks, links...) + } +} + var measuredTag = Tag(keyMeasured, 1) // Measured marks this span to be measured for metrics and stats calculations. diff --git a/ddtrace/tracer/option_test.go b/ddtrace/tracer/option_test.go index 77e52858c8..767b25ef9b 100644 --- a/ddtrace/tracer/option_test.go +++ b/ddtrace/tracer/option_test.go @@ -24,6 +24,7 @@ import ( "testing" "time" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" "gopkg.in/DataDog/dd-trace-go.v1/ddtrace/ext" "gopkg.in/DataDog/dd-trace-go.v1/internal/globalconfig" "gopkg.in/DataDog/dd-trace-go.v1/internal/namingschema" @@ -883,6 +884,19 @@ func TestServiceName(t *testing.T) { }) } +func TestStartWithLink(t *testing.T) { + assert := assert.New(t) + + links := []ddtrace.SpanLink{{TraceID: 1, SpanID: 2}, {TraceID: 3, SpanID: 4}} + span := newTracer().StartSpan("test.request", WithSpanLinks(links)).(*span) + + assert.Len(span.SpanLinks, 2) + assert.Equal(span.SpanLinks[0].TraceID, uint64(1)) + assert.Equal(span.SpanLinks[0].SpanID, uint64(2)) + assert.Equal(span.SpanLinks[1].TraceID, uint64(3)) + assert.Equal(span.SpanLinks[1].SpanID, uint64(4)) +} + func TestTagSeparators(t *testing.T) { assert := assert.New(t) diff --git a/ddtrace/tracer/span.go b/ddtrace/tracer/span.go index 8ee8888765..b3aa4b2d7c 100644 --- a/ddtrace/tracer/span.go +++ b/ddtrace/tracer/span.go @@ -64,18 +64,19 @@ type errorConfig struct { type span struct { sync.RWMutex `msg:"-"` // all fields are protected by this RWMutex - Name string `msg:"name"` // operation name - Service string `msg:"service"` // service name (i.e. "grpc.server", "http.request") - Resource string `msg:"resource"` // resource name (i.e. "/user?id=123", "SELECT * FROM users") - Type string `msg:"type"` // protocol associated with the span (i.e. "web", "db", "cache") - Start int64 `msg:"start"` // span start time expressed in nanoseconds since epoch - Duration int64 `msg:"duration"` // duration of the span expressed in nanoseconds - Meta map[string]string `msg:"meta,omitempty"` // arbitrary map of metadata - Metrics map[string]float64 `msg:"metrics,omitempty"` // arbitrary map of numeric metrics - SpanID uint64 `msg:"span_id"` // identifier of this span - TraceID uint64 `msg:"trace_id"` // lower 64-bits of the root span identifier - ParentID uint64 `msg:"parent_id"` // identifier of the span's direct parent - Error int32 `msg:"error"` // error status of the span; 0 means no errors + Name string `msg:"name"` // operation name + Service string `msg:"service"` // service name (i.e. "grpc.server", "http.request") + Resource string `msg:"resource"` // resource name (i.e. "/user?id=123", "SELECT * FROM users") + Type string `msg:"type"` // protocol associated with the span (i.e. "web", "db", "cache") + Start int64 `msg:"start"` // span start time expressed in nanoseconds since epoch + Duration int64 `msg:"duration"` // duration of the span expressed in nanoseconds + Meta map[string]string `msg:"meta,omitempty"` // arbitrary map of metadata + Metrics map[string]float64 `msg:"metrics,omitempty"` // arbitrary map of numeric metrics + SpanID uint64 `msg:"span_id"` // identifier of this span + TraceID uint64 `msg:"trace_id"` // lower 64-bits of the root span identifier + ParentID uint64 `msg:"parent_id"` // identifier of the span's direct parent + Error int32 `msg:"error"` // error status of the span; 0 means no errors + SpanLinks []ddtrace.SpanLink `msg:"span_links"` // links to other spans goExecTraced bool `msg:"-"` noDebugStack bool `msg:"-"` // disables debug stack traces diff --git a/ddtrace/tracer/span_msgp.go b/ddtrace/tracer/span_msgp.go index 16bb758f8c..602ba239e9 100644 --- a/ddtrace/tracer/span_msgp.go +++ b/ddtrace/tracer/span_msgp.go @@ -1,18 +1,101 @@ -// Unless explicitly stated otherwise all files in this repository are licensed -// under the Apache License Version 2.0. -// This product includes software developed at Datadog (https://www.datadoghq.com/). -// Copyright 2016 Datadog, Inc. - package tracer -// NOTE: THIS FILE WAS PRODUCED BY THE -// MSGP CODE GENERATION TOOL (github.com/tinylib/msgp) -// DO NOT EDIT +// Code generated by github.com/tinylib/msgp DO NOT EDIT. import ( "github.com/tinylib/msgp/msgp" + "gopkg.in/DataDog/dd-trace-go.v1/ddtrace" ) +// DecodeMsg implements msgp.Decodable +func (z *errorConfig) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "noDebugStack": + z.noDebugStack, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "noDebugStack") + return + } + case "stackFrames": + z.stackFrames, err = dc.ReadUint() + if err != nil { + err = msgp.WrapError(err, "stackFrames") + return + } + case "stackSkip": + z.stackSkip, err = dc.ReadUint() + if err != nil { + err = msgp.WrapError(err, "stackSkip") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z errorConfig) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "noDebugStack" + err = en.Append(0x83, 0xac, 0x6e, 0x6f, 0x44, 0x65, 0x62, 0x75, 0x67, 0x53, 0x74, 0x61, 0x63, 0x6b) + if err != nil { + return + } + err = en.WriteBool(z.noDebugStack) + if err != nil { + err = msgp.WrapError(err, "noDebugStack") + return + } + // write "stackFrames" + err = en.Append(0xab, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x46, 0x72, 0x61, 0x6d, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteUint(z.stackFrames) + if err != nil { + err = msgp.WrapError(err, "stackFrames") + return + } + // write "stackSkip" + err = en.Append(0xa9, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x53, 0x6b, 0x69, 0x70) + if err != nil { + return + } + err = en.WriteUint(z.stackSkip) + if err != nil { + err = msgp.WrapError(err, "stackSkip") + return + } + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z errorConfig) Msgsize() (s int) { + s = 1 + 13 + msgp.BoolSize + 12 + msgp.UintSize + 10 + msgp.UintSize + return +} + // DecodeMsg implements msgp.Decodable func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var field []byte @@ -20,52 +103,61 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var zb0001 uint32 zb0001, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err) return } for zb0001 > 0 { zb0001-- field, err = dc.ReadMapKeyPtr() if err != nil { + err = msgp.WrapError(err) return } switch msgp.UnsafeString(field) { case "name": z.Name, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Name") return } case "service": z.Service, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Service") return } case "resource": z.Resource, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Resource") return } case "type": z.Type, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Type") return } case "start": z.Start, err = dc.ReadInt64() if err != nil { + err = msgp.WrapError(err, "Start") return } case "duration": z.Duration, err = dc.ReadInt64() if err != nil { + err = msgp.WrapError(err, "Duration") return } case "meta": var zb0002 uint32 zb0002, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err, "Meta") return } - if z.Meta == nil && zb0002 > 0 { + if z.Meta == nil { z.Meta = make(map[string]string, zb0002) } else if len(z.Meta) > 0 { for key := range z.Meta { @@ -78,10 +170,12 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var za0002 string za0001, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Meta") return } za0002, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Meta", za0001) return } z.Meta[za0001] = za0002 @@ -90,9 +184,10 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var zb0003 uint32 zb0003, err = dc.ReadMapHeader() if err != nil { + err = msgp.WrapError(err, "Metrics") return } - if z.Metrics == nil && zb0003 > 0 { + if z.Metrics == nil { z.Metrics = make(map[string]float64, zb0003) } else if len(z.Metrics) > 0 { for key := range z.Metrics { @@ -105,10 +200,12 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { var za0004 float64 za0003, err = dc.ReadString() if err != nil { + err = msgp.WrapError(err, "Metrics") return } za0004, err = dc.ReadFloat64() if err != nil { + err = msgp.WrapError(err, "Metrics", za0003) return } z.Metrics[za0003] = za0004 @@ -116,26 +213,50 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { case "span_id": z.SpanID, err = dc.ReadUint64() if err != nil { + err = msgp.WrapError(err, "SpanID") return } case "trace_id": z.TraceID, err = dc.ReadUint64() if err != nil { + err = msgp.WrapError(err, "TraceID") return } case "parent_id": z.ParentID, err = dc.ReadUint64() if err != nil { + err = msgp.WrapError(err, "ParentID") return } case "error": z.Error, err = dc.ReadInt32() if err != nil { + err = msgp.WrapError(err, "Error") return } + case "span_links": + var zb0004 uint32 + zb0004, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "SpanLinks") + return + } + if cap(z.SpanLinks) >= int(zb0004) { + z.SpanLinks = (z.SpanLinks)[:zb0004] + } else { + z.SpanLinks = make([]ddtrace.SpanLink, zb0004) + } + for za0005 := range z.SpanLinks { + err = z.SpanLinks[za0005].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "SpanLinks", za0005) + return + } + } default: err = dc.Skip() if err != nil { + err = msgp.WrapError(err) return } } @@ -145,14 +266,33 @@ func (z *span) DecodeMsg(dc *msgp.Reader) (err error) { // EncodeMsg implements msgp.Encodable func (z *span) EncodeMsg(en *msgp.Writer) (err error) { - // map header, size 12 + // omitempty: check for empty values + zb0001Len := uint32(13) + var zb0001Mask uint16 /* 13 bits */ + if z.Meta == nil { + zb0001Len-- + zb0001Mask |= 0x40 + } + if z.Metrics == nil { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if zb0001Len == 0 { + return + } // write "name" - err = en.Append(0x8c, 0xa4, 0x6e, 0x61, 0x6d, 0x65) + err = en.Append(0xa4, 0x6e, 0x61, 0x6d, 0x65) if err != nil { return } err = en.WriteString(z.Name) if err != nil { + err = msgp.WrapError(err, "Name") return } // write "service" @@ -162,6 +302,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteString(z.Service) if err != nil { + err = msgp.WrapError(err, "Service") return } // write "resource" @@ -171,6 +312,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteString(z.Resource) if err != nil { + err = msgp.WrapError(err, "Resource") return } // write "type" @@ -180,6 +322,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteString(z.Type) if err != nil { + err = msgp.WrapError(err, "Type") return } // write "start" @@ -189,6 +332,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteInt64(z.Start) if err != nil { + err = msgp.WrapError(err, "Start") return } // write "duration" @@ -198,45 +342,56 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteInt64(z.Duration) if err != nil { + err = msgp.WrapError(err, "Duration") return } - // write "meta" - err = en.Append(0xa4, 0x6d, 0x65, 0x74, 0x61) - if err != nil { - return - } - err = en.WriteMapHeader(uint32(len(z.Meta))) - if err != nil { - return - } - for za0001, za0002 := range z.Meta { - err = en.WriteString(za0001) + if (zb0001Mask & 0x40) == 0 { // if not empty + // write "meta" + err = en.Append(0xa4, 0x6d, 0x65, 0x74, 0x61) if err != nil { return } - err = en.WriteString(za0002) + err = en.WriteMapHeader(uint32(len(z.Meta))) if err != nil { + err = msgp.WrapError(err, "Meta") return } + for za0001, za0002 := range z.Meta { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Meta", za0001) + return + } + } } - // write "metrics" - err = en.Append(0xa7, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) - if err != nil { - return - } - err = en.WriteMapHeader(uint32(len(z.Metrics))) - if err != nil { - return - } - for za0003, za0004 := range z.Metrics { - err = en.WriteString(za0003) + if (zb0001Mask & 0x80) == 0 { // if not empty + // write "metrics" + err = en.Append(0xa7, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) if err != nil { return } - err = en.WriteFloat64(za0004) + err = en.WriteMapHeader(uint32(len(z.Metrics))) if err != nil { + err = msgp.WrapError(err, "Metrics") return } + for za0003, za0004 := range z.Metrics { + err = en.WriteString(za0003) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + err = en.WriteFloat64(za0004) + if err != nil { + err = msgp.WrapError(err, "Metrics", za0003) + return + } + } } // write "span_id" err = en.Append(0xa7, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x69, 0x64) @@ -245,6 +400,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteUint64(z.SpanID) if err != nil { + err = msgp.WrapError(err, "SpanID") return } // write "trace_id" @@ -254,6 +410,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteUint64(z.TraceID) if err != nil { + err = msgp.WrapError(err, "TraceID") return } // write "parent_id" @@ -263,6 +420,7 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteUint64(z.ParentID) if err != nil { + err = msgp.WrapError(err, "ParentID") return } // write "error" @@ -272,8 +430,26 @@ func (z *span) EncodeMsg(en *msgp.Writer) (err error) { } err = en.WriteInt32(z.Error) if err != nil { + err = msgp.WrapError(err, "Error") + return + } + // write "span_links" + err = en.Append(0xaa, 0x73, 0x70, 0x61, 0x6e, 0x5f, 0x6c, 0x69, 0x6e, 0x6b, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.SpanLinks))) + if err != nil { + err = msgp.WrapError(err, "SpanLinks") return } + for za0005 := range z.SpanLinks { + err = z.SpanLinks[za0005].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "SpanLinks", za0005) + return + } + } return } @@ -293,7 +469,10 @@ func (z *span) Msgsize() (s int) { s += msgp.StringPrefixSize + len(za0003) + msgp.Float64Size } } - s += 8 + msgp.Uint64Size + 9 + msgp.Uint64Size + 10 + msgp.Uint64Size + 6 + msgp.Int32Size + s += 8 + msgp.Uint64Size + 9 + msgp.Uint64Size + 10 + msgp.Uint64Size + 6 + msgp.Int32Size + 11 + msgp.ArrayHeaderSize + for za0005 := range z.SpanLinks { + s += z.SpanLinks[za0005].Msgsize() + } return } @@ -302,6 +481,7 @@ func (z *spanList) DecodeMsg(dc *msgp.Reader) (err error) { var zb0002 uint32 zb0002, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err) return } if cap((*z)) >= int(zb0002) { @@ -313,6 +493,7 @@ func (z *spanList) DecodeMsg(dc *msgp.Reader) (err error) { if dc.IsNil() { err = dc.ReadNil() if err != nil { + err = msgp.WrapError(err, zb0001) return } (*z)[zb0001] = nil @@ -322,6 +503,7 @@ func (z *spanList) DecodeMsg(dc *msgp.Reader) (err error) { } err = (*z)[zb0001].DecodeMsg(dc) if err != nil { + err = msgp.WrapError(err, zb0001) return } } @@ -333,6 +515,7 @@ func (z *spanList) DecodeMsg(dc *msgp.Reader) (err error) { func (z spanList) EncodeMsg(en *msgp.Writer) (err error) { err = en.WriteArrayHeader(uint32(len(z))) if err != nil { + err = msgp.WrapError(err) return } for zb0003 := range z { @@ -344,6 +527,7 @@ func (z spanList) EncodeMsg(en *msgp.Writer) (err error) { } else { err = z[zb0003].EncodeMsg(en) if err != nil { + err = msgp.WrapError(err, zb0003) return } } @@ -369,6 +553,7 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { var zb0003 uint32 zb0003, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err) return } if cap((*z)) >= int(zb0003) { @@ -380,6 +565,7 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { var zb0004 uint32 zb0004, err = dc.ReadArrayHeader() if err != nil { + err = msgp.WrapError(err, zb0001) return } if cap((*z)[zb0001]) >= int(zb0004) { @@ -391,6 +577,7 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { if dc.IsNil() { err = dc.ReadNil() if err != nil { + err = msgp.WrapError(err, zb0001, zb0002) return } (*z)[zb0001][zb0002] = nil @@ -400,6 +587,7 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { } err = (*z)[zb0001][zb0002].DecodeMsg(dc) if err != nil { + err = msgp.WrapError(err, zb0001, zb0002) return } } @@ -412,11 +600,13 @@ func (z *spanLists) DecodeMsg(dc *msgp.Reader) (err error) { func (z spanLists) EncodeMsg(en *msgp.Writer) (err error) { err = en.WriteArrayHeader(uint32(len(z))) if err != nil { + err = msgp.WrapError(err) return } for zb0005 := range z { err = en.WriteArrayHeader(uint32(len(z[zb0005]))) if err != nil { + err = msgp.WrapError(err, zb0005) return } for zb0006 := range z[zb0005] { @@ -428,6 +618,7 @@ func (z spanLists) EncodeMsg(en *msgp.Writer) (err error) { } else { err = z[zb0005][zb0006].EncodeMsg(en) if err != nil { + err = msgp.WrapError(err, zb0005, zb0006) return } } diff --git a/ddtrace/tracer/tracer.go b/ddtrace/tracer/tracer.go index ddef5ffa6f..1193de56bc 100644 --- a/ddtrace/tracer/tracer.go +++ b/ddtrace/tracer/tracer.go @@ -503,6 +503,10 @@ func (t *tracer) StartSpan(operationName string, options ...ddtrace.StartSpanOpt Start: startTime, noDebugStack: t.config.noDebugStack, } + for _, link := range opts.SpanLinks { + span.SpanLinks = append(span.SpanLinks, link) + } + if t.config.hostname != "" { span.setMeta(keyHostname, t.config.hostname) } diff --git a/ddtrace/tracer/writer_test.go b/ddtrace/tracer/writer_test.go index dd6cc61d0a..6c4f9de7aa 100644 --- a/ddtrace/tracer/writer_test.go +++ b/ddtrace/tracer/writer_test.go @@ -373,7 +373,7 @@ func TestTraceWriterFlushRetries(t *testing.T) { sentCounts := map[string]int64{ "datadog.tracer.decode_error": 1, - "datadog.tracer.flush_bytes": 172, + "datadog.tracer.flush_bytes": 184, "datadog.tracer.flush_traces": 1, } droppedCounts := map[string]int64{ From 7595b2d35406693ebd3a0a2b8acc80f99d1d7268 Mon Sep 17 00:00:00 2001 From: Julio Guerra Date: Mon, 5 Feb 2024 09:57:16 +0100 Subject: [PATCH 13/13] ci/appsec: refresher (#2537) --- .../apps/appsec-test-contrib-submodules.sh | 11 +++++- .github/workflows/appsec.yml | 38 +++++++------------ 2 files changed, 24 insertions(+), 25 deletions(-) diff --git a/.github/workflows/apps/appsec-test-contrib-submodules.sh b/.github/workflows/apps/appsec-test-contrib-submodules.sh index 8dbf577e48..bca4177520 100755 --- a/.github/workflows/apps/appsec-test-contrib-submodules.sh +++ b/.github/workflows/apps/appsec-test-contrib-submodules.sh @@ -41,7 +41,16 @@ fi $runner "$JUNIT_REPORT.xml" "." ./appsec/... ./internal/appsec/... -SCOPES=("gin-gonic/gin" "google.golang.org/grpc" "net/http" "gorilla/mux" "go-chi/chi" "go-chi/chi.v5" "labstack/echo.v4") +SCOPES=( + "gin-gonic/gin" \ + "google.golang.org/grpc" \ + "net/http" "gorilla/mux" \ + "go-chi/chi" "go-chi/chi.v5" \ + "labstack/echo.v4" \ + "99designs/gqlgen" \ + "graphql-go/graphql" \ + "graph-gophers/graphql-go" +) for SCOPE in "${SCOPES[@]}"; do contrib=$(basename "$SCOPE") echo "Running appsec tests for contrib/$SCOPE" diff --git a/.github/workflows/appsec.yml b/.github/workflows/appsec.yml index 71b07825e7..c42ac619a1 100644 --- a/.github/workflows/appsec.yml +++ b/.github/workflows/appsec.yml @@ -30,7 +30,7 @@ jobs: native: strategy: matrix: - runs-on: [ macos-13, macos-12, macos-11, ubuntu-22.04, ubuntu-20.04 ] + runs-on: [ macos-14, macos-13, macos-12, ubuntu-22.04, ubuntu-20.04 ] go-version: [ "1.21", "1.20", "1.19" ] cgo_enabled: [ "0", "1" ] # test it compiles with and without cgo appsec_enabled: # test it compiles with and without appsec enabled @@ -44,22 +44,16 @@ jobs: cgocheck: # 1.21 deprecates the GODEBUG=cgocheck=2 value, replacing it with GOEXPERIMENT=cgocheck2 GOEXPERIMENT=cgocheck2 fail-fast: false + name: native ${{ toJSON(matrix) }} runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.ref }} - - uses: actions/setup-go@v3 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go-version }} - - name: Go modules cache - uses: actions/cache@v3 - with: - path: ~/go/pkg/mod - key: go-pkg-mod-${{ hashFiles('**/go.sum') }} - restore-keys: go-pkg-mod- - - name: go test shell: bash run: | @@ -74,7 +68,7 @@ jobs: files: ${{ env.JUNIT_REPORT }}*.xml tags: go:${{ matrix.go-version }},arch:${{ runner.arch }},os:${{ runner.os }} - # Tests cases were appsec end up being disable + # Tests cases were appsec end up being disabled waf-disabled: strategy: fail-fast: false @@ -87,20 +81,14 @@ jobs: include: - runs-on: windows-latest go-args: "" + name: disabled ${{ toJSON(matrix) }} runs-on: ${{ matrix.runs-on }} steps: - - uses: actions/checkout@v3 - - uses: actions/setup-go@v3 # TODO: rely on v4 which now provides github caching by default + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: stable - - name: Go modules cache - uses: actions/cache@v3 - with: - path: ~/go/pkg/mod - key: go-pkg-mod-${{ hashFiles('**/go.sum') }} - restore-keys: go-pkg-mod- - - name: go test shell: bash run: | @@ -118,6 +106,7 @@ jobs: # Same tests but on the official golang container for linux golang-linux-container: + name: golang-containers ${{ toJSON(matrix) }} runs-on: ubuntu-latest container: image: golang:${{ matrix.go-version }}-${{ matrix.distribution }} @@ -138,7 +127,7 @@ jobs: fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.ref }} # Install gcc and the libc headers on alpine images @@ -169,6 +158,7 @@ jobs: linux-arm64: runs-on: ubuntu-latest + name: linux/arm64 ${{ toJSON(matrix) }} strategy: matrix: cgo_enabled: # test it compiles with and without the cgo @@ -179,17 +169,17 @@ jobs: - false fail-fast: false steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: ref: ${{ inputs.ref || github.ref }} - name: Go modules cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/go/pkg/mod key: go-pkg-mod-${{ hashFiles('**/go.sum') }} restore-keys: go-pkg-mod- - name: Set up QEMU - uses: docker/setup-qemu-action@v2 + uses: docker/setup-qemu-action@v3 with: platforms: arm64 - run: |