From 0576d9f1b111f59ed1ac688d287abae4bddbf980 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 19 Mar 2025 15:44:52 +0000 Subject: [PATCH 01/11] Allow disabling the lambda integration --- .../Lambda/HandlerWrapperSetHandlerIntegration.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs index 8f3bee451276..ce573fd8ab57 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs @@ -68,6 +68,12 @@ private static string ConvertPayloadStream(Stream payloadStream) public object OnDelegateBegin(object sender, ref TArg1 arg) { + if (!Tracer.Instance.Settings.IsIntegrationEnabled(IntegrationId.AwsLambda)) + { + // integration disabled, don't create a scope, skip this trace + return CallTargetState.GetDefault(); + } + LambdaCommon.Log("DelegateWrapper Running OnDelegateBegin"); Scope scope; @@ -101,6 +107,12 @@ public TReturn OnDelegateEnd(object sender, TReturn returnValue, Except /// public async Task OnDelegateEndAsync(object sender, TInnerReturn returnValue, Exception exception, object state) { + if (!Tracer.Instance.Settings.IsIntegrationEnabled(IntegrationId.AwsLambda)) + { + // integration disabled, don't create a scope, skip this trace + return returnValue; + } + LambdaCommon.Log("DelegateWrapper Running OnDelegateEndAsync"); try { From b29792881af43a3d6ccbdb2612aa8e7068ef167b Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 19 Mar 2025 15:57:45 +0000 Subject: [PATCH 02/11] Extract common parts from test --- .../AWS/AwsLambdaTests.cs | 60 +++++++++++-------- 1 file changed, 34 insertions(+), 26 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 49e8308ac85d..83e256c302df 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -9,6 +9,7 @@ using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; +using Datadog.Trace.Configuration; using Datadog.Trace.ExtensionMethods; using Datadog.Trace.TestHelpers; using FluentAssertions; @@ -23,6 +24,16 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.AWS [Trait("RequiresDockerDependency", "true")] public class AwsLambdaTests : TestHelper { + private const int ExpectedRequests = + 9 // param tests + + 3 // param with context + + 5 // base instrumentation + + 6 // other parameter types + + 3 // throwing (manual only) + + 3 // throwing with context + + 8 // Generic types + + 1; // Toplevel Statement + private static readonly Regex StackRegex = new(@"( error.stack:)(?:\n|\r){1,2}(?:[^,]*(?:\n|\r){1,2})+.*(,(?:\r|\n){1,2})"); private static readonly Regex ErrorMsgRegex = new(@"( error.msg:).*(,(\r|\n){1,2})"); @@ -49,39 +60,17 @@ public async Task SubmitsTraces() agent.Output = Output; using (await RunSampleAndWaitForExit(agent)) { - var requests = 9 // param tests - + 3 // param with context - + 5 // base instrumentation - + 6 // other parameter types - + 3 // throwing (manual only) - + 3 // throwing with context - + 8 // Generic types - + 1; // Toplevel Statement - - var expectedSpans = requests * 2; // we manually instrument each request too + // we manually instrument each request too + var expectedSpans = ExpectedRequests * 2; var spans = agent.WaitForSpans(expectedSpans, 15_000).ToArray(); // Verify all the "with context" end invocations have a corresponding start context using var s = new AssertionScope(); - var startWithContext = extensionWithContext.StartInvocations.ToList(); - var endWithContext = extensionWithContext.EndInvocations.ToList(); - startWithContext.Should().OnlyContain(x => x.TraceId.HasValue); - endWithContext.Should().OnlyContain(x => x.TraceId.HasValue); - var contextPairs = extensionWithContext.EndInvocations - .Select(x => (start: startWithContext.SingleOrDefault(y => y.TraceId == x.TraceId), end: x)) - .ToList(); - contextPairs.Should().OnlyContain(x => x.start != null); - var withContextSpans = contextPairs.Select(x => ToMockSpan(x.end, x.start.Created)); - - // We can't match no-context start invocations with the end invocations, so just use the end ones - var noContextSpans = extensionNoContext.EndInvocations.Select(x => ToMockSpan(x, startTime: null)); + var allExtensionSpans = GetExtensionSpans(extensionWithContext, extensionNoContext); // Create the complete traces - var allSpans = withContextSpans - .Concat(noContextSpans) - .Concat(spans) - .ToList(); + var allSpans = allExtensionSpans.Concat(spans).ToList(); var settings = VerifyHelper.GetSpanVerifierSettings(); @@ -94,6 +83,25 @@ await VerifyHelper.VerifySpans(allSpans, settings) } } + private static IEnumerable GetExtensionSpans(MockLambdaExtension extensionWithContext, MockLambdaExtension extensionNoContext) + { + var startWithContext = extensionWithContext.StartInvocations.ToList(); + var endWithContext = extensionWithContext.EndInvocations.ToList(); + startWithContext.Should().OnlyContain(x => x.TraceId.HasValue); + endWithContext.Should().OnlyContain(x => x.TraceId.HasValue); + var contextPairs = extensionWithContext.EndInvocations + .Select(x => (start: startWithContext.SingleOrDefault(y => y.TraceId == x.TraceId), end: x)) + .ToList(); + contextPairs.Should().OnlyContain(x => x.start != null); + var withContextSpans = contextPairs.Select(x => ToMockSpan(x.end, x.start.Created)); + + // We can't match no-context start invocations with the end invocations, so just use the end ones + var noContextSpans = extensionNoContext.EndInvocations.Select(x => ToMockSpan(x, startTime: null)); + + var allExtensionSpans = withContextSpans.Concat(noContextSpans); + return allExtensionSpans; + } + private static MockSpan ToMockSpan(MockLambdaExtension.EndExtensionRequest endInvocation, DateTimeOffset? startTime) { var start = startTime ?? endInvocation.Created.AddMilliseconds(100); From 38edc61bf1d54bf9e2fce1cde39b39490e65e577 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 19 Mar 2025 15:58:09 +0000 Subject: [PATCH 03/11] Add additional test --- .../AWS/AwsLambdaTests.cs | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 83e256c302df..9953fd022c16 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -83,6 +83,43 @@ await VerifyHelper.VerifySpans(allSpans, settings) } } + [SkippableFact] + [Trait("Category", "ArmUnsupported")] + [Trait("Category", "Lambda")] + public async Task IntegrationDisabled() + { + // See documentation at docs/development/Serverless.md for examples and diagrams + if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("IsAlpine"))) + { + Output.WriteLine("Skipping"); + return; + } + + const string expectedOperationName = "lambda.invocation"; + + SetEnvironmentVariable($"DD_TRACE_{nameof(IntegrationId.AwsLambda)}_ENABLED", "false"); + + using var extensionWithContext = new MockLambdaExtension(shouldSendContext: true, port: 9004, Output); + using var extensionNoContext = new MockLambdaExtension(shouldSendContext: false, port: 9003, Output); + using var agent = EnvironmentHelper.GetMockAgent(fixedPort: 5002); + agent.Output = Output; + using (await RunSampleAndWaitForExit(agent)) + { + // we manually instrument each request + we have http client spans + var expectedSpans = ExpectedRequests * 2; + + var agentSpans = agent.WaitForSpans(expectedSpans, 15_000).ToArray(); + + using var s = new AssertionScope(); + var allExtensionSpans = GetExtensionSpans(extensionWithContext, extensionNoContext); + + // Create the complete traces + var allSpans = allExtensionSpans.Concat(agentSpans).ToList(); + + allSpans.Should().NotContain(x => x.Name == expectedOperationName); + } + } + private static IEnumerable GetExtensionSpans(MockLambdaExtension extensionWithContext, MockLambdaExtension extensionNoContext) { var startWithContext = extensionWithContext.StartInvocations.ToList(); From 5629def24b553f1f0ca7d7d3a42df34c60d5c545 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 19 Mar 2025 18:03:15 +0000 Subject: [PATCH 04/11] Revert "Add additional test" This reverts commit ac3d3931196672a33658a1378d216f7c50f3d4e0. --- .../AWS/AwsLambdaTests.cs | 37 ------------------- 1 file changed, 37 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 9953fd022c16..83e256c302df 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -83,43 +83,6 @@ await VerifyHelper.VerifySpans(allSpans, settings) } } - [SkippableFact] - [Trait("Category", "ArmUnsupported")] - [Trait("Category", "Lambda")] - public async Task IntegrationDisabled() - { - // See documentation at docs/development/Serverless.md for examples and diagrams - if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("IsAlpine"))) - { - Output.WriteLine("Skipping"); - return; - } - - const string expectedOperationName = "lambda.invocation"; - - SetEnvironmentVariable($"DD_TRACE_{nameof(IntegrationId.AwsLambda)}_ENABLED", "false"); - - using var extensionWithContext = new MockLambdaExtension(shouldSendContext: true, port: 9004, Output); - using var extensionNoContext = new MockLambdaExtension(shouldSendContext: false, port: 9003, Output); - using var agent = EnvironmentHelper.GetMockAgent(fixedPort: 5002); - agent.Output = Output; - using (await RunSampleAndWaitForExit(agent)) - { - // we manually instrument each request + we have http client spans - var expectedSpans = ExpectedRequests * 2; - - var agentSpans = agent.WaitForSpans(expectedSpans, 15_000).ToArray(); - - using var s = new AssertionScope(); - var allExtensionSpans = GetExtensionSpans(extensionWithContext, extensionNoContext); - - // Create the complete traces - var allSpans = allExtensionSpans.Concat(agentSpans).ToList(); - - allSpans.Should().NotContain(x => x.Name == expectedOperationName); - } - } - private static IEnumerable GetExtensionSpans(MockLambdaExtension extensionWithContext, MockLambdaExtension extensionNoContext) { var startWithContext = extensionWithContext.StartInvocations.ToList(); From e69c309ffde6ed190952851846624af449b27062 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 19 Mar 2025 18:03:18 +0000 Subject: [PATCH 05/11] Revert "Extract common parts from test" This reverts commit 14f2911c8ce6f3e29161ce0f5f4fa9219daa14eb. --- .../AWS/AwsLambdaTests.cs | 60 ++++++++----------- 1 file changed, 26 insertions(+), 34 deletions(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 83e256c302df..49e8308ac85d 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -9,7 +9,6 @@ using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; -using Datadog.Trace.Configuration; using Datadog.Trace.ExtensionMethods; using Datadog.Trace.TestHelpers; using FluentAssertions; @@ -24,16 +23,6 @@ namespace Datadog.Trace.ClrProfiler.IntegrationTests.AWS [Trait("RequiresDockerDependency", "true")] public class AwsLambdaTests : TestHelper { - private const int ExpectedRequests = - 9 // param tests - + 3 // param with context - + 5 // base instrumentation - + 6 // other parameter types - + 3 // throwing (manual only) - + 3 // throwing with context - + 8 // Generic types - + 1; // Toplevel Statement - private static readonly Regex StackRegex = new(@"( error.stack:)(?:\n|\r){1,2}(?:[^,]*(?:\n|\r){1,2})+.*(,(?:\r|\n){1,2})"); private static readonly Regex ErrorMsgRegex = new(@"( error.msg:).*(,(\r|\n){1,2})"); @@ -60,17 +49,39 @@ public async Task SubmitsTraces() agent.Output = Output; using (await RunSampleAndWaitForExit(agent)) { - // we manually instrument each request too - var expectedSpans = ExpectedRequests * 2; + var requests = 9 // param tests + + 3 // param with context + + 5 // base instrumentation + + 6 // other parameter types + + 3 // throwing (manual only) + + 3 // throwing with context + + 8 // Generic types + + 1; // Toplevel Statement + + var expectedSpans = requests * 2; // we manually instrument each request too var spans = agent.WaitForSpans(expectedSpans, 15_000).ToArray(); // Verify all the "with context" end invocations have a corresponding start context using var s = new AssertionScope(); - var allExtensionSpans = GetExtensionSpans(extensionWithContext, extensionNoContext); + var startWithContext = extensionWithContext.StartInvocations.ToList(); + var endWithContext = extensionWithContext.EndInvocations.ToList(); + startWithContext.Should().OnlyContain(x => x.TraceId.HasValue); + endWithContext.Should().OnlyContain(x => x.TraceId.HasValue); + var contextPairs = extensionWithContext.EndInvocations + .Select(x => (start: startWithContext.SingleOrDefault(y => y.TraceId == x.TraceId), end: x)) + .ToList(); + contextPairs.Should().OnlyContain(x => x.start != null); + var withContextSpans = contextPairs.Select(x => ToMockSpan(x.end, x.start.Created)); + + // We can't match no-context start invocations with the end invocations, so just use the end ones + var noContextSpans = extensionNoContext.EndInvocations.Select(x => ToMockSpan(x, startTime: null)); // Create the complete traces - var allSpans = allExtensionSpans.Concat(spans).ToList(); + var allSpans = withContextSpans + .Concat(noContextSpans) + .Concat(spans) + .ToList(); var settings = VerifyHelper.GetSpanVerifierSettings(); @@ -83,25 +94,6 @@ await VerifyHelper.VerifySpans(allSpans, settings) } } - private static IEnumerable GetExtensionSpans(MockLambdaExtension extensionWithContext, MockLambdaExtension extensionNoContext) - { - var startWithContext = extensionWithContext.StartInvocations.ToList(); - var endWithContext = extensionWithContext.EndInvocations.ToList(); - startWithContext.Should().OnlyContain(x => x.TraceId.HasValue); - endWithContext.Should().OnlyContain(x => x.TraceId.HasValue); - var contextPairs = extensionWithContext.EndInvocations - .Select(x => (start: startWithContext.SingleOrDefault(y => y.TraceId == x.TraceId), end: x)) - .ToList(); - contextPairs.Should().OnlyContain(x => x.start != null); - var withContextSpans = contextPairs.Select(x => ToMockSpan(x.end, x.start.Created)); - - // We can't match no-context start invocations with the end invocations, so just use the end ones - var noContextSpans = extensionNoContext.EndInvocations.Select(x => ToMockSpan(x, startTime: null)); - - var allExtensionSpans = withContextSpans.Concat(noContextSpans); - return allExtensionSpans; - } - private static MockSpan ToMockSpan(MockLambdaExtension.EndExtensionRequest endInvocation, DateTimeOffset? startTime) { var start = startTime ?? endInvocation.Created.AddMilliseconds(100); From 6a2b42f32636c22e065e5d7ffe50801c043ba4f9 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Wed, 19 Mar 2025 18:15:11 +0000 Subject: [PATCH 06/11] Add scenarios where the app is disabled --- docker-compose.serverless.yml | 57 +++++++++++++++++++ .../AWS/AwsLambdaTests.cs | 3 +- .../Samples.AWS.Lambda/Program.cs | 8 ++- 3 files changed, 66 insertions(+), 2 deletions(-) diff --git a/docker-compose.serverless.yml b/docker-compose.serverless.yml index f7ae65285da4..c88cbbb03b34 100644 --- a/docker-compose.serverless.yml +++ b/docker-compose.serverless.yml @@ -41,6 +41,8 @@ services: - serverless-lambda-generic-complex - serverless-lambda-generic-complex-nested - serverless-lambda-toplevel-statement + - serverless-lambda-disabled-no-param-sync + - serverless-lambda-disabled-toplevel-statement - serverless-dummy-api environment: - AWS_LAMBDA_ENDPOINT_NO_PARAM_SYNC=http://serverless-lambda-no-param-sync:8080 @@ -81,6 +83,8 @@ services: - AWS_LAMBDA_ENDPOINT_GENERICBASE_COMPLEX=http://serverless-lambda-generic-complex:8080 - AWS_LAMBDA_ENDPOINT_GENERICBASE_COMPLEX_NESTED=http://serverless-lambda-generic-complex-nested:8080 - AWS_LAMBDA_ENDPOINT_TOPLEVEL_STATEMENT=http://serverless-lambda-toplevel-statement:8080 + - AWS_LAMBDA_ENDPOINT_DISABLED_NO_PARAM_SYNC=http://serverless-lambda-disabled-no-param-sync:8080 + - AWS_LAMBDA_ENDPOINT_DISABLED_TOPLEVEL_STATEMENT=http://serverless-lambda-disabled-toplevel-statement:8080 - DUMMY_API_HOST=http://serverless-dummy-api:9005 StartDependencies.Serverless: @@ -124,6 +128,8 @@ services: - serverless-lambda-generic-complex - serverless-lambda-generic-complex-nested - serverless-lambda-toplevel-statement + - serverless-lambda-disabled-no-param-sync + - serverless-lambda-disabled-toplevel-statement - serverless-dummy-api environment: - TIMEOUT_LENGTH=120 @@ -166,6 +172,8 @@ services: serverless-lambda-generic-complex:8080 serverless-lambda-generic-complex-nested:8080 serverless-lambda-toplevel-statement:8080 + serverless-lambda-disabled-no-param-sync:8080 + serverless-lambda-disabled-toplevel-statement:8080 serverless-dummy-api:9005 serverless-lambda-no-param-sync: @@ -1042,6 +1050,55 @@ services: - ./:/project - ./artifacts/build_data/logs/serverless-lambda-toplevel-statement:/var/log/datadog/dotnet + serverless-lambda-disabled-no-param-sync: + build: + context: ./artifacts/bin/Samples.AWS.Lambda + dockerfile: serverless.lambda.dockerfile + args: + - lambdaBaseImage=${lambdaBaseImage:-public.ecr.aws/lambda/dotnet:6} + - framework=${framework:-net6.0} + depends_on: + - serverless-dummy-api + command: "Samples.AWS.Lambda::Samples.AWS.Lambda.Function::HandlerNoParamSync" + image: dd-trace-dotnet/serverless-lambda-disabled-no-param-sync + environment: + - DD_TRACE_AGENT_URL=http://integrationtests:5002 + - _DD_EXTENSION_ENDPOINT=http://integrationtests:9003 + - DUMMY_API_HOST=http://serverless-dummy-api:9005 + - DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 + - DD_TRACE_DEBUG=1 + - DD_TRACE_AwsLambda_ENABLED=0 + ports: + - "8080" + volumes: + - ./:/project + - ./artifacts/build_data/logs/serverless-lambda-no-param-sync:/var/log/datadog/dotnet + + serverless-lambda-disabled-toplevel-statement: + build: + context: ./artifacts/bin/Samples.Amazon.Lambda.RuntimeSupport + dockerfile: serverless.lambda.dockerfile + args: + - lambdaBaseImage=${lambdaBaseImage:-public.ecr.aws/lambda/dotnet:6} + - framework=${framework:-net6.0} + depends_on: + - serverless-dummy-api + command: "Samples.Amazon.Lambda.RuntimeSupport" + image: dd-trace-dotnet/serverless-lambda-disabled-toplevel-statement + environment: + - DD_TRACE_AGENT_URL=http://integrationtests:5002 + - _DD_EXTENSION_ENDPOINT=http://integrationtests:9003 + - DUMMY_API_HOST=http://serverless-dummy-api:9005 + - DD_INSTRUMENTATION_TELEMETRY_ENABLED=0 + - DD_TRACE_ENABLED=0 + - DD_TRACE_DEBUG=1 + - DD_TRACE_AwsLambda_ENABLED=0 + ports: + - "8080" + volumes: + - ./:/project + - ./artifacts/build_data/logs/serverless-lambda-toplevel-statement:/var/log/datadog/dotnet + # The serverless function calls this API, which always returns 200 OK serverless-dummy-api: image: andrewlock/ok-api:latest diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 49e8308ac85d..68eb43fe8562 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -56,7 +56,8 @@ public async Task SubmitsTraces() + 3 // throwing (manual only) + 3 // throwing with context + 8 // Generic types - + 1; // Toplevel Statement + + 1 // Toplevel Statement + + 2; // Disabled var expectedSpans = requests * 2; // we manually instrument each request too diff --git a/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs b/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs index 6a901a5af2db..88cc7f031f6c 100644 --- a/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs +++ b/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs @@ -121,7 +121,13 @@ private static async Task Main(string[] args) // Toplevel Statements Thread.Sleep(1000); await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_TOPLEVEL_STATEMENT")); - + + // Disabled + Thread.Sleep(1000); + await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_DISABLED_NO_PARAM_SYNC")); + Thread.Sleep(1000); + await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_DISABLED_TOPLEVEL_STATEMENT")); + static async Task Post(string url) { HttpClient client = new HttpClient(); From cffce89a327079c8325a7eed8c88a87b91dce589 Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 20 Mar 2025 11:22:43 +0000 Subject: [PATCH 07/11] Fix count --- .../AWS/AwsLambdaTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index 68eb43fe8562..ade2908d821a 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -59,7 +59,9 @@ public async Task SubmitsTraces() + 1 // Toplevel Statement + 2; // Disabled - var expectedSpans = requests * 2; // we manually instrument each request too + // We have HTTP Request + Manual span + Lambda span for each request + // -2 for the "lambda integration disabled" cases + var expectedSpans = (requests * 2) - 2; var spans = agent.WaitForSpans(expectedSpans, 15_000).ToArray(); From cd6c8365b6bf086e7a7d9b9440df2a374502338d Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 20 Mar 2025 14:38:34 +0000 Subject: [PATCH 08/11] Try to avoid some flake --- .../snapshots/AwsLambdaTests.verified.txt | 40 +++++++++++++++++++ .../Samples.AWS.Lambda/Program.cs | 14 ++++--- 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/tracer/test/snapshots/AwsLambdaTests.verified.txt b/tracer/test/snapshots/AwsLambdaTests.verified.txt index 373d510ed821..baeb490d3fd2 100644 --- a/tracer/test/snapshots/AwsLambdaTests.verified.txt +++ b/tracer/test/snapshots/AwsLambdaTests.verified.txt @@ -1978,5 +1978,45 @@ Metrics: { _dd.top_level: 1.0 } + }, + { + TraceId: Id_153, + SpanId: Id_154, + Name: manual.HandlerNoParamSync, + Resource: manual.HandlerNoParamSync, + Service: Samples.AWS.Lambda, + Tags: { + language: dotnet, + runtime-id: Guid_39 + }, + Metrics: { + process_id: 0, + _dd.top_level: 1.0, + _dd.tracer_kr: 1.0, + _sampling_priority_v1: 1.0 + } + }, + { + TraceId: Id_153, + SpanId: Id_155, + Name: http.request, + Resource: GET serverless-dummy-api:9005/function/HandlerNoParamSync, + Service: Samples.AWS.Lambda-http-client, + Type: http, + ParentId: Id_154, + Tags: { + component: WebRequest, + http.method: GET, + http.status_code: 200, + http.url: http://serverless-dummy-api:9005/function/HandlerNoParamSync, + language: dotnet, + out.host: serverless-dummy-api, + runtime-id: Guid_39, + span.kind: client, + _dd.base_service: Samples.AWS.Lambda + }, + Metrics: { + _dd.top_level: 1.0 + } } ] \ No newline at end of file diff --git a/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs b/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs index 88cc7f031f6c..d6fc9bf1f9f9 100644 --- a/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs +++ b/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs @@ -34,6 +34,13 @@ private static async Task Main(string[] args) // in the corresponding container. Each of the lambda instances are configured to use // ONE of the handler methods described in the #handler region below. + // Disabled + Thread.Sleep(1000); + await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_DISABLED_NO_PARAM_SYNC")); + Thread.Sleep(1000); + await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_DISABLED_TOPLEVEL_STATEMENT")); + + // Standard handlers await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_NO_PARAM_SYNC")); Thread.Sleep(1000); await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_ONE_PARAM_SYNC")); @@ -122,14 +129,9 @@ private static async Task Main(string[] args) Thread.Sleep(1000); await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_TOPLEVEL_STATEMENT")); - // Disabled - Thread.Sleep(1000); - await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_DISABLED_NO_PARAM_SYNC")); - Thread.Sleep(1000); - await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_DISABLED_TOPLEVEL_STATEMENT")); - static async Task Post(string url) { + Console.WriteLine("Sending request to " + url); HttpClient client = new HttpClient(); client.BaseAddress = new Uri(url); client.DefaultRequestHeaders From ae908f67d6fe918dc7b66578ac5ec17297a14aea Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 20 Mar 2025 15:13:00 +0000 Subject: [PATCH 09/11] Try to fix disabled bits --- docker-compose.serverless.yml | 4 ++-- .../AWS/AwsLambdaTests.cs | 6 +++--- .../integrations/Samples.AWS.Lambda/Program.cs | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.serverless.yml b/docker-compose.serverless.yml index c88cbbb03b34..dd9b443b3f5c 100644 --- a/docker-compose.serverless.yml +++ b/docker-compose.serverless.yml @@ -1072,7 +1072,7 @@ services: - "8080" volumes: - ./:/project - - ./artifacts/build_data/logs/serverless-lambda-no-param-sync:/var/log/datadog/dotnet + - ./artifacts/build_data/logs/serverless-lambda-disabled-no-param-sync:/var/log/datadog/dotnet serverless-lambda-disabled-toplevel-statement: build: @@ -1097,7 +1097,7 @@ services: - "8080" volumes: - ./:/project - - ./artifacts/build_data/logs/serverless-lambda-toplevel-statement:/var/log/datadog/dotnet + - ./artifacts/build_data/logs/serverless-lambda-disabled-toplevel-statement:/var/log/datadog/dotnet # The serverless function calls this API, which always returns 200 OK serverless-dummy-api: diff --git a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs index ade2908d821a..6e180e5d2a02 100644 --- a/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs +++ b/tracer/test/Datadog.Trace.ClrProfiler.IntegrationTests/AWS/AwsLambdaTests.cs @@ -59,9 +59,9 @@ public async Task SubmitsTraces() + 1 // Toplevel Statement + 2; // Disabled - // We have HTTP Request + Manual span + Lambda span for each request - // -2 for the "lambda integration disabled" cases - var expectedSpans = (requests * 2) - 2; + // We have HTTP Request + Manual span + Fake Lambda span for each request + // For disabled tests, we still get all the direct spans, we just lose the fake Lambda ones + var expectedSpans = requests * 2; var spans = agent.WaitForSpans(expectedSpans, 15_000).ToArray(); diff --git a/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs b/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs index d6fc9bf1f9f9..1ca14be37f1f 100644 --- a/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs +++ b/tracer/test/test-applications/integrations/Samples.AWS.Lambda/Program.cs @@ -40,7 +40,7 @@ private static async Task Main(string[] args) Thread.Sleep(1000); await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_DISABLED_TOPLEVEL_STATEMENT")); - // Standard handlers + // Param tests await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_NO_PARAM_SYNC")); Thread.Sleep(1000); await Post(Environment.GetEnvironmentVariable("AWS_LAMBDA_ENDPOINT_ONE_PARAM_SYNC")); From 9c6fc5636bf0bff7abcf6d3f54f0cfda93b1ed9a Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Thu, 20 Mar 2025 18:02:54 +0000 Subject: [PATCH 10/11] Try get some logs --- .azure-pipelines/ultimate-pipeline.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines/ultimate-pipeline.yml b/.azure-pipelines/ultimate-pipeline.yml index 3f08c81e09a9..f8139c394e59 100644 --- a/.azure-pipelines/ultimate-pipeline.yml +++ b/.azure-pipelines/ultimate-pipeline.yml @@ -2711,7 +2711,9 @@ stages: serverless-lambda-generic-abstract-async \ serverless-lambda-generic-complex \ serverless-lambda-generic-complex-nested \ - serverless-lambda-toplevel-statement + serverless-lambda-toplevel-statement \ + serverless-lambda-disabled-no-param-sync \ + serverless-lambda-disabled-toplevel-statement env: baseImage: $(baseImage) framework: $(publishTargetFramework) From b89adb2ebdced83e293df1b5e2dc7217cafb322c Mon Sep 17 00:00:00 2001 From: Andrew Lock Date: Fri, 21 Mar 2025 10:12:38 +0000 Subject: [PATCH 11/11] Make sure we flush traces even if the lambda span is disabled --- .../AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs | 4 +++- .../AutoInstrumentation/AWS/Lambda/LambdaCommon.cs | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs index ce573fd8ab57..0c004a76c4bc 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/HandlerWrapperSetHandlerIntegration.cs @@ -109,7 +109,9 @@ public async Task OnDelegateEndAsync(object sender, { if (!Tracer.Instance.Settings.IsIntegrationEnabled(IntegrationId.AwsLambda)) { - // integration disabled, don't create a scope, skip this trace + // integration disabled, don't create a scope, skip this trace, + // but we still need to make sure we flush any traces + await LambdaCommon.FlushTracesAsync().ConfigureAwait(false); return returnValue; } diff --git a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/LambdaCommon.cs b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/LambdaCommon.cs index e6e29669d0b3..0084dbc38b53 100644 --- a/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/LambdaCommon.cs +++ b/tracer/src/Datadog.Trace/ClrProfiler/AutoInstrumentation/AWS/Lambda/LambdaCommon.cs @@ -67,7 +67,7 @@ internal static void SendEndInvocation(ILambdaExtensionRequest requestBuilder, S } } - internal static async Task EndInvocationAsync(string returnValue, Exception exception, Scope scope, ILambdaExtensionRequest requestBuilder) + internal static async Task FlushTracesAsync() { try { @@ -79,6 +79,11 @@ await Tracer.Instance.TracerManager.AgentWriter.FlushTracesAsync() { Log("Could not flush to the extension", ex, false); } + } + + internal static async Task EndInvocationAsync(string returnValue, Exception exception, Scope scope, ILambdaExtensionRequest requestBuilder) + { + await FlushTracesAsync().ConfigureAwait(false); try {