From eb1ea2dbe5ca0d77ebde6ce56ec9da8c0e6e627a Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Mon, 13 Nov 2023 10:02:05 +0100 Subject: [PATCH 01/11] Disable most smoke tests for J9 JVMs Lot of flakiness running smoke tests on IBM JVMs. Disable tests for now on those JVMs as this is not priority for fixing them use annotation `@DisabledIf` and `datadog.trace.api.Platform#isJ9` --- .../SymbolExtractionTransformerTest.java | 72 ++++--------------- .../smoketest/LogProbesIntegrationTest.java | 6 ++ .../smoketest/ProbeStateIntegrationTest.java | 5 ++ .../SpanDecorationProbesIntegrationTests.java | 17 +++-- .../smoketest/SpanProbesIntegrationTest.java | 2 + 5 files changed, 36 insertions(+), 66 deletions(-) diff --git a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java index 1d57f4f22b6..01f4cfed5d8 100644 --- a/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java +++ b/dd-java-agent/agent-debugger/src/test/java/com/datadog/debugger/symbol/SymbolExtractionTransformerTest.java @@ -9,7 +9,6 @@ import com.datadog.debugger.sink.SymbolSink; import datadog.trace.api.Config; -import datadog.trace.api.Platform; import java.io.IOException; import java.lang.instrument.Instrumentation; import java.net.URISyntaxException; @@ -20,6 +19,7 @@ import org.joor.Reflect; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; import org.mockito.Mockito; class SymbolExtractionTransformerTest { @@ -55,11 +55,8 @@ public void noIncludesFilterOutDatadogClass() throws IOException, URISyntaxExcep } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction01() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction01"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction01.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -124,11 +121,8 @@ public void symbolExtraction01() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction02() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction02"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction02.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -155,11 +149,8 @@ public void symbolExtraction02() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction03() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction03"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction03.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -226,11 +217,8 @@ public void symbolExtraction03() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction04() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction04"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction04.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -301,11 +289,8 @@ public void symbolExtraction04() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction05() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction05"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction05.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -350,11 +335,8 @@ public void symbolExtraction05() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction06() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction06"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction06.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -399,11 +381,8 @@ public void symbolExtraction06() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction07() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction07"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction07.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -434,11 +413,8 @@ public void symbolExtraction07() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction08() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction08"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction08.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -471,11 +447,8 @@ public void symbolExtraction08() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction09() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction09"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction09.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -566,11 +539,8 @@ public void symbolExtraction09() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction10() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction10"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction10.java"; when(config.getDebuggerSymbolFlushThreshold()).thenReturn(2); @@ -621,11 +591,8 @@ public void symbolExtraction10() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction11() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction11"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction11.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -658,11 +625,8 @@ public void symbolExtraction11() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction12() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction12"; final String SOURCE_FILE = SYMBOL_PACKAGE_DIR + "SymbolExtraction12.java"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); @@ -735,11 +699,8 @@ public void symbolExtraction12() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction13() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction13"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); SymbolExtractionTransformer transformer = @@ -800,11 +761,8 @@ public void symbolExtraction13() throws IOException, URISyntaxException { } @Test + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") public void symbolExtraction14() throws IOException, URISyntaxException { - if (Platform.isJ9()) { - // Skip for IBM JVM - return; - } final String CLASS_NAME = SYMBOL_PACKAGE + "SymbolExtraction14"; SymbolSinkMock symbolSinkMock = new SymbolSinkMock(config); SymbolExtractionTransformer transformer = diff --git a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/LogProbesIntegrationTest.java b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/LogProbesIntegrationTest.java index 6acf81476f8..02df8902dad 100644 --- a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/LogProbesIntegrationTest.java +++ b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/LogProbesIntegrationTest.java @@ -32,6 +32,7 @@ import okhttp3.mockwebserver.RecordedRequest; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; public class LogProbesIntegrationTest extends SimpleAppDebuggerIntegrationTest { @Test @@ -237,12 +238,14 @@ void testMultiProbes() throws Exception { @Test @DisplayName("testSamplingSnapshotDefault") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testSamplingSnapshotDefault() throws Exception { doSamplingSnapshot(null, MethodLocation.EXIT); } @Test @DisplayName("testSamplingSnapshotDefaultWithConditionAtEntry") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testSamplingSnapshotDefaultWithConditionAtEntry() throws Exception { doSamplingSnapshot( new ProbeCondition(DSL.when(DSL.eq(value(1), value(1))), "1 == 1"), MethodLocation.ENTRY); @@ -250,6 +253,7 @@ void testSamplingSnapshotDefaultWithConditionAtEntry() throws Exception { @Test @DisplayName("testSamplingSnapshotDefaultWithConditionAtExit") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testSamplingSnapshotDefaultWithConditionAtExit() throws Exception { doSamplingSnapshot( new ProbeCondition(DSL.when(DSL.eq(value(1), value(1))), "1 == 1"), MethodLocation.EXIT); @@ -278,6 +282,7 @@ private void doSamplingSnapshot(ProbeCondition probeCondition, MethodLocation ev @Test @DisplayName("testSamplingLogDefault") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testSamplingLogDefault() throws Exception { batchSize = 100; final int LOOP_COUNT = 1000; @@ -301,6 +306,7 @@ void testSamplingLogDefault() throws Exception { @Test @DisplayName("testSamplingLogCustom") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testSamplingLogCustom() throws Exception { final int LOOP_COUNT = 1000; final String LOG_TEMPLATE = "log line {argInt} {argStr} {argDouble} {argMap} {argVar}"; diff --git a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProbeStateIntegrationTest.java b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProbeStateIntegrationTest.java index 4ea1ddc688f..9876f013021 100644 --- a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProbeStateIntegrationTest.java +++ b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProbeStateIntegrationTest.java @@ -12,10 +12,12 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; public class ProbeStateIntegrationTest extends ServerAppDebuggerIntegrationTest { @Test @DisplayName("testAddRemoveProbes") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testAddRemoveProbes() throws Exception { LogProbe logProbe = LogProbe.builder().probeId(PROBE_ID).where(TEST_APP_CLASS_NAME, FULL_METHOD_NAME).build(); @@ -37,6 +39,7 @@ void testAddRemoveProbes() throws Exception { @Test @DisplayName("testDisableEnableProbes") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testDisableEnableProbes() throws Exception { LogProbe logProbe = LogProbe.builder().probeId(PROBE_ID).where(TEST_APP_CLASS_NAME, FULL_METHOD_NAME).build(); @@ -58,6 +61,7 @@ void testDisableEnableProbes() throws Exception { @Test @DisplayName("testDisableEnableProbesUsingDenyList") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testDisableEnableProbesUsingDenyList() throws Exception { LogProbe logProbe = LogProbe.builder().probeId(PROBE_ID).where(TEST_APP_CLASS_NAME, FULL_METHOD_NAME).build(); @@ -88,6 +92,7 @@ void testDisableEnableProbesUsingDenyList() throws Exception { @Test @DisplayName("testDisableEnableProbesUsingAllowList") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testDisableEnableProbesUsingAllowList() throws Exception { LogProbe logProbe = LogProbe.builder().probeId(PROBE_ID).where(TEST_APP_CLASS_NAME, FULL_METHOD_NAME).build(); diff --git a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanDecorationProbesIntegrationTests.java b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanDecorationProbesIntegrationTests.java index 634ca3e4129..7aed4e56b16 100644 --- a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanDecorationProbesIntegrationTests.java +++ b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanDecorationProbesIntegrationTests.java @@ -16,7 +16,6 @@ import com.datadog.debugger.el.ProbeCondition; import com.datadog.debugger.el.expressions.BooleanExpression; import com.datadog.debugger.probe.SpanDecorationProbe; -import datadog.trace.api.Platform; import datadog.trace.bootstrap.debugger.EvaluationError; import datadog.trace.test.agent.decoder.DecodedSpan; import java.nio.file.Path; @@ -26,6 +25,7 @@ import java.util.concurrent.atomic.AtomicInteger; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; public class SpanDecorationProbesIntegrationTests extends ServerAppDebuggerIntegrationTest { @@ -39,11 +39,10 @@ protected ProcessBuilder createProcessBuilder(Path logFilePath, String... params @Test @DisplayName("testMethodSimpleTagNoCondition") + @DisabledIf( + value = "datadog.trace.api.Platform#isJ9", + disabledReason = "we cannot get local variable debug info") void testMethodSimpleTagNoCondition() throws Exception { - if (Platform.isJ9()) { - // skip for J9/OpenJ9 as we cannot get local variable debug info. - return; - } SpanDecorationProbe spanDecorationProbe = SpanDecorationProbe.builder() .probeId(PROBE_ID) @@ -70,11 +69,10 @@ void testMethodSimpleTagNoCondition() throws Exception { @Test @DisplayName("testMethodMultiTagsMultiConditions") + @DisabledIf( + value = "datadog.trace.api.Platform#isJ9", + disabledReason = "we cannot get local variable debug info") void testMethodMultiTagsMultiConditions() throws Exception { - if (Platform.isJ9()) { - // skip for J9/OpenJ9 as we cannot get local variable debug info. - return; - } List decorations = Arrays.asList( createDecoration( @@ -242,6 +240,7 @@ void testMethodMultiTagValueError() throws Exception { @Test @DisplayName("testSamplingSpanDecoration") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testSamplingSpanDecoration() throws Exception { SpanDecorationProbe spanDecorationProbe = SpanDecorationProbe.builder() diff --git a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanProbesIntegrationTest.java b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanProbesIntegrationTest.java index 8a63c6c1ea3..b7598f121a6 100644 --- a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanProbesIntegrationTest.java +++ b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/SpanProbesIntegrationTest.java @@ -11,6 +11,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.condition.DisabledIf; public class SpanProbesIntegrationTest extends SimpleAppDebuggerIntegrationTest { @@ -55,6 +56,7 @@ void testLineRangeSpan() throws Exception { @Test @DisplayName("testSingleLineSpan") + @DisabledIf(value = "datadog.trace.api.Platform#isJ9", disabledReason = "Flaky on J9 JVMs") void testSingleLineSpan() throws Exception { final String METHOD_NAME = "fullMethod"; final String EXPECTED_UPLOADS = "2"; // 2 probe statuses: RECEIVED + ERROR From 17734ae5fa179e2a62f76d51baef8daf8cda1824 Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Mon, 13 Nov 2023 11:45:30 +0100 Subject: [PATCH 02/11] Update system-tests to 6998b307ce0e8d206ca6eb99045651dcc8bc853f (#6199) --- .circleci/config.continue.yml.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.continue.yml.j2 b/.circleci/config.continue.yml.j2 index 6009f29fdbf..5d1486aa3c6 100644 --- a/.circleci/config.continue.yml.j2 +++ b/.circleci/config.continue.yml.j2 @@ -36,7 +36,7 @@ instrumentation_modules: &instrumentation_modules "dd-java-agent/instrumentation debugger_modules: &debugger_modules "dd-java-agent/agent-debugger|dd-java-agent/agent-bootstrap|dd-java-agent/agent-builder|internal-api|communication|dd-trace-core" profiling_modules: &profiling_modules "dd-java-agent/agent-profiling" -default_system_tests_commit: &default_system_tests_commit 2487cea5160a398549743d2cfd927a863792e3bd +default_system_tests_commit: &default_system_tests_commit 6998b307ce0e8d206ca6eb99045651dcc8bc853f parameters: nightly: From 0990524beba6f02ddd9ae218c73f31c8655d1485 Mon Sep 17 00:00:00 2001 From: jean-philippe bempel Date: Mon, 13 Nov 2023 11:04:25 +0100 Subject: [PATCH 03/11] Add line probe tracer smoke tests Add smoke tests for line log probes and tracer --- .../TracerDebuggerIntegrationTest.java | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/TracerDebuggerIntegrationTest.java b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/TracerDebuggerIntegrationTest.java index 4288e0c49b5..4b1884dd4b9 100644 --- a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/TracerDebuggerIntegrationTest.java +++ b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/TracerDebuggerIntegrationTest.java @@ -1,8 +1,9 @@ package datadog.smoketest; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; +import static com.datadog.debugger.util.LogProbeTestHelper.parseTemplate; +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; import com.datadog.debugger.agent.JsonSnapshotSerializer; @@ -56,7 +57,7 @@ void testTracer() throws Exception { .build(); JsonSnapshotSerializer.IntakeRequest request = doTestTracer(logProbe); Snapshot snapshot = request.getDebugger().getSnapshot(); - assertEquals("123356536", snapshot.getProbe().getId()); + assertEquals(PROBE_ID.getId(), snapshot.getProbe().getId()); assertTrue(Pattern.matches("[0-9a-f]+", request.getTraceId())); assertTrue(Pattern.matches("\\d+", request.getSpanId())); assertFalse( @@ -78,7 +79,7 @@ void testTracerDynamicLog() throws Exception { .build(); JsonSnapshotSerializer.IntakeRequest request = doTestTracer(logProbe); Snapshot snapshot = request.getDebugger().getSnapshot(); - assertEquals("123356536", snapshot.getProbe().getId()); + assertEquals(PROBE_ID.getId(), snapshot.getProbe().getId()); assertTrue(Pattern.matches("[0-9a-f]+", request.getTraceId())); assertTrue(Pattern.matches("\\d+", request.getSpanId())); assertFalse( @@ -96,12 +97,51 @@ void testTracerSameMethod() throws Exception { .build(); JsonSnapshotSerializer.IntakeRequest request = doTestTracer(logProbe); Snapshot snapshot = request.getDebugger().getSnapshot(); - assertEquals("123356536", snapshot.getProbe().getId()); + assertEquals(PROBE_ID.getId(), snapshot.getProbe().getId()); assertEquals(42, snapshot.getCaptures().getEntry().getArguments().get("argInt").getValue()); assertTrue(Pattern.matches("[0-9a-f]+", request.getTraceId())); assertTrue(Pattern.matches("\\d+", request.getSpanId())); } + @Test + @DisplayName("testTracerLineSnapshotProbe") + void testTracerLineSnapshotProbe() throws Exception { + LogProbe logProbe = + LogProbe.builder() + .probeId(PROBE_ID) + // on line: System.out.println(argInt); + .where("WebController.java", 15) + .captureSnapshot(true) + .build(); + JsonSnapshotSerializer.IntakeRequest request = doTestTracer(logProbe); + Snapshot snapshot = request.getDebugger().getSnapshot(); + assertEquals(PROBE_ID.getId(), snapshot.getProbe().getId()); + assertEquals( + 42, snapshot.getCaptures().getLines().get(15).getArguments().get("argInt").getValue()); + assertTrue(Pattern.matches("[0-9a-f]+", request.getTraceId())); + assertTrue(Pattern.matches("\\d+", request.getSpanId())); + } + + @Test + @DisplayName("testTracerLineDynamicLogProbe") + void testTracerLineDynamicLogProbe() throws Exception { + final String LOG_TEMPLATE = "processWithArg {argInt}"; + LogProbe logProbe = + LogProbe.builder() + .probeId(PROBE_ID) + .template(LOG_TEMPLATE, parseTemplate(LOG_TEMPLATE)) + // on line: System.out.println(argInt); + .where("WebController.java", 15) + .captureSnapshot(false) + .build(); + JsonSnapshotSerializer.IntakeRequest request = doTestTracer(logProbe); + Snapshot snapshot = request.getDebugger().getSnapshot(); + assertEquals(PROBE_ID.getId(), snapshot.getProbe().getId()); + assertEquals("processWithArg 42", request.getMessage()); + assertTrue(Pattern.matches("[0-9a-f]+", request.getTraceId())); + assertTrue(Pattern.matches("\\d+", request.getSpanId())); + } + private JsonSnapshotSerializer.IntakeRequest doTestTracer(LogProbe logProbe) throws Exception { setCurrentConfiguration(createConfig(logProbe)); String httpPort = String.valueOf(PortUtils.randomOpenPort()); From 6382bb4601302d3500dbb60093820a2af7d356ba Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:55:23 +0100 Subject: [PATCH 04/11] Fix error when trying to get class stream for Mockito mocks (#6183) --- .../agent-ci-visibility/build.gradle | 1 + .../trace/civisibility/source/Utils.java | 23 +++++++++++++++++-- .../civisibility/source/index/RepoIndex.java | 15 ++++-------- .../RepoIndexSourcePathResolverTest.groovy | 12 ++++++++++ 4 files changed, 39 insertions(+), 12 deletions(-) diff --git a/dd-java-agent/agent-ci-visibility/build.gradle b/dd-java-agent/agent-ci-visibility/build.gradle index 9917f17cbdb..d3d0ad668cb 100644 --- a/dd-java-agent/agent-ci-visibility/build.gradle +++ b/dd-java-agent/agent-ci-visibility/build.gradle @@ -73,6 +73,7 @@ excludedClassesCoverage += [ "datadog.trace.civisibility.source.index.RepoIndexBuilder.RepoIndexingFileVisitor", "datadog.trace.civisibility.source.index.RepoIndexFetcher", "datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver", + "datadog.trace.civisibility.source.Utils", "datadog.trace.civisibility.utils.ShellCommandExecutor", "datadog.trace.civisibility.utils.ShellCommandExecutor.OutputParser", "datadog.trace.civisibility.utils.SpanUtils" diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/Utils.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/Utils.java index f9023ab5b60..8c826776656 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/Utils.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/Utils.java @@ -7,7 +7,26 @@ public abstract class Utils { public static InputStream getClassStream(Class clazz) throws IOException { String className = clazz.getName(); - String classPath = "/" + className.replace('.', '/') + ".class"; - return clazz.getResourceAsStream(classPath); + InputStream classStream = clazz.getResourceAsStream(toResourceName(className)); + if (classStream != null) { + return classStream; + } else { + // might be auto-generated inner class (e.g. Mockito mock) + String topLevelClassName = stripNestedClassNames(clazz.getName()); + return clazz.getResourceAsStream(toResourceName(topLevelClassName)); + } + } + + private static String toResourceName(String className) { + return "/" + className.replace('.', '/') + ".class"; + } + + public static String stripNestedClassNames(String className) { + int innerClassNameIdx = className.indexOf('$'); + if (innerClassNameIdx >= 0) { + return className.substring(0, innerClassNameIdx); + } else { + return className; + } } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndex.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndex.java index d8010aeb072..54537b18810 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndex.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/source/index/RepoIndex.java @@ -50,7 +50,7 @@ public List getRootPackages() { @Nullable public String getSourcePath(@Nonnull Class c) { - String topLevelClassName = stripNestedClassNames(c.getName()); + String topLevelClassName = Utils.stripNestedClassNames(c.getName()); SourceType sourceType = detectSourceType(c); String extension = sourceType.getExtension(); String classNameWithExtension = topLevelClassName + extension; @@ -107,15 +107,6 @@ private SourceType detectSourceType(Class c) { return SourceType.JAVA; } - private String stripNestedClassNames(String className) { - int innerClassNameIdx = className.indexOf('$'); - if (innerClassNameIdx >= 0) { - return className.substring(0, innerClassNameIdx); - } else { - return className; - } - } - /** * Names of package-private classes do not have to correspond to the names of their source code * files. For such classes filename is extracted from SourceFile attribute that is available in @@ -126,6 +117,10 @@ private String getSourcePathForPackagePrivateOrNonJavaClass(Class c) { SourceFileAttributeVisitor sourceFileAttributeVisitor = new SourceFileAttributeVisitor(); try (InputStream classStream = Utils.getClassStream(c)) { + if (classStream == null) { + log.debug("Could not get input stream for class {}", c.getName()); + return null; + } ClassReader classReader = new ClassReader(classStream); classReader.accept( sourceFileAttributeVisitor, ClassReader.SKIP_CODE | ClassReader.SKIP_FRAMES); diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/RepoIndexSourcePathResolverTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/RepoIndexSourcePathResolverTest.groovy index 59a29662254..57b6b0ad229 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/RepoIndexSourcePathResolverTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/source/index/RepoIndexSourcePathResolverTest.groovy @@ -72,6 +72,17 @@ class RepoIndexSourcePathResolverTest extends Specification { sourcePathResolver.getSourcePath(PackagePrivateClass) == expectedSourcePath } + def "test source path resolution for class nested into package-private class"() { + setup: + def expectedSourcePath = givenSourceFile(RepoIndexSourcePathResolverTest, repoRoot + "/src") + + when: + def sourcePathResolver = new RepoIndexSourcePathResolver(repoRoot, packageResolver, fileSystem) + + then: + sourcePathResolver.getSourcePath(PackagePrivateClass.NestedIntoPackagePrivateClass) == expectedSourcePath + } + def "test source path resolution for non-java class whose file name is different from class name"() { setup: def expectedSourcePath = givenSourceFile(RepoIndexSourcePathResolverTest, repoRoot + "/src") @@ -173,6 +184,7 @@ class RepoIndexSourcePathResolverTest extends Specification { @PackageScope class PackagePrivateClass { + class NestedIntoPackagePrivateClass {} } class PublicClassWhoseNameDoesNotCorrespondToFileName { From a37fb7ec2977ae0a154e985f326000140e672121 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:55:58 +0100 Subject: [PATCH 05/11] Fix error when serializing skippable tests whose names contain non-ASCII characters (#6182) --- .../config/SkippableTestsSerializer.java | 22 +++++++++++-------- .../SkippableTestsSerializerTest.groovy | 2 ++ 2 files changed, 15 insertions(+), 9 deletions(-) rename dd-java-agent/agent-ci-visibility/src/{main/java => test/groovy}/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy (90%) diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializer.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializer.java index 3250f3a6389..e01e56785c9 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializer.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializer.java @@ -26,10 +26,10 @@ public static ByteBuffer serialize(Collection skippableTests) { String name = test.getName(); String parameters = test.getParameters(); - length += suite.length(); - length += name.length(); + length += suite.getBytes(CHARSET).length; + length += name.getBytes(CHARSET).length; if (parameters != null) { - length += parameters.length(); + length += parameters.getBytes(CHARSET).length; } } @@ -41,14 +41,18 @@ public static ByteBuffer serialize(Collection skippableTests) { String name = test.getName(); String parameters = test.getParameters(); - buffer.putInt(suite.length()); - buffer.put(suite.getBytes(CHARSET)); - buffer.putInt(name.length()); - buffer.put(name.getBytes(CHARSET)); + byte[] suiteBytes = suite.getBytes(CHARSET); + buffer.putInt(suiteBytes.length); + buffer.put(suiteBytes); + + byte[] nameBytes = name.getBytes(CHARSET); + buffer.putInt(nameBytes.length); + buffer.put(nameBytes); if (parameters != null) { - buffer.putInt(parameters.length()); - buffer.put(parameters.getBytes(CHARSET)); + byte[] parametersBytes = parameters.getBytes(CHARSET); + buffer.putInt(parametersBytes.length); + buffer.put(parametersBytes); } else { buffer.putInt(-1); } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy similarity index 90% rename from dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy rename to dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy index db382fc92a1..76494f2c43b 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/SkippableTestsSerializerTest.groovy @@ -22,6 +22,8 @@ class SkippableTestsSerializerTest extends Specification { tests << [ // single test [["suite", "name", null]], + [["suite", "𝕄 add user properties 𝕎 addUserProperties()", null]], + // non-ASCII characters [["suite", "name", "parameters"]], [["suite", "name", "{\"metadata\":{\"test_name\":\"test display name with #a #b #c\"}}"]], // multiple tests From 3fb888b81b6546784736580fc772c598297869d7 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Mon, 13 Nov 2023 12:56:12 +0100 Subject: [PATCH 06/11] Fail fast when tracer versions do not match in parent and child processes (#6168) --- .../trace/civisibility/CiVisibilitySystem.java | 13 +++++++++++++ .../config/ModuleExecutionSettingsFactoryImpl.java | 6 ++++++ .../trace/api/config/CiVisibilityConfig.java | 2 ++ .../src/main/java/datadog/trace/api/Config.java | 8 ++++++++ 4 files changed, 29 insertions(+) diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java index 1c3000726fa..4f4a29255b6 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java @@ -1,6 +1,7 @@ package datadog.trace.civisibility; import datadog.communication.ddagent.SharedCommunicationObjects; +import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.Config; import datadog.trace.api.civisibility.CIVisibility; import datadog.trace.api.civisibility.InstrumentationBridge; @@ -52,6 +53,7 @@ import datadog.trace.civisibility.source.index.RepoIndexProvider; import datadog.trace.civisibility.source.index.RepoIndexSourcePathResolver; import datadog.trace.util.Strings; +import datadog.trace.util.throwable.FatalAgentMisconfigurationError; import java.net.InetSocketAddress; import java.nio.file.FileSystems; import java.nio.file.Path; @@ -74,6 +76,17 @@ public static void start(SharedCommunicationObjects sco) { return; } + String injectedTracerVersion = config.getCiVisibilityInjectedTracerVersion(); + if (injectedTracerVersion != null + && !injectedTracerVersion.equals(TracerVersion.TRACER_VERSION)) { + throw new FatalAgentMisconfigurationError( + "Running JVM with tracer version " + + TracerVersion.TRACER_VERSION + + " however parent process attempted to inject " + + injectedTracerVersion + + ". Do not inject the tracer into the forked JVMs manually, or ensure the manually injected version is the same as the one injected automatically"); + } + sco.createRemaining(config); GitClient.Factory gitClientFactory = buildGitClientFactory(config); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java index 6122ac1aabe..6e4c4b29de4 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java @@ -1,5 +1,6 @@ package datadog.trace.civisibility.config; +import datadog.communication.ddagent.TracerVersion; import datadog.trace.api.Config; import datadog.trace.api.civisibility.config.Configurations; import datadog.trace.api.civisibility.config.ModuleExecutionSettings; @@ -156,6 +157,11 @@ private Map getPropertiesPropagatedToChildProcess( CiVisibilityConfig.CIVISIBILITY_BUILD_INSTRUMENTATION_ENABLED), Boolean.toString(false)); + propagatedSystemProperties.put( + Strings.propertyNameToSystemPropertyName( + CiVisibilityConfig.CIVISIBILITY_INJECTED_TRACER_VERSION), + TracerVersion.TRACER_VERSION); + return propagatedSystemProperties; } diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java index 1a3d861c3ac..b23ce0fd206 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java @@ -60,6 +60,8 @@ public final class CiVisibilityConfig { "civisibility.module.execution.settings.cache.size"; public static final String CIVISIBILITY_COVERAGE_SEGMENTS_ENABLED = "civisibility.coverage.segments.enabled"; + public static final String CIVISIBILITY_INJECTED_TRACER_VERSION = + "civisibility.injected.tracer.version"; private CiVisibilityConfig() {} } diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index a322a4c8f1d..b5b898012fa 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -155,6 +155,7 @@ import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_GIT_UNSHALLOW_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_TIMEOUT_MILLIS; +import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_INJECTED_TRACER_VERSION; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_ITR_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_JACOCO_GRADLE_SOURCE_SETS; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_EXCLUDES; @@ -716,6 +717,7 @@ static class HostNameHolder { private final boolean ciVisibilityRepoIndexSharingEnabled; private final int ciVisibilityModuleExecutionSettingsCacheSize; private final boolean ciVisibilityCoverageSegmentsEnabled; + private final String ciVisibilityInjectedTracerVersion; private final boolean remoteConfigEnabled; private final boolean remoteConfigIntegrityCheckEnabled; @@ -1651,6 +1653,8 @@ private Config(final ConfigProvider configProvider, final InstrumenterConfig ins configProvider.getInteger(CIVISIBILITY_MODULE_EXECUTION_SETTINGS_CACHE_SIZE, 16); ciVisibilityCoverageSegmentsEnabled = configProvider.getBoolean(CIVISIBILITY_COVERAGE_SEGMENTS_ENABLED, false); + ciVisibilityInjectedTracerVersion = + configProvider.getString(CIVISIBILITY_INJECTED_TRACER_VERSION); remoteConfigEnabled = configProvider.getBoolean(REMOTE_CONFIG_ENABLED, DEFAULT_REMOTE_CONFIG_ENABLED); @@ -2733,6 +2737,10 @@ public boolean isCiVisibilityCoverageSegmentsEnabled() { return ciVisibilityCoverageSegmentsEnabled; } + public String getCiVisibilityInjectedTracerVersion() { + return ciVisibilityInjectedTracerVersion; + } + public String getAppSecRulesFile() { return appSecRulesFile; } From 5dce4fdae63a4f4f1000d022179df0bdf6d94580 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko <121111529+nikita-tkachenko-datadog@users.noreply.github.com> Date: Mon, 13 Nov 2023 16:28:45 +0100 Subject: [PATCH 07/11] Add x-datadog-trace-id header to ITR backend requests (#6189) --- .../trace/civisibility/CiVisibilitySystem.java | 6 ++---- .../communication/BackendApiFactory.java | 6 ++++-- .../civisibility/communication/EvpProxyApi.java | 11 ++++++++++- .../civisibility/communication/IntakeApi.java | 17 +++++++++++++++-- .../config/ConfigurationApiImplTest.groovy | 3 ++- .../civisibility/git/tree/GitDataApiTest.groovy | 3 ++- 6 files changed, 35 insertions(+), 11 deletions(-) diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java index 4f4a29255b6..51ba05d9709 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/CiVisibilitySystem.java @@ -147,8 +147,6 @@ private static DDBuildSystemSession.Factory buildSystemSessionFactory( CoverageProbeStoreFactory coverageProbeStoreFactory, GitClient.Factory gitClientFactory) { BackendApiFactory backendApiFactory = new BackendApiFactory(config, sco); - BackendApi backendApi = backendApiFactory.createBackendApi(); - return (String projectName, Path projectRoot, String startCommand, @@ -178,6 +176,7 @@ private static DDBuildSystemSession.Factory buildSystemSessionFactory( TestDecorator testDecorator = new TestDecoratorImpl(buildSystemName, ciTags); TestModuleRegistry testModuleRegistry = new TestModuleRegistry(); + BackendApi backendApi = backendApiFactory.createBackendApi(); GitDataUploader gitDataUploader = buildGitDataUploader(config, gitInfoProvider, gitClientFactory, backendApi, repoRoot); ModuleExecutionSettingsFactory moduleExecutionSettingsFactory = @@ -241,8 +240,6 @@ private static DDTestFrameworkSession.Factory testFrameworkSessionFactory( CoverageProbeStoreFactory coverageProbeStoreFactory, GitClient.Factory gitClientFactory) { BackendApiFactory backendApiFactory = new BackendApiFactory(config, sco); - BackendApi backendApi = backendApiFactory.createBackendApi(); - return (String projectName, Path projectRoot, String component, Long startTime) -> { CIProviderInfoFactory ciProviderInfoFactory = new CIProviderInfoFactory(config); CIProviderInfo ciProviderInfo = ciProviderInfoFactory.createCIProviderInfo(projectRoot); @@ -288,6 +285,7 @@ private static DDTestFrameworkSession.Factory testFrameworkSessionFactory( // either we are in the build system // or we are in the tests JVM and the build system is not instrumented if (parentProcessSessionId == null || parentProcessModuleId == null) { + BackendApi backendApi = backendApiFactory.createBackendApi(); GitDataUploader gitDataUploader = buildGitDataUploader(config, gitInfoProvider, gitClientFactory, backendApi, repoRoot); RepoIndexProvider indexProvider = new RepoIndexBuilder(repoRoot, FileSystems.getDefault()); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java index e0f6c512b61..934e68868ee 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/BackendApiFactory.java @@ -33,17 +33,19 @@ public BackendApiFactory(Config config, SharedCommunicationObjects sharedCommuni "Agentless mode is enabled and api key is not set. Please set application key"); } long timeoutMillis = config.getCiVisibilityBackendApiTimeoutMillis(); - return new IntakeApi(site, apiKey, timeoutMillis, retryPolicyFactory); + String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); + return new IntakeApi(site, apiKey, traceId, timeoutMillis, retryPolicyFactory); } DDAgentFeaturesDiscovery featuresDiscovery = sharedCommunicationObjects.featuresDiscovery(config); featuresDiscovery.discoverIfOutdated(); if (featuresDiscovery.supportsEvpProxy()) { + String traceId = config.getIdGenerationStrategy().generateTraceId().toString(); String evpProxyEndpoint = featuresDiscovery.getEvpProxyEndpoint(); HttpUrl evpProxyUrl = sharedCommunicationObjects.agentUrl.resolve(evpProxyEndpoint); return new EvpProxyApi( - evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient); + traceId, evpProxyUrl, retryPolicyFactory, sharedCommunicationObjects.okHttpClient); } log.warn( diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java index 0353cd80fc7..2329b86b061 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/EvpProxyApi.java @@ -19,14 +19,21 @@ public class EvpProxyApi implements BackendApi { private static final String API_VERSION = "v2"; private static final String X_DATADOG_EVP_SUBDOMAIN_HEADER = "X-Datadog-EVP-Subdomain"; + private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id"; + private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id"; private static final String API_SUBDOMAIN = "api"; + private final String traceId; private final HttpRetryPolicy.Factory retryPolicyFactory; private final HttpUrl evpProxyUrl; private final OkHttpClient httpClient; public EvpProxyApi( - HttpUrl evpProxyUrl, HttpRetryPolicy.Factory retryPolicyFactory, OkHttpClient httpClient) { + String traceId, + HttpUrl evpProxyUrl, + HttpRetryPolicy.Factory retryPolicyFactory, + OkHttpClient httpClient) { + this.traceId = traceId; this.evpProxyUrl = evpProxyUrl.resolve(String.format("api/%s/", API_VERSION)); this.retryPolicyFactory = retryPolicyFactory; this.httpClient = httpClient; @@ -41,6 +48,8 @@ public T post( new Request.Builder() .url(url) .addHeader(X_DATADOG_EVP_SUBDOMAIN_HEADER, API_SUBDOMAIN) + .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) + .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId) .post(requestBody) .build(); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java index d8617376b9c..55728a277d2 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/communication/IntakeApi.java @@ -20,15 +20,23 @@ public class IntakeApi implements BackendApi { private static final String API_VERSION = "v2"; private static final String DD_API_KEY_HEADER = "dd-api-key"; + private static final String X_DATADOG_TRACE_ID_HEADER = "x-datadog-trace-id"; + private static final String X_DATADOG_PARENT_ID_HEADER = "x-datadog-parent-id"; private final String apiKey; + private final String traceId; private final HttpRetryPolicy.Factory retryPolicyFactory; private final HttpUrl hostUrl; private final OkHttpClient httpClient; public IntakeApi( - String site, String apiKey, long timeoutMillis, HttpRetryPolicy.Factory retryPolicyFactory) { + String site, + String apiKey, + String traceId, + long timeoutMillis, + HttpRetryPolicy.Factory retryPolicyFactory) { this.apiKey = apiKey; + this.traceId = traceId; this.retryPolicyFactory = retryPolicyFactory; final String ciVisibilityAgentlessUrlStr = Config.get().getCiVisibilityAgentlessUrl(); @@ -47,7 +55,12 @@ public T post( throws IOException { HttpUrl url = hostUrl.resolve(uri); Request.Builder requestBuilder = - new Request.Builder().url(url).post(requestBody).addHeader(DD_API_KEY_HEADER, apiKey); + new Request.Builder() + .url(url) + .post(requestBody) + .addHeader(DD_API_KEY_HEADER, apiKey) + .addHeader(X_DATADOG_TRACE_ID_HEADER, traceId) + .addHeader(X_DATADOG_PARENT_ID_HEADER, traceId); Request request = requestBuilder.build(); HttpRetryPolicy retryPolicy = retryPolicyFactory.create(); diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy index 53c0c31ee0a..41fcbdd54b0 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ConfigurationApiImplTest.groovy @@ -135,10 +135,11 @@ class ConfigurationApiImplTest extends Specification { } private BackendApi givenEvpProxy() { + String traceId = "a-trace-id" HttpUrl proxyUrl = HttpUrl.get(intakeServer.address) HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0) OkHttpClient client = OkHttpUtils.buildHttpClient(proxyUrl, REQUEST_TIMEOUT_MILLIS) - return new EvpProxyApi(proxyUrl, retryPolicyFactory, client) + return new EvpProxyApi(traceId, proxyUrl, retryPolicyFactory, client) } private static TracerEnvironment givenTracerEnvironment() { diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy index f0c6edfdca3..c280fe51f76 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/git/tree/GitDataApiTest.groovy @@ -114,10 +114,11 @@ class GitDataApiTest extends Specification { } private BackendApi givenEvpProxy() { + String traceId = "a-trace-id" HttpUrl proxyUrl = HttpUrl.get(intakeServer.address) HttpRetryPolicy.Factory retryPolicyFactory = new HttpRetryPolicy.Factory(5, 100, 2.0) OkHttpClient client = OkHttpUtils.buildHttpClient(proxyUrl, REQUEST_TIMEOUT_MILLIS) - return new EvpProxyApi(proxyUrl, retryPolicyFactory, client) + return new EvpProxyApi(traceId, proxyUrl, retryPolicyFactory, client) } private Path givenPackFile() { From 28afc961e6ddcd50ebc3c511d2e4cd46b9c31f62 Mon Sep 17 00:00:00 2001 From: Andrew Munn Date: Mon, 13 Nov 2023 11:24:49 -0500 Subject: [PATCH 08/11] Obfuscate Cassandra statements (#6187) * normalize CharSequence * add normalization in decorator * spotless * normalize v3 test * normalize test span * unused import * remove single quotes --- .../src/test/groovy/CassandraClientTest.groovy | 6 +++++- .../cassandra/CassandraClientDecorator.java | 18 ++++++++++++++++++ .../src/test/groovy/CassandraClientTest.groovy | 6 +++++- .../cassandra4/CassandraClientDecorator.java | 18 ++++++++++++++++++ .../src/test/groovy/CassandraClientTest.groovy | 6 +++++- .../trace/api/normalize/SQLNormalizer.java | 4 ++++ 6 files changed, 55 insertions(+), 3 deletions(-) diff --git a/dd-java-agent/instrumentation/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy index 8775d27baec..946e1b762a5 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra-3.8/src/test/groovy/CassandraClientTest.groovy @@ -134,11 +134,15 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { "SELECT * FROM users where name = 'alice' ALLOW FILTERING" | "async_test" | true } + String normalize(String statement){ + return statement.replaceAll("'alice'", "?") + } + def cassandraSpan(TraceAssert trace, String statement, String keyspace, boolean renameService, Object parentSpan = null, Throwable exception = null) { trace.span { serviceName renameService && keyspace ? keyspace : service() operationName operation() - resourceName statement + resourceName normalize(statement) spanType DDSpanTypes.CASSANDRA if (parentSpan == null) { parent() diff --git a/dd-java-agent/instrumentation/datastax-cassandra-3/src/main/java/datadog/trace/instrumentation/datastax/cassandra/CassandraClientDecorator.java b/dd-java-agent/instrumentation/datastax-cassandra-3/src/main/java/datadog/trace/instrumentation/datastax/cassandra/CassandraClientDecorator.java index 28d5b32eddf..ffe7c81fb25 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra-3/src/main/java/datadog/trace/instrumentation/datastax/cassandra/CassandraClientDecorator.java +++ b/dd-java-agent/instrumentation/datastax-cassandra-3/src/main/java/datadog/trace/instrumentation/datastax/cassandra/CassandraClientDecorator.java @@ -3,11 +3,15 @@ import com.datastax.driver.core.Host; import com.datastax.driver.core.ResultSet; import com.datastax.driver.core.Session; +import datadog.trace.api.cache.DDCache; +import datadog.trace.api.cache.DDCaches; import datadog.trace.api.naming.SpanNaming; +import datadog.trace.api.normalize.SQLNormalizer; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; import datadog.trace.bootstrap.instrumentation.decorator.DBTypeProcessingDatabaseClientDecorator; +import java.util.function.ToIntFunction; public class CassandraClientDecorator extends DBTypeProcessingDatabaseClientDecorator { private static final String DB_TYPE = "cassandra"; @@ -19,6 +23,15 @@ public class CassandraClientDecorator extends DBTypeProcessingDatabaseClientDeco public static final CassandraClientDecorator DECORATE = new CassandraClientDecorator(); + private static final int COMBINED_STATEMENT_LIMIT = 2 * 1024 * 1024; // chars + private static final ToIntFunction STATEMENT_WEIGHER = UTF8BytesString::length; + private static final DDCache CACHED_STATEMENTS = + DDCaches.newFixedSizeWeightedCache(512, STATEMENT_WEIGHER, COMBINED_STATEMENT_LIMIT); + + protected static UTF8BytesString normalizedQuery(CharSequence sql) { + return CACHED_STATEMENTS.computeIfAbsent(sql, SQLNormalizer::normalizeCharSequence); + } + @Override protected String[] instrumentationNames() { return new String[] {"cassandra"}; @@ -60,6 +73,11 @@ protected String dbHostname(Session session) { return null; } + public AgentSpan onStatement(final AgentSpan span, final CharSequence statement) { + span.setResourceName(normalizedQuery(statement)); + return span; + } + public AgentSpan onResponse(final AgentSpan span, final ResultSet result) { if (result != null) { final Host host = result.getExecutionInfo().getQueriedHost(); diff --git a/dd-java-agent/instrumentation/datastax-cassandra-3/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra-3/src/test/groovy/CassandraClientTest.groovy index 8775d27baec..946e1b762a5 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra-3/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra-3/src/test/groovy/CassandraClientTest.groovy @@ -134,11 +134,15 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { "SELECT * FROM users where name = 'alice' ALLOW FILTERING" | "async_test" | true } + String normalize(String statement){ + return statement.replaceAll("'alice'", "?") + } + def cassandraSpan(TraceAssert trace, String statement, String keyspace, boolean renameService, Object parentSpan = null, Throwable exception = null) { trace.span { serviceName renameService && keyspace ? keyspace : service() operationName operation() - resourceName statement + resourceName normalize(statement) spanType DDSpanTypes.CASSANDRA if (parentSpan == null) { parent() diff --git a/dd-java-agent/instrumentation/datastax-cassandra-4/src/main/java/datadog/trace/instrumentation/datastax/cassandra4/CassandraClientDecorator.java b/dd-java-agent/instrumentation/datastax-cassandra-4/src/main/java/datadog/trace/instrumentation/datastax/cassandra4/CassandraClientDecorator.java index 98817ca2b5d..dc1cb7d5b31 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra-4/src/main/java/datadog/trace/instrumentation/datastax/cassandra4/CassandraClientDecorator.java +++ b/dd-java-agent/instrumentation/datastax-cassandra-4/src/main/java/datadog/trace/instrumentation/datastax/cassandra4/CassandraClientDecorator.java @@ -5,7 +5,10 @@ import com.datastax.oss.driver.api.core.metadata.Node; import com.datastax.oss.driver.api.core.servererrors.CoordinatorException; import com.datastax.oss.driver.api.core.session.Session; +import datadog.trace.api.cache.DDCache; +import datadog.trace.api.cache.DDCaches; import datadog.trace.api.naming.SpanNaming; +import datadog.trace.api.normalize.SQLNormalizer; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.InternalSpanTypes; import datadog.trace.bootstrap.instrumentation.api.UTF8BytesString; @@ -13,6 +16,7 @@ import java.net.InetSocketAddress; import java.net.SocketAddress; import java.util.Objects; +import java.util.function.ToIntFunction; public class CassandraClientDecorator extends DBTypeProcessingDatabaseClientDecorator { private static final String DB_TYPE = "cassandra"; @@ -24,6 +28,15 @@ public class CassandraClientDecorator extends DBTypeProcessingDatabaseClientDeco public static final CassandraClientDecorator DECORATE = new CassandraClientDecorator(); + private static final int COMBINED_STATEMENT_LIMIT = 2 * 1024 * 1024; // chars + private static final ToIntFunction STATEMENT_WEIGHER = UTF8BytesString::length; + private static final DDCache CACHED_STATEMENTS = + DDCaches.newFixedSizeWeightedCache(512, STATEMENT_WEIGHER, COMBINED_STATEMENT_LIMIT); + + protected static UTF8BytesString normalizedQuery(CharSequence sql) { + return CACHED_STATEMENTS.computeIfAbsent(sql, SQLNormalizer::normalizeCharSequence); + } + @Override protected String[] instrumentationNames() { return new String[] {"cassandra"}; @@ -64,6 +77,11 @@ protected String dbHostname(Session session) { return null; } + public AgentSpan onStatement(final AgentSpan span, final CharSequence statement) { + span.setResourceName(normalizedQuery(statement)); + return span; + } + public AgentSpan onResponse(final AgentSpan span, final ResultSet result) { if (result != null) { return onResponse(span, result.getExecutionInfo().getCoordinator()); diff --git a/dd-java-agent/instrumentation/datastax-cassandra-4/src/test/groovy/CassandraClientTest.groovy b/dd-java-agent/instrumentation/datastax-cassandra-4/src/test/groovy/CassandraClientTest.groovy index bdafd0f4a1f..21bb4bf3ff3 100644 --- a/dd-java-agent/instrumentation/datastax-cassandra-4/src/test/groovy/CassandraClientTest.groovy +++ b/dd-java-agent/instrumentation/datastax-cassandra-4/src/test/groovy/CassandraClientTest.groovy @@ -210,11 +210,15 @@ abstract class CassandraClientTest extends VersionedNamingTestBase { .withConfigLoader(configLoader) } + String normalize(String statement){ + return statement.replaceAll("'alice'", "?") + } + def cassandraSpan(TraceAssert trace, String statement, String keyspace, boolean renameService, Object parentSpan = null, Throwable throwable = null) { trace.span { serviceName renameService && keyspace ? keyspace : service() operationName operation() - resourceName statement + resourceName normalize(statement) spanType DDSpanTypes.CASSANDRA if (parentSpan == null) { parent() diff --git a/internal-api/src/main/java/datadog/trace/api/normalize/SQLNormalizer.java b/internal-api/src/main/java/datadog/trace/api/normalize/SQLNormalizer.java index 8400b26cd60..d8bfd67f076 100644 --- a/internal-api/src/main/java/datadog/trace/api/normalize/SQLNormalizer.java +++ b/internal-api/src/main/java/datadog/trace/api/normalize/SQLNormalizer.java @@ -40,6 +40,10 @@ public final class SQLNormalizer { } } + public static UTF8BytesString normalizeCharSequence(CharSequence sql) { + return normalize(sql.toString()); + } + public static UTF8BytesString normalize(String sql) { byte[] utf8 = sql.getBytes(UTF_8); try { From 36f0e4bae74025e1348d2382659236f4ca739a92 Mon Sep 17 00:00:00 2001 From: Bruce Bujon Date: Mon, 13 Nov 2023 17:55:42 +0100 Subject: [PATCH 09/11] Ensure OTel span kind is set (#6205) --- .../trace/OtelSpanBuilder.java | 12 +++++- .../OpenTelemetry14ConventionsTest.groovy | 14 +++---- .../test/groovy/OpenTelemetry14Test.groovy | 42 ++++++++++++------- .../context/ContextTest.groovy | 6 ++- .../propagation/AbstractPropagatorTest.groovy | 5 +-- .../smoketest/Play28OTelSmokeTest.groovy | 12 +++--- 6 files changed, 55 insertions(+), 36 deletions(-) diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/main/java/datadog/trace/instrumentation/opentelemetry14/trace/OtelSpanBuilder.java b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/main/java/datadog/trace/instrumentation/opentelemetry14/trace/OtelSpanBuilder.java index aca54bda20b..ca9d7b9809b 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/main/java/datadog/trace/instrumentation/opentelemetry14/trace/OtelSpanBuilder.java +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/main/java/datadog/trace/instrumentation/opentelemetry14/trace/OtelSpanBuilder.java @@ -2,6 +2,7 @@ import static datadog.trace.instrumentation.opentelemetry14.trace.OtelConventions.toSpanType; import static datadog.trace.instrumentation.opentelemetry14.trace.OtelExtractedContext.extract; +import static io.opentelemetry.api.trace.SpanKind.INTERNAL; import datadog.trace.bootstrap.instrumentation.api.AgentSpan; import datadog.trace.bootstrap.instrumentation.api.AgentTracer; @@ -19,9 +20,11 @@ @ParametersAreNonnullByDefault public class OtelSpanBuilder implements SpanBuilder { private final AgentTracer.SpanBuilder delegate; + private boolean spanKindSet; public OtelSpanBuilder(AgentTracer.SpanBuilder delegate) { this.delegate = delegate; + this.spanKindSet = false; } @Override @@ -105,7 +108,10 @@ public SpanBuilder setAttribute(AttributeKey key, T value) { @Override public SpanBuilder setSpanKind(SpanKind spanKind) { - this.delegate.withSpanType(toSpanType(spanKind)); + if (spanKind != null) { + this.delegate.withSpanType(toSpanType(spanKind)); + this.spanKindSet = true; + } return this; } @@ -117,6 +123,10 @@ public SpanBuilder setStartTimestamp(long startTimestamp, TimeUnit unit) { @Override public Span startSpan() { + // Ensure the span kind is set + if (!this.spanKindSet) { + setSpanKind(INTERNAL); + } AgentSpan delegate = this.delegate.start(); return new OtelSpan(delegate); } diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/OpenTelemetry14ConventionsTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/OpenTelemetry14ConventionsTest.groovy index 558e09162b4..39ccac0a843 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/OpenTelemetry14ConventionsTest.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/OpenTelemetry14ConventionsTest.groovy @@ -7,7 +7,6 @@ import spock.lang.Subject import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_CLIENT import static datadog.trace.bootstrap.instrumentation.api.Tags.SPAN_KIND_SERVER -import static datadog.trace.instrumentation.opentelemetry14.trace.OtelConventions.DEFAULT_OPERATION_NAME import static datadog.trace.instrumentation.opentelemetry14.trace.OtelConventions.toSpanType import static io.opentelemetry.api.trace.SpanKind.CLIENT import static io.opentelemetry.api.trace.SpanKind.CONSUMER @@ -29,9 +28,7 @@ class OpenTelemetry14ConventionsTest extends AgentTestRunner { def "test span name conventions"() { when: def builder = tracer.spanBuilder("some-name") - if (kind != null) { - builder.setSpanKind(kind) - } + .setSpanKind(kind) attributes.forEach { key, value -> builder.setAttribute(key, value) } builder.startSpan() .end() @@ -41,9 +38,7 @@ class OpenTelemetry14ConventionsTest extends AgentTestRunner { trace(1) { span { parent() - if (kind != null) { - spanType toSpanType(kind) - } + spanType toSpanType(kind == null ? INTERNAL : kind) operationName "$expectedOperationName" resourceName "some-name" } @@ -53,7 +48,7 @@ class OpenTelemetry14ConventionsTest extends AgentTestRunner { where: kind | attributes | expectedOperationName // Fallback behavior - null | [:] | DEFAULT_OPERATION_NAME + null | [:] | "internal" // Internal spans INTERNAL | [:] | "internal" // Server spans @@ -133,7 +128,8 @@ class OpenTelemetry14ConventionsTest extends AgentTestRunner { trace(1) { span { parent() - operationName "$DEFAULT_OPERATION_NAME" + operationName "internal" + spanType "internal" tags { defaultTags() if (value != null) { diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/OpenTelemetry14Test.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/OpenTelemetry14Test.groovy index fb529889e59..0e4f5207249 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/OpenTelemetry14Test.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/OpenTelemetry14Test.groovy @@ -44,13 +44,15 @@ class OpenTelemetry14Test extends AgentTestRunner { trace(2) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "some-name" + spanType "internal" } span { childOfPrevious() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "other-name" + spanType "internal" } } } @@ -72,13 +74,15 @@ class OpenTelemetry14Test extends AgentTestRunner { trace(2) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "some-name" + spanType "internal" } span { childOfPrevious() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "other-name" + spanType "internal" } } } @@ -118,15 +122,17 @@ class OpenTelemetry14Test extends AgentTestRunner { trace(1) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName"some-name" + spanType "internal" } } trace(1) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName"other-name" + spanType "internal" } } } @@ -139,8 +145,6 @@ class OpenTelemetry14Test extends AgentTestRunner { anotherSpan.end() when: - // Adding link is not supported - builder.addLink(anotherSpan.getSpanContext()) // Adding event is not supported def result = builder.startSpan() result.addEvent("some-event") @@ -149,10 +153,13 @@ class OpenTelemetry14Test extends AgentTestRunner { then: assertTraces(2) { trace(1) { - span {} + span { + spanType "internal"} } trace(1) { - span {} + span { + spanType "internal" + } } } } @@ -201,7 +208,8 @@ class OpenTelemetry14Test extends AgentTestRunner { trace(1) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" + spanType "internal" if (tagSpan) { resourceName "other-resource" } else if (tagBuilder) { @@ -303,8 +311,9 @@ class OpenTelemetry14Test extends AgentTestRunner { trace(1) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "some-name" + spanType "internal" errored true tags { @@ -355,8 +364,9 @@ class OpenTelemetry14Test extends AgentTestRunner { trace(1) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "some-name" + spanType "internal" errored false tags { defaultTags() @@ -396,8 +406,9 @@ class OpenTelemetry14Test extends AgentTestRunner { trace(1) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "some-name" + spanType "internal" errored false tags { defaultTags() @@ -458,8 +469,9 @@ class OpenTelemetry14Test extends AgentTestRunner { trace(1) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName"some-name" + spanType "internal" errored true tags { defaultTags() diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/opentelemetry14/context/ContextTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/opentelemetry14/context/ContextTest.groovy index bd834f9aefb..72d23559268 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/opentelemetry14/context/ContextTest.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/opentelemetry14/context/ContextTest.groovy @@ -224,8 +224,9 @@ class ContextTest extends AgentTestRunner { trace(3) { span { parent() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "some-name" + spanType "internal" } span { childOfPrevious() @@ -233,8 +234,9 @@ class ContextTest extends AgentTestRunner { } span { childOfPrevious() - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "another-name" + spanType "internal" } } } diff --git a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/opentelemetry14/context/propagation/AbstractPropagatorTest.groovy b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/opentelemetry14/context/propagation/AbstractPropagatorTest.groovy index 977f5f9c40a..80c988de26d 100644 --- a/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/opentelemetry14/context/propagation/AbstractPropagatorTest.groovy +++ b/dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.4/src/test/groovy/opentelemetry14/context/propagation/AbstractPropagatorTest.groovy @@ -13,8 +13,6 @@ import spock.lang.Subject import javax.annotation.Nullable -import static datadog.trace.instrumentation.opentelemetry14.trace.OtelConventions.DEFAULT_OPERATION_NAME - abstract class AbstractPropagatorTest extends AgentTestRunner { @Subject def tracer = GlobalOpenTelemetry.get().tracerProvider.get("propagator" + Math.random()) // TODO FIX LATER @@ -88,8 +86,9 @@ abstract class AbstractPropagatorTest extends AgentTestRunner { assertTraces(1) { trace(1) { span { - operationName DEFAULT_OPERATION_NAME + operationName "internal" resourceName "some-name" + spanType "internal" traceDDId(DD128bTraceId.fromHex(traceId)) parentSpanId(DDSpanId.fromHex(spanId).toLong() as BigInteger) } diff --git a/dd-smoke-tests/play-2.8-otel/src/test/groovy/datadog/smoketest/Play28OTelSmokeTest.groovy b/dd-smoke-tests/play-2.8-otel/src/test/groovy/datadog/smoketest/Play28OTelSmokeTest.groovy index 7ab1e90ae44..2f8655ac7c4 100644 --- a/dd-smoke-tests/play-2.8-otel/src/test/groovy/datadog/smoketest/Play28OTelSmokeTest.groovy +++ b/dd-smoke-tests/play-2.8-otel/src/test/groovy/datadog/smoketest/Play28OTelSmokeTest.groovy @@ -78,9 +78,9 @@ abstract class Play28OTelSmokeTest extends AbstractServerSmokeTest { // is undefined. boolean isOk = true def allowed = /|^\[${serverProviderName()}.request:GET \/welcome[sj] - |(\[otel_unknown:filter1(\[otel_unknown:filter\d])(\[otel_unknown:filter\d])(\[otel_unknown:filter\d])])? - |\[play.request:GET \/welcome[sj]\[otel_unknown:action1\[otel_unknown:action2\[otel_unknown:do-get\[play-ws.request:GET \/hello\/\?]]]]] - |(\[otel_unknown:filter1(\[otel_unknown:filter\d])(\[otel_unknown:filter\d])(\[otel_unknown:filter\d])])? + |(\[internal:filter1(\[internal:filter\d])(\[internal:filter\d])(\[internal:filter\d])])? + |\[play.request:GET \/welcome[sj]\[internal:action1\[internal:action2\[internal:do-get\[play-ws.request:GET \/hello\/\?]]]]] + |(\[internal:filter1(\[internal:filter\d])(\[internal:filter\d])(\[internal:filter\d])])? |]$/.stripMargin().replaceAll("[\n\r]", "") // Ignore [command_execution], which can be generated by hostname calls from Config/Telemetry on some systems. @@ -94,9 +94,9 @@ abstract class Play28OTelSmokeTest extends AbstractServerSmokeTest { |traceCounts=${traceCounts}""".stripMargin() def matches = matcher.head().findAll{ it != null } isOk &= matches.size() == 5 - isOk &= matches.contains("[otel_unknown:filter2]") - isOk &= matches.contains("[otel_unknown:filter3]") - isOk &= matches.contains("[otel_unknown:filter4]") + isOk &= matches.contains("[internal:filter2]") + isOk &= matches.contains("[internal:filter3]") + isOk &= matches.contains("[internal:filter4]") assert isOk : """\ |Trace ${it.key} does not match allowed pattern: |pattern=${allowed} From 0ecb5fdd045dfea4d817f40cf80f63c5d8fe9b23 Mon Sep 17 00:00:00 2001 From: "Santiago M. Mola" Date: Mon, 13 Nov 2023 18:49:27 +0100 Subject: [PATCH 10/11] Exclude net.jodah.failsafe from IAST (#6200) --- .../trace/instrumentation/iastinstrumenter/iast_exclusion.trie | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dd-java-agent/instrumentation/iast-instrumenter/src/main/resources/datadog/trace/instrumentation/iastinstrumenter/iast_exclusion.trie b/dd-java-agent/instrumentation/iast-instrumenter/src/main/resources/datadog/trace/instrumentation/iastinstrumenter/iast_exclusion.trie index 05e3234670b..f5e4c64bbce 100644 --- a/dd-java-agent/instrumentation/iast-instrumenter/src/main/resources/datadog/trace/instrumentation/iastinstrumenter/iast_exclusion.trie +++ b/dd-java-agent/instrumentation/iast-instrumenter/src/main/resources/datadog/trace/instrumentation/iastinstrumenter/iast_exclusion.trie @@ -163,6 +163,8 @@ 1 nano.xml.* 1 net.bytebuddy.* 1 net.jcip.* +# Weak randomness false positive in net.jodah.failsafe.RetryPolicyExecutor +1 net.jodah.failsafe.* # Amusing weak randomness false positive in XXHashFactory 1 net.jpountz.xxhash.* 1 net.logstash.* From 00f26717de3859315fc8daa71219048f870bbf4b Mon Sep 17 00:00:00 2001 From: Stuart McCulloch Date: Mon, 13 Nov 2023 22:42:30 +0000 Subject: [PATCH 11/11] Bump integrations-core submodule to 7.49.0 (#6207) --- dd-java-agent/agent-jmxfetch/integrations-core | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dd-java-agent/agent-jmxfetch/integrations-core b/dd-java-agent/agent-jmxfetch/integrations-core index 03aed80d105..d211ef6ec4c 160000 --- a/dd-java-agent/agent-jmxfetch/integrations-core +++ b/dd-java-agent/agent-jmxfetch/integrations-core @@ -1 +1 @@ -Subproject commit 03aed80d105aa81b047e74c6da086165cac5ff6f +Subproject commit d211ef6ec4c9584df4b9f430dd10382f169d4727