From 616ecc6a4ff278f69077f3d00ac261a00204c363 Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko Date: Tue, 28 May 2024 13:35:39 +0200 Subject: [PATCH] Create commmon backend-mock module to use in smoke tests --- .../civisibility/CiVisibilitySmokeTest.groovy | 208 +------------- .../instrumentation/maven3/MavenUtils.java | 10 +- dd-smoke-tests/backend-mock/build.gradle | 7 + .../datadog/smoketest/MockBackend.groovy | 263 ++++++++++++++++++ dd-smoke-tests/gradle/build.gradle | 3 +- .../smoketest/GradleDaemonSmokeTest.groovy | 25 +- dd-smoke-tests/log-injection/build.gradle | 1 + .../smoketest/LogInjectionSmokeTest.groovy | 65 +++-- dd-smoke-tests/maven/build.gradle | 3 +- .../datadog/smoketest/MavenSmokeTest.groovy | 64 +++-- settings.gradle | 1 + 11 files changed, 390 insertions(+), 260 deletions(-) create mode 100644 dd-smoke-tests/backend-mock/build.gradle create mode 100644 dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy index 1ab3a12f8ee..645cc1965d9 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilitySmokeTest.groovy @@ -1,181 +1,11 @@ package datadog.trace.civisibility -import com.fasterxml.jackson.databind.ObjectMapper -import datadog.trace.agent.test.server.http.TestHttpServer -import datadog.trace.test.util.MultipartRequestParser -import org.apache.commons.io.IOUtils -import org.msgpack.jackson.dataformat.MessagePackFactory -import spock.lang.AutoCleanup -import spock.lang.Shared -import spock.lang.Specification -import spock.util.concurrent.PollingConditions - -import java.util.concurrent.ConcurrentLinkedQueue -import java.util.zip.GZIPInputStream -import java.util.zip.GZIPOutputStream -import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer +import spock.lang.Specification abstract class CiVisibilitySmokeTest extends Specification { - @Shared - ObjectMapper msgPackMapper = new ObjectMapper(new MessagePackFactory()) - - @Shared - Queue> receivedTraces = new ConcurrentLinkedQueue<>() - - @Shared - Queue> receivedCoverages = new ConcurrentLinkedQueue<>() - - @Shared - Queue> receivedTelemetryMetrics = new ConcurrentLinkedQueue<>() - - @Shared - Queue> receivedTelemetryDistributions = new ConcurrentLinkedQueue<>() - - @Shared - boolean codeCoverageEnabled = true - - @Shared - boolean testsSkippingEnabled = true - - @Shared - boolean flakyRetriesEnabled = false - - def setup() { - receivedTraces.clear() - receivedCoverages.clear() - receivedTelemetryMetrics.clear() - receivedTelemetryDistributions.clear() - } - - @Shared - @AutoCleanup - protected TestHttpServer intakeServer = httpServer { - handlers { - prefix("/api/v2/citestcycle") { - def contentEncodingHeader = request.getHeader("Content-Encoding") - def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") - def requestBody = gzipEnabled ? CiVisibilitySmokeTest.decompress(request.body) : request.body - def decodedEvent = msgPackMapper.readValue(requestBody, Map) - receivedTraces.add(decodedEvent) - - response.status(200).send() - } - - prefix("/api/v2/citestcov") { - def contentEncodingHeader = request.getHeader("Content-Encoding") - def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") - def requestBody = gzipEnabled ? CiVisibilitySmokeTest.decompress(request.body) : request.body - def parsed = MultipartRequestParser.parseRequest(requestBody, request.headers.get("Content-Type")) - def coverages = parsed.get("coverage1") - for (def coverage : coverages) { - def decodedCoverage = msgPackMapper.readValue(coverage.get(), Map) - receivedCoverages.add(decodedCoverage) - } - - response.status(202).send() - } - - prefix("/api/v2/libraries/tests/services/setting") { - // not compressing settings response on purpose, to better mimic real backend behavior: - // it may choose to compress the response or not based on its size, - // so smaller responses (like those of /setting endpoint) are uncompressed, - // while the larger ones (skippable and flaky test lists) are compressed - response.status(200).send(('{ "data": { "type": "ci_app_tracers_test_service_settings", "id": "uuid", "attributes": { ' - + '"code_coverage": ' + codeCoverageEnabled - + ', "tests_skipping": ' + testsSkippingEnabled - + ', "flaky_test_retries_enabled": ' + flakyRetriesEnabled + '} } }').bytes) - } - - prefix("/api/v2/ci/tests/skippable") { - response.status(200).addHeader("Content-Encoding", "gzip").send(CiVisibilitySmokeTest.compress(('{ "data": [{' + - ' "id": "d230520a0561ee2f",' + - ' "type": "test",' + - ' "attributes": {' + - ' "configurations": {' + - ' "test.bundle": "Maven Smoke Tests Project maven-surefire-plugin default-test"' + - ' },' + - ' "name": "test_to_skip_with_itr",' + - ' "suite": "datadog.smoke.TestSucceed"' + - ' }' + - '}, {' + - ' "id": "d230520a0561ee2g",' + - ' "type": "test",' + - ' "attributes": {' + - ' "configurations": {' + - ' "test.bundle": ":test"' + - ' },' + - ' "name": "test_to_skip_with_itr",' + - ' "suite": "datadog.smoke.TestSucceed"' + - ' }' + - '}] ' + - '}').bytes)) - } - - prefix("/api/v2/ci/libraries/tests/flaky") { - response.status(200).addHeader("Content-Encoding", "gzip").send(CiVisibilitySmokeTest.compress(('{ "data": [{' + - ' "id": "d230520a0561ee2f",' + - ' "type": "test",' + - ' "attributes": {' + - ' "configurations": {' + - ' "test.bundle": "Maven Smoke Tests Project maven-surefire-plugin default-test"' + - ' },' + - ' "name": "test_failed",' + - ' "suite": "datadog.smoke.TestFailed"' + - ' }' + - '}, {' + - ' "id": "d230520a0561ee2g",' + - ' "type": "test",' + - ' "attributes": {' + - ' "configurations": {' + - ' "test.bundle": ":test"' + - ' },' + - ' "name": "test_failed",' + - ' "suite": "datadog.smoke.TestFailed"' + - ' }' + - '}] ' + - '}').bytes)) - } - - prefix("/api/v2/apmtelemetry") { - def telemetryRequest = CiVisibilityTestUtils.JSON_MAPPER.readerFor(Map.class).readValue(request.body) - def requestType = telemetryRequest["request_type"] - if (requestType == "message-batch") { - for (def message : telemetryRequest["payload"]) { - def payload = message["payload"] - if (message["request_type"] == 'generate-metrics') { - receivedTelemetryMetrics.addAll((List) payload["series"]) - } else if (message["request_type"] == 'distributions') { - receivedTelemetryDistributions.addAll((List) payload["series"]) - } - } - } - response.status(202).send() - } - } - } - - private static byte[] compress(byte[] bytes) { - def baos = new ByteArrayOutputStream() - try (GZIPOutputStream zip = new GZIPOutputStream(baos)) { - IOUtils.copy(new ByteArrayInputStream(bytes), zip) - } - return baos.toByteArray() - } - - private static byte[] decompress(byte[] bytes) { - def baos = new ByteArrayOutputStream() - try (GZIPInputStream zip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { - IOUtils.copy(zip, baos) - } - return baos.toByteArray() - } - - protected verifyEventsAndCoverages(String projectName, String toolchain, String toolchainVersion, int expectedEventsCount, int expectedCoveragesCount) { - def events = waitForEvents(expectedEventsCount) - def coverages = waitForCoverages(expectedCoveragesCount) - + protected verifyEventsAndCoverages(String projectName, String toolchain, String toolchainVersion, List> events, List> coverages) { def additionalReplacements = ["content.meta.['test.toolchain']": "$toolchain:$toolchainVersion"] // uncomment to generate expected data templates @@ -192,7 +22,7 @@ abstract class CiVisibilitySmokeTest extends Specification { * Currently the check is not performed for Gradle builds: * Gradle daemon started with Gradle TestKit outlives the test, so the final telemetry flush happens after the assertions. */ - protected verifyTelemetryMetrics( int expectedEventsCount) { + protected verifyTelemetryMetrics(List> receivedTelemetryMetrics, List> receivedTelemetryDistributions, int expectedEventsCount) { int eventsCreated = 0, eventsFinished = 0 for (Map metric : receivedTelemetryMetrics) { if (metric["metric"] == "event_created") { @@ -212,36 +42,4 @@ abstract class CiVisibilitySmokeTest extends Specification { // an even more basic smoke check for distributions: assert that we received some assert !receivedTelemetryDistributions.isEmpty() } - - protected List> waitForEvents(int expectedEventsSize) { - def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) - traceReceiveConditions.eventually { - int eventsSize = 0 - for (Map trace : receivedTraces) { - eventsSize += trace["events"].size() - } - assert eventsSize >= expectedEventsSize - } - - List> events = new ArrayList<>() - while (!receivedTraces.isEmpty()) { - def trace = receivedTraces.poll() - events.addAll((List>) trace["events"]) - } - return events - } - - protected List> waitForCoverages(int traceSize) { - def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) - traceReceiveConditions.eventually { - assert receivedCoverages.size() == traceSize - } - - List> coverages = new ArrayList<>() - while (!receivedCoverages.isEmpty()) { - def trace = receivedCoverages.poll() - coverages.addAll((List>) trace["coverages"]) - } - return coverages - } } diff --git a/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenUtils.java b/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenUtils.java index 0af804a208c..67f5950acd2 100644 --- a/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenUtils.java +++ b/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenUtils.java @@ -290,6 +290,8 @@ public static String getUniqueModuleName(MavenProject project, MojoExecution moj private static final MethodHandle SESSION_FIELD = METHOD_HANDLES.privateFieldGetter(MavenSession.class, "session"); private static final MethodHandle LOOKUP_FIELD = + METHOD_HANDLES.privateFieldGetter("org.apache.maven.internal.impl.AbstractSession", "lookup"); + private static final MethodHandle ALTERNATIVE_LOOKUP_FIELD = METHOD_HANDLES.privateFieldGetter("org.apache.maven.internal.impl.DefaultSession", "lookup"); private static final MethodHandle LOOKUP_METHOD = METHOD_HANDLES.method("org.apache.maven.api.services.Lookup", "lookup", Class.class); @@ -301,8 +303,12 @@ public static PlexusContainer getContainer(MavenSession mavenSession) { } Object /* org.apache.maven.internal.impl.DefaultSession */ session = METHOD_HANDLES.invoke(SESSION_FIELD, mavenSession); - Object /* org.apache.maven.api.services.Lookup */ lookup = - METHOD_HANDLES.invoke(LOOKUP_FIELD, session); + Object /* org.apache.maven.api.services.Lookup */ lookup; + if (LOOKUP_FIELD != null) { + lookup = METHOD_HANDLES.invoke(LOOKUP_FIELD, session); + } else { + lookup = METHOD_HANDLES.invoke(ALTERNATIVE_LOOKUP_FIELD, session); + } return METHOD_HANDLES.invoke(LOOKUP_METHOD, lookup, PlexusContainer.class); } } diff --git a/dd-smoke-tests/backend-mock/build.gradle b/dd-smoke-tests/backend-mock/build.gradle new file mode 100644 index 00000000000..76f6e1754a6 --- /dev/null +++ b/dd-smoke-tests/backend-mock/build.gradle @@ -0,0 +1,7 @@ +apply from: "$rootDir/gradle/java.gradle" +description = 'Mock Datadog backend used by smoke tests.' + +dependencies { + api project(':dd-smoke-tests') + api testFixtures(project(':dd-java-agent:agent-ci-visibility')) +} diff --git a/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy new file mode 100644 index 00000000000..ef189aaf16d --- /dev/null +++ b/dd-smoke-tests/backend-mock/src/main/groovy/datadog/smoketest/MockBackend.groovy @@ -0,0 +1,263 @@ +package datadog.smoketest + +import com.fasterxml.jackson.databind.ObjectMapper +import datadog.trace.agent.test.server.http.TestHttpServer +import datadog.trace.test.util.MultipartRequestParser +import org.apache.commons.io.IOUtils +import org.msgpack.jackson.dataformat.MessagePackFactory +import spock.util.concurrent.PollingConditions + +import java.util.concurrent.ConcurrentLinkedQueue +import java.util.concurrent.CopyOnWriteArrayList +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream + +import static datadog.trace.agent.test.server.http.TestHttpServer.httpServer + +class MockBackend implements AutoCloseable { + + private static final ObjectMapper MSG_PACK_MAPPER = new ObjectMapper(new MessagePackFactory()) + private static final ObjectMapper JSON_MAPPER = new ObjectMapper() + + private final Queue> receivedTraces = new ConcurrentLinkedQueue<>() + + private final Queue> receivedCoverages = new ConcurrentLinkedQueue<>() + + private final Queue> receivedTelemetryMetrics = new ConcurrentLinkedQueue<>() + + private final Queue> receivedTelemetryDistributions = new ConcurrentLinkedQueue<>() + private final Queue> receivedLogs = new ConcurrentLinkedQueue<>() + + private final Collection> skippableTests = new CopyOnWriteArrayList<>() + private final Collection> flakyTests = new CopyOnWriteArrayList<>() + + private boolean codeCoverageEnabled = true + + private boolean testsSkippingEnabled = true + + private boolean flakyRetriesEnabled = false + + void reset() { + receivedTraces.clear() + receivedCoverages.clear() + receivedTelemetryMetrics.clear() + receivedTelemetryDistributions.clear() + receivedLogs.clear() + + skippableTests.clear() + flakyTests.clear() + } + + @Override + void close() throws Exception { + intakeServer.close() + } + + void givenFlakyRetries(boolean flakyRetries) { + this.flakyRetriesEnabled = flakyRetries + } + + void givenFlakyTest(String module, String suite, String name) { + flakyTests.add(["module": module, "suite": suite, "name": name]) + } + + void givenSkippableTest(String module, String suite, String name) { + skippableTests.add(["module": module, "suite": suite, "name": name]) + } + + String getIntakeUrl() { + return intakeServer.address.toString() + } + + private final TestHttpServer intakeServer = httpServer { + handlers { + prefix("/api/v2/citestcycle") { + def contentEncodingHeader = request.getHeader("Content-Encoding") + def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") + def requestBody = gzipEnabled ? MockBackend.decompress(request.body) : request.body + def decodedEvent = MSG_PACK_MAPPER.readValue(requestBody, Map) + receivedTraces.add(decodedEvent) + + response.status(200).send() + } + + prefix("/api/v2/citestcov") { + def contentEncodingHeader = request.getHeader("Content-Encoding") + def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") + def requestBody = gzipEnabled ? MockBackend.decompress(request.body) : request.body + def parsed = MultipartRequestParser.parseRequest(requestBody, request.headers.get("Content-Type")) + def coverages = parsed.get("coverage1") + for (def coverage : coverages) { + def decodedCoverage = MSG_PACK_MAPPER.readValue(coverage.get(), Map) + receivedCoverages.add(decodedCoverage) + } + + response.status(202).send() + } + + prefix("/api/v2/libraries/tests/services/setting") { + // not compressing settings response on purpose, to better mimic real backend behavior: + // it may choose to compress the response or not based on its size, + // so smaller responses (like those of /setting endpoint) are uncompressed, + // while the larger ones (skippable and flaky test lists) are compressed + response.status(200).send(('{ "data": { "type": "ci_app_tracers_test_service_settings", "id": "uuid", "attributes": { ' + + '"code_coverage": ' + codeCoverageEnabled + + ', "tests_skipping": ' + testsSkippingEnabled + + ', "flaky_test_retries_enabled": ' + flakyRetriesEnabled + '} } }').bytes) + } + + prefix("/api/v2/ci/tests/skippable") { + StringBuilder skippableTestsResponse = new StringBuilder("[") + def i = skippableTests.iterator() + while (i.hasNext()) { + def test = i.next() + skippableTestsResponse.append(""" + { + "id": "${UUID.randomUUID().toString()}", + "type": "test", + "attributes": { + "configurations": { + "test.bundle": "$test.module" + }, + "name": "$test.name", + "suite": "$test.suite" + } + } + """) + if (i.hasNext()) { + skippableTestsResponse.append(',') + } + } + skippableTestsResponse.append("]") + + response.status(200) + .addHeader("Content-Encoding", "gzip") + .send(MockBackend.compress((""" { "data": $skippableTestsResponse } """).bytes)) + } + + prefix("/api/v2/ci/libraries/tests/flaky") { + StringBuilder flakyTestsResponse = new StringBuilder("[") + def i = flakyTests.iterator() + while (i.hasNext()) { + def test = i.next() + flakyTestsResponse.append(""" + { + "id": "${UUID.randomUUID().toString()}", + "type": "test", + "attributes": { + "configurations": { + "test.bundle": "$test.module" + }, + "name": "$test.name", + "suite": "$test.suite" + } + } + """) + if (i.hasNext()) { + flakyTestsResponse.append(',') + } + } + flakyTestsResponse.append("]") + + response.status(200) + .addHeader("Content-Encoding", "gzip") + .send(MockBackend.compress((""" { "data": $flakyTestsResponse } """).bytes)) + } + + prefix("/api/v2/apmtelemetry") { + def telemetryRequest = JSON_MAPPER.readerFor(Map.class).readValue(request.body) + def requestType = telemetryRequest["request_type"] + if (requestType == "message-batch") { + for (def message : telemetryRequest["payload"]) { + def payload = message["payload"] + if (message["request_type"] == 'generate-metrics') { + receivedTelemetryMetrics.addAll((List) payload["series"]) + } else if (message["request_type"] == 'distributions') { + receivedTelemetryDistributions.addAll((List) payload["series"]) + } + } + } + response.status(202).send() + } + + prefix("/api/v2/logs") { + def contentEncodingHeader = request.getHeader("Content-Encoding") + def gzipEnabled = contentEncodingHeader != null && contentEncodingHeader.contains("gzip") + def requestBody = gzipEnabled ? MockBackend.decompress(request.body) : request.body + def decodedEvent = JSON_MAPPER.readValue(requestBody, List) + receivedLogs.addAll(decodedEvent) + + response.status(200).send() + } + } + } + + private static byte[] compress(byte[] bytes) { + def baos = new ByteArrayOutputStream() + try (GZIPOutputStream zip = new GZIPOutputStream(baos)) { + IOUtils.copy(new ByteArrayInputStream(bytes), zip) + } + return baos.toByteArray() + } + + private static byte[] decompress(byte[] bytes) { + def baos = new ByteArrayOutputStream() + try (GZIPInputStream zip = new GZIPInputStream(new ByteArrayInputStream(bytes))) { + IOUtils.copy(zip, baos) + } + return baos.toByteArray() + } + + List> waitForEvents(int expectedEventsSize) { + def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) + traceReceiveConditions.eventually { + int eventsSize = 0 + for (Map trace : receivedTraces) { + eventsSize += trace["events"].size() + } + assert eventsSize >= expectedEventsSize + } + + List> events = new ArrayList<>() + while (!receivedTraces.isEmpty()) { + def trace = receivedTraces.poll() + events.addAll((List>) trace["events"]) + } + return events + } + + List> waitForCoverages(int traceSize) { + def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) + traceReceiveConditions.eventually { + assert receivedCoverages.size() == traceSize + } + + List> coverages = new ArrayList<>() + while (!receivedCoverages.isEmpty()) { + def trace = receivedCoverages.poll() + coverages.addAll((List>) trace["coverages"]) + } + return coverages + } + + List> waitForLogs(int expectedCount) { + def traceReceiveConditions = new PollingConditions(timeout: 15, initialDelay: 1, delay: 0.5, factor: 1) + traceReceiveConditions.eventually { + assert receivedLogs.size() == expectedCount + } + + List> logs = new ArrayList<>() + while (!receivedLogs.isEmpty()) { + logs.add(receivedLogs.poll()) + } + return logs + } + + List> getAllReceivedTelemetryMetrics() { + return new ArrayList(receivedTelemetryMetrics) + } + + List> getAllReceivedTelemetryDistributions() { + return new ArrayList(receivedTelemetryDistributions) + } +} diff --git a/dd-smoke-tests/gradle/build.gradle b/dd-smoke-tests/gradle/build.gradle index a16b49b635e..8063eccd959 100644 --- a/dd-smoke-tests/gradle/build.gradle +++ b/dd-smoke-tests/gradle/build.gradle @@ -6,9 +6,8 @@ apply from: "$rootDir/gradle/java.gradle" description = 'Gradle Daemon Instrumentation Smoke Tests.' dependencies { - testImplementation project(':dd-smoke-tests') testImplementation gradleTestKit() - testImplementation testFixtures(project(':dd-java-agent:agent-ci-visibility')) + testImplementation project(':dd-smoke-tests:backend-mock') } test { diff --git a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy b/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy index 29c686d2e08..3392e62a284 100644 --- a/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy +++ b/dd-smoke-tests/gradle/src/test/groovy/datadog/smoketest/GradleDaemonSmokeTest.groovy @@ -24,6 +24,7 @@ import org.gradle.wrapper.Install import org.gradle.wrapper.PathAssembler import org.gradle.wrapper.WrapperConfiguration import org.junit.jupiter.api.Assumptions +import spock.lang.AutoCleanup import spock.lang.Shared import spock.lang.TempDir import spock.util.environment.Jvm @@ -60,24 +61,36 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { @TempDir Path projectFolder + @Shared + @AutoCleanup + MockBackend mockBackend = new MockBackend() + def setupSpec() { givenGradleProperties() } + def setup() { + mockBackend.reset() + } + @Flaky("https://github.com/DataDog/dd-trace-java/issues/7024") def "test #projectName, v#gradleVersion, configCache: #configurationCache"() { givenGradleVersionIsCompatibleWithCurrentJvm(gradleVersion) givenConfigurationCacheIsCompatibleWithCurrentPlatform(configurationCache) givenGradleProjectFiles(projectName) - givenFlakyRetriesEnabled(flakyRetries) ensureDependenciesDownloaded(gradleVersion) + mockBackend.givenFlakyRetries(flakyRetries) + mockBackend.givenFlakyTest(":test", "datadog.smoke.TestFailed", "test_failed") + mockBackend.givenSkippableTest(":test", "datadog.smoke.TestSucceed", "test_to_skip_with_itr") + BuildResult buildResult = runGradleTests(gradleVersion, successExpected, configurationCache) if (successExpected) { assertBuildSuccessful(buildResult) } - verifyEventsAndCoverages(projectName, "gradle", gradleVersion, expectedTraces, expectedCoverages) + + verifyEventsAndCoverages(projectName, "gradle", gradleVersion, mockBackend.waitForEvents(expectedTraces), mockBackend.waitForCoverages(expectedCoverages)) if (configurationCache) { // if configuration cache is enabled, run the build one more time @@ -85,7 +98,7 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { BuildResult buildResultWithConfigCacheEntry = runGradleTests(gradleVersion, successExpected, configurationCache) assertBuildSuccessful(buildResultWithConfigCacheEntry) - verifyEventsAndCoverages(projectName, "gradle", gradleVersion, expectedTraces, expectedCoverages) + verifyEventsAndCoverages(projectName, "gradle", gradleVersion, mockBackend.waitForEvents(expectedTraces), mockBackend.waitForCoverages(expectedCoverages)) } where: @@ -136,7 +149,7 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_CIPROVIDER_INTEGRATION_ENABLED)}=false," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION)}=$JACOCO_PLUGIN_VERSION," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_SEGMENTS_ENABLED)}=true," + - "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${intakeServer.address.toString()}" + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${mockBackend.intakeUrl}" Files.write(testKitFolder.resolve("gradle.properties"), gradleProperties.getBytes()) } @@ -290,8 +303,4 @@ class GradleDaemonSmokeTest extends CiVisibilitySmokeTest { return root.get("version").asText() } } - - private void givenFlakyRetriesEnabled(boolean flakyRetries) { - this.flakyRetriesEnabled = flakyRetries - } } diff --git a/dd-smoke-tests/log-injection/build.gradle b/dd-smoke-tests/log-injection/build.gradle index e4583414bc1..e5fe596af8c 100644 --- a/dd-smoke-tests/log-injection/build.gradle +++ b/dd-smoke-tests/log-injection/build.gradle @@ -123,6 +123,7 @@ dependencies { floggerSlf4jBackend 'com.google.flogger:flogger-slf4j-backend:0.5.1' testImplementation project(':dd-smoke-tests') + testImplementation project(':dd-smoke-tests:backend-mock') } def generateTestingJar(String interfaceName, String backend, List configurationsList) { diff --git a/dd-smoke-tests/log-injection/src/test/groovy/datadog/smoketest/LogInjectionSmokeTest.groovy b/dd-smoke-tests/log-injection/src/test/groovy/datadog/smoketest/LogInjectionSmokeTest.groovy index 17c75fa47c8..6c9a15fd036 100644 --- a/dd-smoke-tests/log-injection/src/test/groovy/datadog/smoketest/LogInjectionSmokeTest.groovy +++ b/dd-smoke-tests/log-injection/src/test/groovy/datadog/smoketest/LogInjectionSmokeTest.groovy @@ -2,6 +2,9 @@ package datadog.smoketest import com.squareup.moshi.Moshi import com.squareup.moshi.Types +import datadog.trace.api.config.CiVisibilityConfig +import datadog.trace.api.config.GeneralConfig +import spock.lang.AutoCleanup import spock.lang.Shared import static datadog.trace.api.config.TraceInstrumentationConfig.TRACE_128_BIT_TRACEID_LOGGING_ENABLED @@ -23,6 +26,8 @@ abstract class LogInjectionSmokeTest extends AbstractSmokeTest { // Estimate for the amount of time instrumentation, plus request, plus some extra static final int TIMEOUT_SECS = 30 + static final String LOG4J2_BACKEND = "Log4j2" + @Shared File outputLogFile @@ -37,6 +42,14 @@ abstract class LogInjectionSmokeTest extends AbstractSmokeTest { @Shared boolean trace128bits = false + @Shared + @AutoCleanup + MockBackend mockBackend = new MockBackend() + + def setup() { + mockBackend.reset() + } + @Override ProcessBuilder createProcessBuilder() { def jarName = getClass().simpleName @@ -78,6 +91,12 @@ abstract class LogInjectionSmokeTest extends AbstractSmokeTest { command.add("-Ddd.$TRACE_128_BIT_TRACEID_GENERATION_ENABLED=true" as String) command.add("-Ddd.$TRACE_128_BIT_TRACEID_LOGGING_ENABLED=true" as String) } + if (supportsDirectLogSubmission()) { + // currently direct log submission only works in CI Visibility agentless mode + command.add("-Ddd.$CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED=true" as String) + command.add("-Ddd.$CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL=${mockBackend.intakeUrl}" as String) + command.add("-Ddd.$GeneralConfig.AGENTLESS_LOG_SUBMISSION_URL=${mockBackend.intakeUrl}" as String) + } command.addAll(additionalArguments()) command.addAll((String[]) ["-jar", loggingJar]) @@ -105,6 +124,10 @@ abstract class LogInjectionSmokeTest extends AbstractSmokeTest { return true } + def supportsDirectLogSubmission() { + return backend() == LOG4J2_BACKEND + } + abstract backend() def cleanupSpec() { @@ -157,36 +180,40 @@ abstract class LogInjectionSmokeTest extends AbstractSmokeTest { if (!getClass().simpleName.contains("Log4j2Backend")) { assert logLines.every { it["backend"] == backend() } } + return assertParsedJsonLinesWithInjection(logLines, firstTraceId, firstSpanId, secondTraceId, secondSpanId, thirdTraceId, thirdSpanId, forthTraceId, forthSpanId) + } + + private assertParsedJsonLinesWithInjection(List logLines, String firstTraceId, String firstSpanId, String secondTraceId, String secondSpanId, String thirdTraceId, String thirdSpanId, String forthTraceId, String forthSpanId) { assert logLines.every { getFromContext(it, "dd.service") == noTags ? null : SERVICE_NAME } - assert logLines.every { getFromContext(it,"dd.version") == noTags ? null : VERSION } - assert logLines.every { getFromContext(it,"dd.env") == noTags ? null : ENV } + assert logLines.every { getFromContext(it, "dd.version") == noTags ? null : VERSION } + assert logLines.every { getFromContext(it, "dd.env") == noTags ? null : ENV } - assert getFromContext(logLines[0],"dd.trace_id") == null - assert getFromContext(logLines[0],"dd.span_id") == null + assert getFromContext(logLines[0], "dd.trace_id") == null + assert getFromContext(logLines[0], "dd.span_id") == null assert logLines[0]["message"] == "BEFORE FIRST SPAN" assert getFromContext(logLines[1], "dd.trace_id") == firstTraceId assert getFromContext(logLines[1], "dd.span_id") == firstSpanId assert logLines[1]["message"] == "INSIDE FIRST SPAN" - assert getFromContext(logLines[2],"dd.trace_id") == null - assert getFromContext(logLines[2],"dd.span_id") == null + assert getFromContext(logLines[2], "dd.trace_id") == null + assert getFromContext(logLines[2], "dd.span_id") == null assert logLines[2]["message"] == "AFTER FIRST SPAN" assert getFromContext(logLines[3], "dd.trace_id") == secondTraceId - assert getFromContext(logLines[3], "dd.span_id") == secondSpanId + assert getFromContext(logLines[3], "dd.span_id") == secondSpanId assert logLines[3]["message"] == "INSIDE SECOND SPAN" assert getFromContext(logLines[4], "dd.trace_id") == null - assert getFromContext(logLines[4], "dd.span_id") == null + assert getFromContext(logLines[4], "dd.span_id") == null assert logLines[4]["message"] == "INSIDE THIRD SPAN" assert getFromContext(logLines[5], "dd.trace_id") == forthTraceId - assert getFromContext(logLines[5], "dd.span_id") == forthSpanId + assert getFromContext(logLines[5], "dd.span_id") == forthSpanId assert logLines[5]["message"] == "INSIDE FORTH SPAN" assert getFromContext(logLines[6], "dd.trace_id") == null - assert getFromContext(logLines[6], "dd.span_id") == null + assert getFromContext(logLines[6], "dd.span_id") == null assert logLines[6]["message"] == "AFTER FORTH SPAN" return true @@ -272,6 +299,10 @@ abstract class LogInjectionSmokeTest extends AbstractSmokeTest { if (supportsJson()) { assertJsonLinesWithInjection(jsonLogLines, firstTraceId, firstSpanId, secondTraceId, secondSpanId, thirdTraceId, thirdSpanId, forthTraceId, forthSpanId) } + + if (supportsDirectLogSubmission()) { + assertParsedJsonLinesWithInjection(mockBackend.waitForLogs(7), firstTraceId, firstSpanId, secondTraceId, secondSpanId, thirdTraceId, thirdSpanId, forthTraceId, forthSpanId) + } } void checkTraceIdFormat(String traceId) { @@ -323,7 +354,7 @@ class JULInterfaceJULBackend extends JULBackend { } class JULInterfaceLog4j2Backend extends LogInjectionSmokeTest { - def backend() { "Log4j2" } + def backend() { LOG4J2_BACKEND } List additionalArguments() { return ["-Djava.util.logging.manager=org.apache.logging.log4j.jul.LogManager"] @@ -361,7 +392,7 @@ class JCLInterfaceLog4j1LatestBackend extends JCLInterfaceLog4j1Backend {} class JCLInterfaceLog4j1Backend128bTid extends JCLInterfaceLog4j1Backend {} class JCLInterfaceLog4j2Backend extends LogInjectionSmokeTest { - def backend() { "Log4j2" } + def backend() { LOG4J2_BACKEND } // workaround https://github.com/apache/logging-log4j2/issues/1865 List additionalArguments() { @@ -383,7 +414,7 @@ class Log4j1InterfaceLog4j1Backend128bTid extends Log4j1InterfaceLog4j1Backend { class Log4j1InterfaceLog4j1LatestBackend extends Log4j1InterfaceLog4j1Backend {} class Log4j1InterfaceLog4j2Backend extends LogInjectionSmokeTest { - def backend() { "Log4j2" } + def backend() { LOG4J2_BACKEND } } class Log4j1InterfaceLog4j2BackendNoTags extends Log4j1InterfaceLog4j2Backend {} @@ -391,7 +422,7 @@ class Log4j1InterfaceLog4j2Backend128bTid extends Log4j1InterfaceLog4j2Backend { class Log4j1InterfaceLog4j2LatestBackend extends Log4j1InterfaceLog4j2Backend {} class Log4j2InterfaceLog4j2Backend extends LogInjectionSmokeTest { - def backend() { "Log4j2" } + def backend() { LOG4J2_BACKEND } } class Log4j2InterfaceLog4j2BackendNoTags extends Log4j2InterfaceLog4j2Backend {} @@ -416,7 +447,7 @@ class Slf4jInterfaceLog4j1Backend128bTid extends Slf4jInterfaceLog4j1Backend {} class Slf4jInterfaceLog4j1LatestBackend extends Slf4jInterfaceLog4j1Backend {} class Slf4jInterfaceLog4j2Backend extends LogInjectionSmokeTest { - def backend() { "Log4j2" } + def backend() { LOG4J2_BACKEND } } class Slf4jInterfaceLog4j2BackendNoTags extends Slf4jInterfaceLog4j2Backend {} @@ -446,7 +477,7 @@ class Slf4jInterfaceJCLToLog4j1Backend128bTid extends Slf4jInterfaceJCLToLog4j1B class Slf4jInterfaceJCLToLog4j1LatestBackend extends Slf4jInterfaceJCLToLog4j1Backend {} class Slf4jInterfaceJCLToLog4j2Backend extends LogInjectionSmokeTest { - def backend() { "Log4j2" } + def backend() { LOG4J2_BACKEND } // workaround https://github.com/apache/logging-log4j2/issues/1865 List additionalArguments() { @@ -533,7 +564,7 @@ class JBossInterfaceLog4j1Backend128bTid extends JBossInterfaceLog4j1Backend {} class JBossInterfaceLog4j1LatestBackend extends JBossInterfaceLog4j1Backend {} class JBossInterfaceLog4j2Backend extends LogInjectionSmokeTest { - def backend() { "Log4j2" } + def backend() { LOG4J2_BACKEND } } class JBossInterfaceLog4j2BackendNoTags extends JBossInterfaceLog4j2Backend {} diff --git a/dd-smoke-tests/maven/build.gradle b/dd-smoke-tests/maven/build.gradle index 4a39d9a60b8..2d352f03411 100644 --- a/dd-smoke-tests/maven/build.gradle +++ b/dd-smoke-tests/maven/build.gradle @@ -8,8 +8,7 @@ description = 'Maven Instrumentation Smoke Tests.' dependencies { implementation group: 'org.apache.maven.wrapper', name: 'maven-wrapper', version: '3.2.0' - testImplementation project(':dd-smoke-tests') - testImplementation testFixtures(project(':dd-java-agent:agent-ci-visibility')) + testImplementation project(':dd-smoke-tests:backend-mock') } jar { diff --git a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy index 765ceb2bf5a..c4aa3138710 100644 --- a/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy +++ b/dd-smoke-tests/maven/src/test/groovy/datadog/smoketest/MavenSmokeTest.groovy @@ -11,11 +11,15 @@ import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.Response import org.apache.maven.wrapper.MavenWrapperMain +import org.junit.jupiter.api.Assumptions import org.slf4j.Logger import org.slf4j.LoggerFactory import org.w3c.dom.Document import org.w3c.dom.NodeList +import spock.lang.AutoCleanup +import spock.lang.Shared import spock.lang.TempDir +import spock.util.environment.Jvm import javax.xml.parsers.DocumentBuilder import javax.xml.parsers.DocumentBuilderFactory @@ -46,12 +50,26 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { @TempDir Path projectHome + @Shared + @AutoCleanup + MockBackend mockBackend = new MockBackend() + + def setup() { + mockBackend.reset() + } + @Flaky("https://github.com/DataDog/dd-trace-java/issues/7025") def "test #projectName, v#mavenVersion"() { + Assumptions.assumeTrue(Jvm.current.isJavaVersionCompatible(minSupportedJavaVersion), + "Current JVM " + Jvm.current.javaVersion + " is not compatible with minimum required version " + minSupportedJavaVersion) + givenWrapperPropertiesFile(mavenVersion) givenMavenProjectFiles(projectName) givenMavenDependenciesAreLoaded(projectName, mavenVersion) - givenFlakyRetries(flakyRetries) + + mockBackend.givenFlakyRetries(flakyRetries) + mockBackend.givenFlakyTest("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestFailed", "test_failed") + mockBackend.givenSkippableTest("Maven Smoke Tests Project maven-surefire-plugin default-test", "datadog.smoke.TestSucceed", "test_to_skip_with_itr") def exitCode = whenRunningMavenBuild(jacocoCoverage) @@ -60,24 +78,25 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { } else { assert exitCode != 0 } - verifyEventsAndCoverages(projectName, "maven", mavenVersion, expectedEvents, expectedCoverages) - verifyTelemetryMetrics(expectedEvents) + + verifyEventsAndCoverages(projectName, "maven", mavenVersion, mockBackend.waitForEvents(expectedEvents), mockBackend.waitForCoverages(expectedCoverages)) + verifyTelemetryMetrics(mockBackend.getAllReceivedTelemetryMetrics(), mockBackend.getAllReceivedTelemetryDistributions(), expectedEvents) where: - projectName | mavenVersion | expectedEvents | expectedCoverages | expectSuccess | flakyRetries | jacocoCoverage - "test_successful_maven_run" | "3.2.1" | 5 | 1 | true | false | true - "test_successful_maven_run" | "3.5.4" | 5 | 1 | true | false | true - "test_successful_maven_run" | "3.6.3" | 5 | 1 | true | false | true - "test_successful_maven_run" | "3.8.8" | 5 | 1 | true | false | true - "test_successful_maven_run" | "3.9.6" | 5 | 1 | true | false | true - "test_successful_maven_run_surefire_3_0_0" | "3.9.6" | 5 | 1 | true | false | true - "test_successful_maven_run_surefire_3_0_0" | LATEST_MAVEN_VERSION | 5 | 1 | true | false | true - "test_successful_maven_run_builtin_coverage" | "3.9.6" | 5 | 1 | true | false | false - "test_successful_maven_run_with_jacoco_and_argline" | "3.9.6" | 5 | 1 | true | false | true + projectName | mavenVersion | expectedEvents | expectedCoverages | expectSuccess | flakyRetries | jacocoCoverage | minSupportedJavaVersion + "test_successful_maven_run" | "3.2.1" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run" | "3.5.4" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run" | "3.6.3" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run" | "3.8.8" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run" | "3.9.7" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run_surefire_3_0_0" | "3.9.7" | 5 | 1 | true | false | true | 8 + "test_successful_maven_run_surefire_3_0_0" | LATEST_MAVEN_VERSION | 5 | 1 | true | false | true | 17 + "test_successful_maven_run_builtin_coverage" | "3.9.7" | 5 | 1 | true | false | false | 8 + "test_successful_maven_run_with_jacoco_and_argline" | "3.9.7" | 5 | 1 | true | false | true | 8 // "expectedEvents" count for this test case does not include the spans that correspond to Cucumber steps - "test_successful_maven_run_with_cucumber" | "3.9.6" | 4 | 1 | true | false | true - "test_failed_maven_run_flaky_retries" | "3.9.6" | 8 | 1 | false | true | true - "test_successful_maven_run_junit_platform_runner" | "3.9.6" | 4 | 0 | true | false | false + "test_successful_maven_run_with_cucumber" | "3.9.7" | 4 | 1 | true | false | true | 8 + "test_failed_maven_run_flaky_retries" | "3.9.7" | 8 | 1 | false | true | true | 8 + "test_successful_maven_run_junit_platform_runner" | "3.9.7" | 4 | 0 | true | false | false | 8 } private void givenWrapperPropertiesFile(String mavenVersion) { @@ -223,8 +242,9 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_SOURCE_DATA_ROOT_CHECK_ENABLED)}=false," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_GIT_UPLOAD_ENABLED)}=false," + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_COMPILER_PLUGIN_VERSION)}=${JAVAC_PLUGIN_VERSION}," + - "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${intakeServer.address.toString()}," + - "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ONLY_KNOWN_FLAKES)}=true," // to cover flaky test retrieval logic + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_URL)}=${mockBackend.intakeUrl}," + + "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_FLAKY_RETRY_ONLY_KNOWN_FLAKES)}=true," + // to cover flaky test retrieval logic if (injectJacoco) { agentArgument += "${Strings.propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_CODE_COVERAGE_SEGMENTS_ENABLED)}=true," + @@ -240,10 +260,6 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { return [projectHome.toAbsolutePath().toString()] } - void givenFlakyRetries(boolean flakyRetries) { - this.flakyRetriesEnabled = flakyRetries - } - private static class StreamConsumer extends Thread { final InputStream is final String messagePrefix @@ -280,7 +296,7 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { NodeList versionList = doc.getElementsByTagName("latest") if (versionList.getLength() > 0) { def version = versionList.item(0).getTextContent() - if (!version.contains('alpha')) { + if (!version.contains('alpha') && !version.contains('beta')) { LOGGER.info("Will run the 'latest' tests with version ${version}") return version } @@ -291,7 +307,7 @@ class MavenSmokeTest extends CiVisibilitySmokeTest { } catch (Exception e) { LOGGER.warn("Could not get latest maven version", e) } - def hardcodedLatestVersion = "4.0.0-alpha-12" // latest alpha that is known to work + def hardcodedLatestVersion = "4.0.0-beta-3" // latest version that is known to work LOGGER.info("Will run the 'latest' tests with hard-coded version ${hardcodedLatestVersion}") return hardcodedLatestVersion } diff --git a/settings.gradle b/settings.gradle index e43ec454e67..ab21ec01afc 100644 --- a/settings.gradle +++ b/settings.gradle @@ -86,6 +86,7 @@ include ':utils:version-utils' // smoke tests include ':dd-smoke-tests:armeria-grpc' +include ':dd-smoke-tests:backend-mock' include ':dd-smoke-tests:cli' include ':dd-smoke-tests:custom-systemloader' include ':dd-smoke-tests:dynamic-config'