From 50a5b2e2e3e2f62c3b28e4a7c8b98e915cc4aa0e Mon Sep 17 00:00:00 2001 From: Nikita Tkachenko Date: Fri, 5 Jul 2024 14:59:11 +0200 Subject: [PATCH] Inject coverage store field into Thread class --- .../trace/civisibility/config/JvmInfo.java | 62 ++++++++- .../config/JvmInfoFactoryImpl.java | 8 +- .../ModuleExecutionSettingsFactoryImpl.java | 49 +++++-- .../ModuleExecutionSettingsSerializer.java | 4 + .../store/CoreJvmClassReader.java | 66 +++++++++ .../store/CoverageStoreFieldInjector.java | 25 ++++ .../instrumentation/store/JvmPatcher.java | 61 +++++++++ .../trace/civisibility/domain/TestImpl.java | 4 +- .../buildsystem/BuildSystemSessionImpl.java | 1 + .../civisibility/git/tree/GitClient.java | 2 +- .../git/tree/GitDataUploaderImpl.java | 2 +- .../trace/civisibility/utils/FileUtils.java | 42 +++--- .../config/JvmInfoFactoryTest.groovy | 2 +- ...duleExecutionSettingsSerializerTest.groovy | 24 +++- .../store/CoreJvmClassReaderTest.groovy | 20 +++ .../store/JvmPatcherTest.groovy | 26 ++++ .../ipc/ModuleSettingsRequestTest.groovy | 20 +-- .../civisibility/utils/FileUtilsTest.groovy | 2 +- .../CiVisibilityInstrumentationTest.groovy | 1 + .../gradle/CiVisibilityService.java | 5 +- .../gradle/legacy/GradleBuildListener.java | 4 +- .../legacy/GradleProjectConfigurator.groovy | 4 +- .../maven3/MavenLifecycleParticipant.java | 4 +- .../maven3/MavenProjectConfigurator.java | 8 +- .../trace/api/config/CiVisibilityConfig.java | 2 + .../main/java/datadog/trace/api/Config.java | 8 ++ .../config/ModuleExecutionSettings.java | 10 ++ .../civisibility/coverage/CoverageBridge.java | 128 ++++++++++++++---- 28 files changed, 507 insertions(+), 87 deletions(-) create mode 100644 dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/CoreJvmClassReader.java create mode 100644 dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/CoverageStoreFieldInjector.java create mode 100644 dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/JvmPatcher.java create mode 100644 dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/instrumentation/store/CoreJvmClassReaderTest.groovy create mode 100644 dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/instrumentation/store/JvmPatcherTest.groovy diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfo.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfo.java index ab0733a8a48..e80f6f4ff2d 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfo.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfo.java @@ -2,6 +2,8 @@ import datadog.trace.civisibility.ipc.Serializer; import java.nio.ByteBuffer; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Objects; public class JvmInfo { @@ -10,16 +12,40 @@ public class JvmInfo { new JvmInfo( System.getProperty("java.runtime.name"), System.getProperty("java.version"), - System.getProperty("java.vendor")); + System.getProperty("java.class.version"), + System.getProperty("java.vendor"), + System.getProperty("java.home")); + + private static final int JAVA_8_CLASS_VERSION = 52; private final String name; private final String version; + private final int majorClassVersion; private final String vendor; + private final Path home; + + public JvmInfo(String name, String version, String classVersion, String vendor, String home) { + this.name = name; + this.version = version; + this.vendor = vendor; + this.home = Paths.get(home); + + int majorClassVersion; + try { + String[] classVersionTokens = classVersion.split("\\."); + majorClassVersion = Integer.parseInt(classVersionTokens[0]); + } catch (Exception e) { + majorClassVersion = -1; + } + this.majorClassVersion = majorClassVersion; + } - public JvmInfo(String name, String version, String vendor) { + private JvmInfo(String name, String version, int majorClassVersion, String vendor, Path home) { this.name = name; this.version = version; this.vendor = vendor; + this.home = home; + this.majorClassVersion = majorClassVersion; } public String getName() { @@ -30,10 +56,22 @@ public String getVersion() { return version; } + public int getMajorClassVersion() { + return majorClassVersion; + } + public String getVendor() { return vendor; } + public Path getHome() { + return home; + } + + public boolean isModular() { + return majorClassVersion > JAVA_8_CLASS_VERSION; + } + @Override public boolean equals(Object o) { if (this == o) { @@ -45,12 +83,14 @@ public boolean equals(Object o) { JvmInfo jvmInfo = (JvmInfo) o; return Objects.equals(name, jvmInfo.name) && Objects.equals(version, jvmInfo.version) - && Objects.equals(vendor, jvmInfo.vendor); + && Objects.equals(majorClassVersion, jvmInfo.majorClassVersion) + && Objects.equals(vendor, jvmInfo.vendor) + && Objects.equals(home, jvmInfo.home); } @Override public int hashCode() { - return Objects.hash(name, version, vendor); + return Objects.hash(name, version, majorClassVersion, vendor, home); } @Override @@ -61,21 +101,33 @@ public String toString() { + '\'' + ", version='" + version + + ", majorClassVersion='" + + majorClassVersion + '\'' + ", vendor='" + vendor + '\'' + + ", home='" + + home + + '\'' + '}'; } public static void serialize(Serializer serializer, JvmInfo jvmInfo) { serializer.write(jvmInfo.name); serializer.write(jvmInfo.version); + serializer.write(jvmInfo.majorClassVersion); serializer.write(jvmInfo.vendor); + serializer.write(String.valueOf(jvmInfo.home)); } public static JvmInfo deserialize(ByteBuffer buf) { + String name = Serializer.readString(buf); + String version = Serializer.readString(buf); + int majorClassVersion = Serializer.readInt(buf); + String vendor = Serializer.readString(buf); + String home = Serializer.readString(buf); return new JvmInfo( - Serializer.readString(buf), Serializer.readString(buf), Serializer.readString(buf)); + name, version, majorClassVersion, vendor, home != null ? Paths.get(home) : null); } } diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfoFactoryImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfoFactoryImpl.java index 0a3a4a28519..01894e9fcd4 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfoFactoryImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/JvmInfoFactoryImpl.java @@ -64,7 +64,9 @@ private static final class JvmVersionOutputParser public JvmInfo parse(InputStream inputStream) throws IOException { String name = null; String version = null; + String classVersion = null; String vendor = null; + String home = null; BufferedReader bis = new BufferedReader(new InputStreamReader(inputStream, Charset.defaultCharset())); @@ -74,11 +76,15 @@ public JvmInfo parse(InputStream inputStream) throws IOException { name = getPropertyValue(line); } else if (line.contains("java.version ")) { version = getPropertyValue(line); + } else if (line.contains("java.class.version ")) { + classVersion = getPropertyValue(line); } else if (line.contains("java.vendor ")) { vendor = getPropertyValue(line); + } else if (line.contains("java.home ")) { + home = getPropertyValue(line); } } - return new JvmInfo(name, version, vendor); + return new JvmInfo(name, version, classVersion, vendor, home); } private String getPropertyValue(String line) { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java index 7ce67824d55..0d90e232593 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsFactoryImpl.java @@ -9,13 +9,16 @@ import datadog.trace.api.config.CiVisibilityConfig; import datadog.trace.api.git.GitInfo; import datadog.trace.api.git.GitInfoProvider; +import datadog.trace.civisibility.coverage.instrumentation.store.JvmPatcher; import datadog.trace.civisibility.git.tree.GitDataUploader; import datadog.trace.civisibility.source.index.RepoIndex; import datadog.trace.civisibility.source.index.RepoIndexProvider; +import datadog.trace.civisibility.utils.FileUtils; import datadog.trace.util.Strings; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; @@ -65,14 +68,6 @@ public ModuleExecutionSettings create(JvmInfo jvmInfo, @Nullable String moduleNa boolean testSkippingEnabled = isTestSkippingEnabled(ciVisibilitySettings); boolean flakyTestRetriesEnabled = isFlakyTestRetriesEnabled(ciVisibilitySettings); boolean earlyFlakeDetectionEnabled = isEarlyFlakeDetectionEnabled(ciVisibilitySettings); - Map systemProperties = - getPropertiesPropagatedToChildProcess( - itrEnabled, - codeCoverageEnabled, - testSkippingEnabled, - flakyTestRetriesEnabled, - earlyFlakeDetectionEnabled); - LOGGER.info( "CI Visibility settings ({}, {}):\n" + "Intelligent Test Runner - {},\n" @@ -109,6 +104,19 @@ public ModuleExecutionSettings create(JvmInfo jvmInfo, @Nullable String moduleNa Map> knownTestsByModuleName = earlyFlakeDetectionEnabled ? getKnownTests(tracerEnvironment) : null; + Map systemProperties = + getPropertiesPropagatedToChildProcess( + itrEnabled, + codeCoverageEnabled, + testSkippingEnabled, + flakyTestRetriesEnabled, + earlyFlakeDetectionEnabled); + + List jvmOptions = new ArrayList<>(); + if (codeCoverageEnabled && config.isCiVisibilityJvmPatchingEnabled()) { + jvmOptions.addAll(patchJvm(jvmInfo)); + } + List coverageEnabledPackages = getCoverageEnabledPackages(codeCoverageEnabled); return new ModuleExecutionSettings( itrEnabled, @@ -123,6 +131,7 @@ public ModuleExecutionSettings create(JvmInfo jvmInfo, @Nullable String moduleNa ? ciVisibilitySettings.getEarlyFlakeDetectionSettings() : EarlyFlakeDetectionSettings.DEFAULT, systemProperties, + jvmOptions, itrCorrelationId, skippableTestsByModuleName, flakyTests, @@ -130,6 +139,30 @@ public ModuleExecutionSettings create(JvmInfo jvmInfo, @Nullable String moduleNa coverageEnabledPackages); } + /** + * @return The list of JVM options that are needed to prepend the patch to the bootstrap classpath + */ + private List patchJvm(JvmInfo jvmInfo) { + try { + JvmPatcher patcher = new JvmPatcher(jvmInfo); + Path patchPath = patcher.createPatch(); + + Thread cleanupPatch = + new Thread(() -> FileUtils.deleteSafely(patchPath), "dd-ci-vis-patched-classes-remove"); + Runtime.getRuntime().addShutdownHook(cleanupPatch); + + if (jvmInfo.isModular()) { + return Arrays.asList("--patch-module", String.format("java.base=%s", patchPath)); + } else { + return Collections.singletonList(String.format("-Xbootclasspath/p:%s", patchPath)); + } + + } catch (Exception e) { + LOGGER.debug("Failed to patch {}", jvmInfo, e); + return Collections.emptyList(); + } + } + private TracerEnvironment buildTracerEnvironment( String repositoryRoot, JvmInfo jvmInfo, @Nullable String moduleName) { GitInfo gitInfo = GitInfoProvider.INSTANCE.getGitInfo(repositoryRoot); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializer.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializer.java index ab2697f54bc..d3dbdd3e676 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializer.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializer.java @@ -30,6 +30,7 @@ public static ByteBuffer serialize(ModuleExecutionSettings settings) { EarlyFlakeDetectionSettingsSerializer.serialize(s, settings.getEarlyFlakeDetectionSettings()); s.write(settings.getSystemProperties()); + s.write(settings.getJvmOptions()); s.write(settings.getItrCorrelationId()); s.write( settings.getSkippableTestsByModule(), @@ -56,6 +57,8 @@ public static ModuleExecutionSettings deserialize(ByteBuffer buffer) { EarlyFlakeDetectionSettingsSerializer.deserialize(buffer); Map systemProperties = Serializer.readStringMap(buffer); + List jvmOptions = Serializer.readStringList(buffer); + String itrCorrelationId = Serializer.readString(buffer); Map> skippableTestsByModule = Serializer.readMap( @@ -78,6 +81,7 @@ public static ModuleExecutionSettings deserialize(ByteBuffer buffer) { flakyTestRetriesEnabled, earlyFlakeDetectionSettings, systemProperties, + jvmOptions, itrCorrelationId, skippableTestsByModule, flakyTests, diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/CoreJvmClassReader.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/CoreJvmClassReader.java new file mode 100644 index 00000000000..c03e3a78e39 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/CoreJvmClassReader.java @@ -0,0 +1,66 @@ +package datadog.trace.civisibility.coverage.instrumentation.store; + +import datadog.communication.util.IOThrowingFunction; +import datadog.trace.civisibility.config.JvmInfo; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; +import java.util.zip.ZipEntry; +import java.util.zip.ZipFile; + +public class CoreJvmClassReader { + + private static final String JAVA_BASE_MODULE_RELATIVE_PATH = "jmods/java.base.jmod"; + + // java.home contains the "/jre" segment for Java 8 distributions + private static final String RT_JAR_RELATIVE_PATH = "lib/rt.jar"; + + /** + * Performs an action on the bytecode of a core JVM class + * + * @param jvm The JVM distribution + * @param className The name of the class (as returned by {@link Class#getName}, e.g. {@code + * java.lang.Thread}) + * @param action The action to be performed on the class bytecode stream + * @return The return value of the action + * @param The type of the return value + * @throws IOException If the stream could not be retrieved + */ + public T withClassStream( + JvmInfo jvm, String className, IOThrowingFunction action) throws IOException { + if (jvm.isModular()) { + return withClassStreamModularJdk(jvm.getHome(), className, action); + } else { + return withClassStreamPreModularJdk(jvm.getHome(), className, action); + } + } + + private T withClassStreamModularJdk( + Path jvmHome, String className, IOThrowingFunction action) + throws IOException { + Path javaBaseModule = jvmHome.resolve(JAVA_BASE_MODULE_RELATIVE_PATH); + return withZipEntry( + javaBaseModule, "classes/" + className.replace('.', '/') + ".class", action); + } + + private T withClassStreamPreModularJdk( + Path jvmHome, String className, IOThrowingFunction action) + throws IOException { + Path rtJar = jvmHome.resolve(RT_JAR_RELATIVE_PATH); + return withZipEntry(rtJar, className.replace('.', '/') + ".class", action); + } + + private static T withZipEntry( + Path zipFilePath, String entryName, IOThrowingFunction action) + throws IOException { + try (ZipFile zipFile = new ZipFile(zipFilePath.toFile())) { + ZipEntry entry = zipFile.getEntry(entryName); + if (entry == null) { + throw new IOException("Entry " + entryName + " not found in zip file " + zipFilePath); + } + try (InputStream entryStream = zipFile.getInputStream(entry)) { + return action.apply(entryStream); + } + } + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/CoverageStoreFieldInjector.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/CoverageStoreFieldInjector.java new file mode 100644 index 00000000000..9a0a877fd03 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/CoverageStoreFieldInjector.java @@ -0,0 +1,25 @@ +package datadog.trace.civisibility.coverage.instrumentation.store; + +import static datadog.trace.api.civisibility.coverage.CoverageBridge.COVERAGE_STORE_FIELD_NAME; + +import datadog.trace.api.civisibility.coverage.CoverageBridge; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.Opcodes; + +/** + * A {@link ClassVisitor} that injects an instance field named {@link + * CoverageBridge#COVERAGE_STORE_FIELD_NAME} of type {@link Object} into the visited class + */ +public class CoverageStoreFieldInjector extends ClassVisitor { + + public CoverageStoreFieldInjector(ClassVisitor cv) { + super(Opcodes.ASM9, cv); + } + + @Override + public void visitEnd() { + cv.visitField(Opcodes.ACC_PUBLIC, COVERAGE_STORE_FIELD_NAME, "Ljava/lang/Object;", null, null) + .visitEnd(); + super.visitEnd(); + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/JvmPatcher.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/JvmPatcher.java new file mode 100644 index 00000000000..c8267d63fcf --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/coverage/instrumentation/store/JvmPatcher.java @@ -0,0 +1,61 @@ +package datadog.trace.civisibility.coverage.instrumentation.store; + +import datadog.trace.civisibility.config.JvmInfo; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Function; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; + +public class JvmPatcher { + + private static final String DD_TEMP_DIRECTORY_PREFIX = "dd-ci-vis-class-"; + + private final CoreJvmClassReader coreJvmClassReader; + private final JvmInfo jvm; + + public JvmPatcher(JvmInfo jvm) { + this.coreJvmClassReader = new CoreJvmClassReader(); + this.jvm = jvm; + } + + /** + * Instruments core JVM classes: + * + *
    + *
  • {@link Thread} class by injecting a field that is used for coverage data storage + *
+ * + * The instrumented bytecode is written to a temporary folder. The folder contains sub-folders + * corresponding to packages, e.g. instrumented {@link Thread} class is stored in {@code + * temporaryFolder/java/lang/Thread.class} + * + * @return the path to the folder where the instrumented classes are + * @throws Exception If classes could not be instrumented + */ + public Path createPatch() throws Exception { + Path patchedClassesDir = Files.createTempDirectory(DD_TEMP_DIRECTORY_PREFIX); + instrument(Thread.class, CoverageStoreFieldInjector::new, patchedClassesDir); + return patchedClassesDir; + } + + private void instrument( + Class c, Function classVisitor, Path targetDir) + throws Exception { + byte[] instrumentedBytecode = + coreJvmClassReader.withClassStream( + jvm, + c.getName(), + bytecodeStream -> { + ClassReader cr = new ClassReader(bytecodeStream); + ClassWriter cw = new ClassWriter(cr, ClassWriter.COMPUTE_MAXS); + cr.accept(classVisitor.apply(cw), ClassReader.EXPAND_FRAMES); + return cw.toByteArray(); + }); + + Path classPath = targetDir.resolve(c.getName().replace('.', '/') + ".class"); + Files.createDirectories(classPath.getParent()); + Files.write(classPath, instrumentedBytecode); + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java index dc946edd1e7..e256b768548 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/TestImpl.java @@ -80,7 +80,7 @@ public TestImpl( TestIdentifier identifier = new TestIdentifier(testSuiteName, testName, testParameters, null); CoverageStore probeStore = coverageProbeStoreFactory.create(identifier, sourcePathResolver); - CoverageBridge.setThreadLocalCoverageProbeStore(probeStore); + CoverageBridge.pinCoverageStore(probeStore); this.context = new TestContextImpl(probeStore); @@ -218,7 +218,7 @@ public void end(@Nullable Long endTime) { InstrumentationTestBridge.fireBeforeTestEnd(context); - CoverageBridge.removeThreadLocalCoverageProbeStore(); + CoverageBridge.unpinCoverageStore(); // do not process coverage reports for skipped tests if (span.getTag(Tags.TEST_STATUS) != TestStatus.skip) { diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java index 17d040a56e7..f6a0a860d33 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/domain/buildsystem/BuildSystemSessionImpl.java @@ -173,6 +173,7 @@ private SignalResponse onModuleExecutionSettingsRequestReceived(ModuleSettingsRe settings.isFlakyTestRetriesEnabled(), settings.getEarlyFlakeDetectionSettings(), settings.getSystemProperties(), + settings.getJvmOptions(), settings.getItrCorrelationId(), skippableTests, settings.getFlakyTests(moduleName), diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java index dd2c3de03e0..0aa15abc518 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitClient.java @@ -29,7 +29,7 @@ public class GitClient { public static final String HEAD = "HEAD"; - private static final String DD_TEMP_DIRECTORY_PREFIX = "dd-ci-vis-"; + private static final String DD_TEMP_DIRECTORY_PREFIX = "dd-ci-vis-git-"; private final CiVisibilityMetricCollector metricCollector; private final String repoRoot; diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataUploaderImpl.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataUploaderImpl.java index 483285802a5..ac2ff04b34f 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataUploaderImpl.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/git/tree/GitDataUploaderImpl.java @@ -157,7 +157,7 @@ private void uploadGitData() { } } finally { - FileUtils.delete(packFilesDirectory); + FileUtils.deleteSafely(packFilesDirectory); } LOGGER.debug("Git data upload finished"); diff --git a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/FileUtils.java b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/FileUtils.java index 9c4cffdfd4e..91fba722360 100644 --- a/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/FileUtils.java +++ b/dd-java-agent/agent-ci-visibility/src/main/java/datadog/trace/civisibility/utils/FileUtils.java @@ -7,29 +7,39 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; @SuppressForbidden public abstract class FileUtils { + private static final Logger LOGGER = LoggerFactory.getLogger(FileUtils.class); + private FileUtils() {} - public static void delete(Path directory) throws IOException { - Files.walkFileTree( - directory, - new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - Files.delete(file); - return FileVisitResult.CONTINUE; - } + public static void deleteSafely(Path directory) { + try { + Files.walkFileTree( + directory, + new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + Files.delete(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) + throws IOException { + Files.delete(dir); + return FileVisitResult.CONTINUE; + } + }); - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - Files.delete(dir); - return FileVisitResult.CONTINUE; - } - }); + } catch (IOException e) { + LOGGER.debug("Could not delete directory {}", directory, e); + } } /** diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/JvmInfoFactoryTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/JvmInfoFactoryTest.groovy index 1d438fc2780..bb6c455924e 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/JvmInfoFactoryTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/JvmInfoFactoryTest.groovy @@ -21,7 +21,7 @@ class JvmInfoFactoryTest extends Specification { jvmInfo == JvmInfo.CURRENT_JVM } - private Path getCurrentJvmExecutable() { + private static Path getCurrentJvmExecutable() { def currentJvmPath = Paths.get(ProcessUtils.currentJvmPath) if (Files.isExecutable(currentJvmPath)) { return currentJvmPath diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializerTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializerTest.groovy index b72953279b9..a83b11bbc3c 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializerTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/config/ModuleExecutionSettingsSerializerTest.groovy @@ -17,34 +17,44 @@ class ModuleExecutionSettingsSerializerTest extends Specification { where: settings << [ - new ModuleExecutionSettings(false, false, false, false, EarlyFlakeDetectionSettings.DEFAULT, [:], null, [:], null, [:], []), - new ModuleExecutionSettings(true, true, false, true, + new ModuleExecutionSettings( + false, false, false, false, EarlyFlakeDetectionSettings.DEFAULT, [:], [], null, [:], null, [:], []), + new ModuleExecutionSettings( + true, true, false, true, new EarlyFlakeDetectionSettings(true, [], 10), ["a": "b", "propName": "propValue"], + ["f"], "", ["module": [new TestIdentifier("a", "bc", "def", null), new TestIdentifier("abc", "de", "f", null)]], [new TestIdentifier("suite", "name", null, null)], ["bundle": [new TestIdentifier("a", "b", "c", null)]], - ["a", "bcde", "f", "ghhi"]), - new ModuleExecutionSettings(false, false, true, false, + ["a", "bcde", "f", "ghhi"] + ), + new ModuleExecutionSettings( + false, false, true, false, new EarlyFlakeDetectionSettings(true, [new EarlyFlakeDetectionSettings.ExecutionsByDuration(10, 20)], 10), ["a": "b", "propName": "propValue"], + ["f", "e"], "itrCorrelationId", [:], [new TestIdentifier("suite", "name", null, null), new TestIdentifier("a", "b", "c", null)], ["bundle": [new TestIdentifier("a", "b", "c", null), new TestIdentifier("aa", "bb", "cc", null)]], - ["a", "bcde", "f", "ghhi"]), - new ModuleExecutionSettings(true, true, true, true, + ["a", "bcde", "f", "ghhi"] + ), + new ModuleExecutionSettings( + true, true, true, true, new EarlyFlakeDetectionSettings(true, [ new EarlyFlakeDetectionSettings.ExecutionsByDuration(10, 20), new EarlyFlakeDetectionSettings.ExecutionsByDuration(30, 40) ], 10), ["a": "b", "propName": "propValue", "anotherProp": "value"], + ["f", "e", "g"], "itrCorrelationId", ["module": [new TestIdentifier("a", "bc", "def", null), new TestIdentifier("abc", "de", "f", null)], "module-b": [new TestIdentifier("suite", "name", null, null)], "module-c": []], [], ["bundle": [new TestIdentifier("a", "b", "c", null)], "bundle-2": [new TestIdentifier("aa", "bb", "cc", null)]], - []) + [] + ) ] } } diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/instrumentation/store/CoreJvmClassReaderTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/instrumentation/store/CoreJvmClassReaderTest.groovy new file mode 100644 index 00000000000..f209a113db6 --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/instrumentation/store/CoreJvmClassReaderTest.groovy @@ -0,0 +1,20 @@ +package datadog.trace.civisibility.coverage.instrumentation.store + +import datadog.trace.civisibility.config.JvmInfo +import org.apache.commons.io.IOUtils +import spock.lang.Specification + +class CoreJvmClassReaderTest extends Specification { + + def "test reads Thread class"() { + setup: + def reader = new CoreJvmClassReader() + + when: + def threadClassBytecode = reader.withClassStream(JvmInfo.CURRENT_JVM, Thread.class.getName(), is -> IOUtils.toByteArray(is)) + + then: + threadClassBytecode != null + threadClassBytecode.length > 0 + } +} diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/instrumentation/store/JvmPatcherTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/instrumentation/store/JvmPatcherTest.groovy new file mode 100644 index 00000000000..7bb8a57054c --- /dev/null +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/coverage/instrumentation/store/JvmPatcherTest.groovy @@ -0,0 +1,26 @@ +package datadog.trace.civisibility.coverage.instrumentation.store + +import datadog.trace.civisibility.config.JvmInfo +import datadog.trace.civisibility.utils.FileUtils +import spock.lang.Specification + +import java.nio.file.Files + +class JvmPatcherTest extends Specification { + + def "test patches current JVM"() { + setup: + def patcher = new JvmPatcher(JvmInfo.CURRENT_JVM) + + when: + // a basic check to ensure that patching does not fail and that the patched class file exists + def patchFolder = patcher.createPatch() + + then: + Files.exists(patchFolder.resolve("java/lang/Thread.class")) + + cleanup: + FileUtils.deleteSafely(patchFolder) + } +} + diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/ModuleSettingsRequestTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/ModuleSettingsRequestTest.groovy index 151b3698618..bf586afa3ca 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/ModuleSettingsRequestTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/ipc/ModuleSettingsRequestTest.groovy @@ -19,15 +19,17 @@ class ModuleSettingsRequestTest extends Specification { signal << [ new ModuleSettingsRequest(Paths.get("").toString(), JvmInfo.CURRENT_JVM), new ModuleSettingsRequest(null, JvmInfo.CURRENT_JVM), - new ModuleSettingsRequest("abc", new JvmInfo("abc", "def", "ghi")), - new ModuleSettingsRequest("abc", new JvmInfo("", "def", "ghi")), - new ModuleSettingsRequest("abc", new JvmInfo("abc", "", "ghi")), - new ModuleSettingsRequest("abc", new JvmInfo("abc", "def", "")), - new ModuleSettingsRequest("abc", new JvmInfo("", "", "")), - new ModuleSettingsRequest("abc", new JvmInfo(null, "def", "ghi")), - new ModuleSettingsRequest("abc", new JvmInfo("abc", null, "ghi")), - new ModuleSettingsRequest("abc", new JvmInfo("abc", "def", null)), - new ModuleSettingsRequest("abc", new JvmInfo(null, null, null)), + new ModuleSettingsRequest("abc", new JvmInfo("abc", "def", "52.0", "123", "456")), + new ModuleSettingsRequest("abc", new JvmInfo("", "def", "61.0", "", "")), + new ModuleSettingsRequest("abc", new JvmInfo("abc", "", "ghi", null, null)), + new ModuleSettingsRequest("abc", new JvmInfo("abc", "def", "", "123", "456")), + new ModuleSettingsRequest("abc", new JvmInfo("", "", "", "", "")), + new ModuleSettingsRequest("abc", new JvmInfo(null, "def", "52.0", "123", "456")), + new ModuleSettingsRequest("abc", new JvmInfo("abc", null, "52.0", "123", "456")), + new ModuleSettingsRequest("abc", new JvmInfo("abc", "def", null, "123", "456")), + new ModuleSettingsRequest("abc", new JvmInfo("abc", "def", "52.0", null, "456")), + new ModuleSettingsRequest("abc", new JvmInfo("abc", "def", "52.0", null, null)), + new ModuleSettingsRequest("abc", new JvmInfo(null, null, null, null, null)), ] } } diff --git a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/utils/FileUtilsTest.groovy b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/utils/FileUtilsTest.groovy index baf8f01cb17..4653abeb7ef 100644 --- a/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/utils/FileUtilsTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/test/groovy/datadog/trace/civisibility/utils/FileUtilsTest.groovy @@ -19,7 +19,7 @@ class FileUtilsTest extends Specification { Files.createDirectory(temporaryFolder.resolve("childFolder").resolve("childFile")) when: - FileUtils.delete(temporaryFolder) + FileUtils.deleteSafely(temporaryFolder) then: !Files.exists(temporaryFolder) diff --git a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy index 4366e3dfdf9..83826e16840 100644 --- a/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy +++ b/dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy @@ -112,6 +112,7 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner { ], 0) : EarlyFlakeDetectionSettings.DEFAULT, properties, + [], itrEnabled ? "itrCorrelationId" : null, Collections.singletonMap(dummyModule, skippableTests), flakyTests, diff --git a/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/CiVisibilityService.java b/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/CiVisibilityService.java index 879c43c7e5a..f5fdb338897 100644 --- a/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/CiVisibilityService.java +++ b/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/CiVisibilityService.java @@ -64,10 +64,11 @@ public List getCoverageEnabledPackages(Path jvmExecutable) { @SuppressForbidden public Collection getTracerJvmArgs(String taskPath, Path jvmExecutable) { - List jvmArgs = new ArrayList<>(); - ModuleExecutionSettings moduleExecutionSettings = buildEventsHandler.getModuleExecutionSettings(SESSION_KEY, jvmExecutable); + + List jvmArgs = new ArrayList<>(moduleExecutionSettings.getJvmOptions()); + Map propagatedSystemProperties = moduleExecutionSettings.getSystemProperties(); // propagate to child process all "dd." system properties available in current process for (Map.Entry e : propagatedSystemProperties.entrySet()) { diff --git a/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/legacy/GradleBuildListener.java b/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/legacy/GradleBuildListener.java index 8b51e987ebc..3d75f9c7b68 100644 --- a/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/legacy/GradleBuildListener.java +++ b/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/legacy/GradleBuildListener.java @@ -105,7 +105,9 @@ private void configureTestExecutions( for (Task testExecution : testExecutions) { GradleProjectConfigurator.INSTANCE.configureTracer( - testExecution, moduleExecutionSettings.getSystemProperties()); + testExecution, + moduleExecutionSettings.getSystemProperties(), + moduleExecutionSettings.getJvmOptions()); GradleProjectConfigurator.INSTANCE.configureJacoco( testExecution.getProject(), moduleExecutionSettings); } diff --git a/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/legacy/GradleProjectConfigurator.groovy b/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/legacy/GradleProjectConfigurator.groovy index 31ed1257ea2..299b9607f4d 100644 --- a/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/legacy/GradleProjectConfigurator.groovy +++ b/dd-java-agent/instrumentation/gradle/src/main/groovy/datadog/trace/instrumentation/gradle/legacy/GradleProjectConfigurator.groovy @@ -49,9 +49,11 @@ class GradleProjectConfigurator { private static final String JACOCO_PLUGIN_ID = 'jacoco' - void configureTracer(Task task, Map propagatedSystemProperties) { + void configureTracer(Task task, Map propagatedSystemProperties, List jvmOptions) { List jvmArgs = new ArrayList<>(task.jvmArgs != null ? task.jvmArgs : Collections. emptyList()) + jvmArgs.addAll(jvmOptions) + // propagate to child process all "dd." system properties available in current process for (def e : propagatedSystemProperties.entrySet()) { jvmArgs.add("-D" + e.key + '=' + e.value) diff --git a/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenLifecycleParticipant.java b/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenLifecycleParticipant.java index fad85787082..f930ee1ca70 100644 --- a/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenLifecycleParticipant.java +++ b/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenLifecycleParticipant.java @@ -298,7 +298,9 @@ private Void configureTestExecutions( for (MavenTestExecution testExecution : testExecutions) { MavenProjectConfigurator.INSTANCE.configureTracer( - testExecution, moduleExecutionSettings.getSystemProperties()); + testExecution, + moduleExecutionSettings.getSystemProperties(), + moduleExecutionSettings.getJvmOptions()); MavenProjectConfigurator.INSTANCE.configureJacoco(testExecution, moduleExecutionSettings); } return null; diff --git a/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenProjectConfigurator.java b/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenProjectConfigurator.java index 8480c4537ab..a421f32aadf 100644 --- a/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenProjectConfigurator.java +++ b/dd-java-agent/instrumentation/maven-3.2.1/src/main/java/datadog/trace/instrumentation/maven3/MavenProjectConfigurator.java @@ -37,7 +37,9 @@ class MavenProjectConfigurator { private static final String JACOCO_EXCL_CLASS_LOADERS_PROPERTY = "jacoco.exclClassLoaders"; public void configureTracer( - MavenTestExecution mavenTestExecution, Map propagatedSystemProperties) { + MavenTestExecution mavenTestExecution, + Map propagatedSystemProperties, + List jvmOptions) { MojoExecution mojoExecution = mavenTestExecution.getExecution(); PlexusConfiguration pomConfiguration = MavenUtils.getPomConfiguration(mojoExecution); @@ -48,6 +50,10 @@ public void configureTracer( } StringBuilder addedArgLine = new StringBuilder(); + for (String jvmOption : jvmOptions) { + addedArgLine.append(jvmOption).append(" "); + } + // propagate to child process all "dd." system properties available in current process for (Map.Entry e : propagatedSystemProperties.entrySet()) { addedArgLine.append("-D").append(e.getKey()).append('=').append(e.getValue()).append(" "); diff --git a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java index 0a9b9042fdf..c4fb2bd1d43 100644 --- a/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java +++ b/dd-trace-api/src/main/java/datadog/trace/api/config/CiVisibilityConfig.java @@ -66,6 +66,8 @@ public final class CiVisibilityConfig { "civisibility.rum.flush.wait.millis"; public static final String CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER = "civisibility.auto.instrumentation.provider"; + public static final String CIVISIBILITY_JVM_PATCHING_ENABLED = + "civisibility.jvm.patching.enabled"; /* COVERAGE SETTINGS */ public static final String CIVISIBILITY_CODE_COVERAGE_ENABLED = diff --git a/internal-api/src/main/java/datadog/trace/api/Config.java b/internal-api/src/main/java/datadog/trace/api/Config.java index 73e7bb4e6ae..d729887ccf3 100644 --- a/internal-api/src/main/java/datadog/trace/api/Config.java +++ b/internal-api/src/main/java/datadog/trace/api/Config.java @@ -198,6 +198,7 @@ import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_JACOCO_GRADLE_SOURCE_SETS; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_JACOCO_PLUGIN_VERSION; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_JVM_INFO_CACHE_SIZE; +import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_JVM_PATCHING_ENABLED; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_MODULE_EXECUTION_SETTINGS_CACHE_SIZE; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_MODULE_ID; import static datadog.trace.api.config.CiVisibilityConfig.CIVISIBILITY_MODULE_NAME; @@ -830,6 +831,7 @@ static class HostNameHolder { private final boolean ciVisibilityTelemetryEnabled; private final long ciVisibilityRumFlushWaitMillis; private final boolean ciVisibilityAutoInjected; + private final boolean ciVisibilityJvmPatchingEnabled; private final boolean remoteConfigEnabled; private final boolean remoteConfigIntegrityCheckEnabled; @@ -1888,6 +1890,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment()) configProvider.getLong(CIVISIBILITY_RUM_FLUSH_WAIT_MILLIS, 500); ciVisibilityAutoInjected = Strings.isNotBlank(configProvider.getString(CIVISIBILITY_AUTO_INSTRUMENTATION_PROVIDER)); + ciVisibilityJvmPatchingEnabled = + configProvider.getBoolean(CIVISIBILITY_JVM_PATCHING_ENABLED, true); remoteConfigEnabled = configProvider.getBoolean( @@ -3220,6 +3224,10 @@ public boolean isCiVisibilityAutoInjected() { return ciVisibilityAutoInjected; } + public boolean isCiVisibilityJvmPatchingEnabled() { + return ciVisibilityJvmPatchingEnabled; + } + public String getAppSecRulesFile() { return appSecRulesFile; } diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/config/ModuleExecutionSettings.java b/internal-api/src/main/java/datadog/trace/api/civisibility/config/ModuleExecutionSettings.java index b2a990de7cf..d38bdcb0f60 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/config/ModuleExecutionSettings.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/config/ModuleExecutionSettings.java @@ -17,6 +17,7 @@ public class ModuleExecutionSettings { false, EarlyFlakeDetectionSettings.DEFAULT, Collections.emptyMap(), + Collections.emptyList(), null, Collections.emptyMap(), Collections.emptyList(), @@ -29,6 +30,7 @@ public class ModuleExecutionSettings { private final boolean flakyTestRetriesEnabled; private final EarlyFlakeDetectionSettings earlyFlakeDetectionSettings; private final Map systemProperties; + private final List jvmOptions; private final String itrCorrelationId; private final Map> skippableTestsByModule; @Nullable private final Collection flakyTests; @@ -42,6 +44,7 @@ public ModuleExecutionSettings( boolean flakyTestRetriesEnabled, EarlyFlakeDetectionSettings earlyFlakeDetectionSettings, Map systemProperties, + List jvmOptions, String itrCorrelationId, Map> skippableTestsByModule, Collection flakyTests, @@ -53,6 +56,7 @@ public ModuleExecutionSettings( this.flakyTestRetriesEnabled = flakyTestRetriesEnabled; this.earlyFlakeDetectionSettings = earlyFlakeDetectionSettings; this.systemProperties = systemProperties; + this.jvmOptions = jvmOptions; this.itrCorrelationId = itrCorrelationId; this.skippableTestsByModule = skippableTestsByModule; this.flakyTests = flakyTests; @@ -88,6 +92,10 @@ public Map getSystemProperties() { return systemProperties; } + public List getJvmOptions() { + return jvmOptions; + } + @Nullable public String getItrCorrelationId() { return itrCorrelationId; @@ -145,6 +153,7 @@ public boolean equals(Object o) { && testSkippingEnabled == that.testSkippingEnabled && Objects.equals(earlyFlakeDetectionSettings, that.earlyFlakeDetectionSettings) && Objects.equals(systemProperties, that.systemProperties) + && Objects.equals(jvmOptions, that.jvmOptions) && Objects.equals(itrCorrelationId, that.itrCorrelationId) && Objects.equals(skippableTestsByModule, that.skippableTestsByModule) && Objects.equals(flakyTests, that.flakyTests) @@ -160,6 +169,7 @@ public int hashCode() { testSkippingEnabled, earlyFlakeDetectionSettings, systemProperties, + jvmOptions, itrCorrelationId, skippableTestsByModule, flakyTests, diff --git a/internal-api/src/main/java/datadog/trace/api/civisibility/coverage/CoverageBridge.java b/internal-api/src/main/java/datadog/trace/api/civisibility/coverage/CoverageBridge.java index 3fc21129bc2..4e1cd84a7a5 100644 --- a/internal-api/src/main/java/datadog/trace/api/civisibility/coverage/CoverageBridge.java +++ b/internal-api/src/main/java/datadog/trace/api/civisibility/coverage/CoverageBridge.java @@ -2,16 +2,38 @@ import datadog.trace.api.civisibility.InstrumentationTestBridge; import datadog.trace.api.civisibility.domain.TestContext; +import datadog.trace.util.MethodHandles; +import java.lang.invoke.MethodHandle; public abstract class CoverageBridge { - /* - * While it is possible to use activeSpan() to get current coverage store, it adds a lot of overhead. - * This thread local is here as a shortcut for hot code paths. - */ - private static final ThreadLocal COVERAGE_PROBE_STORE = new ThreadLocal<>(); + private static volatile CoverageStore.Registry COVERAGE_PROBE_STORE_REGISTRY; private static volatile CoverageDataSupplier COVERAGE_DATA_SUPPLIER; + /** + * While it is possible to use activeSpan() to get current coverage store, it adds a lot of + * overhead. To reduce the overhead the tracer injects this field into the {@link Thread} class. + * It is, essentially, a faster {@link ThreadLocal}. + */ + public static final String COVERAGE_STORE_FIELD_NAME = "__datadogCoverageStore"; + + private static final MethodHandle COVERAGE_STORE_GETTER; + private static final MethodHandle COVERAGE_STORE_SETTER; + + static { + MethodHandles methodHandles = new MethodHandles(null); + COVERAGE_STORE_GETTER = + methodHandles.privateFieldGetter(Thread.class, COVERAGE_STORE_FIELD_NAME); + COVERAGE_STORE_SETTER = + methodHandles.privateFieldSetter(Thread.class, COVERAGE_STORE_FIELD_NAME); + } + + /** + * Used as a fallback in situations where {@link CoverageBridge#COVERAGE_STORE_FIELD_NAME} could + * not be injected. + */ + private static final ThreadLocal THREAD_LOCAL_COVERAGE_STORE = new ThreadLocal<>(); + public static void registerCoverageProbeStoreRegistry( CoverageStore.Registry coverageProbeStoreRegistry) { COVERAGE_PROBE_STORE_REGISTRY = coverageProbeStoreRegistry; @@ -29,56 +51,104 @@ public static byte[] getCoverageData() { return COVERAGE_DATA_SUPPLIER != null ? COVERAGE_DATA_SUPPLIER.get() : null; } - public static void setThreadLocalCoverageProbeStore(CoverageStore probes) { - COVERAGE_PROBE_STORE.set(probes); + public static void pinCoverageStore(CoverageStore probes) { + if (COVERAGE_STORE_SETTER != null) { + try { + COVERAGE_STORE_SETTER.invokeExact(Thread.currentThread(), (Object) probes); + } catch (Throwable e) { + throw new RuntimeException("Could not pin coverage store", e); + } + + } else { + THREAD_LOCAL_COVERAGE_STORE.set(probes); + } } - public static void removeThreadLocalCoverageProbeStore() { - COVERAGE_PROBE_STORE.remove(); + public static void unpinCoverageStore() { + if (COVERAGE_STORE_SETTER != null) { + try { + COVERAGE_STORE_SETTER.invokeExact(Thread.currentThread(), (Object) null); + } catch (Throwable e) { + throw new RuntimeException("Could not unpin coverage store", e); + } + + } else { + THREAD_LOCAL_COVERAGE_STORE.remove(); + } } /* This method is referenced by name in bytecode added in jacoco instrumentation module */ public static void currentCoverageProbeStoreRecord(Class clazz, long classId, int probeId) { - CoverageStore probes = COVERAGE_PROBE_STORE.get(); - if (probes != null) { - probes.record(clazz, classId, probeId); + CoverageStore probes; + if (COVERAGE_STORE_GETTER != null) { + try { + probes = + (CoverageStore) ((Object) COVERAGE_STORE_GETTER.invokeExact(Thread.currentThread())); + } catch (Throwable e) { + throw new RuntimeException("Could not get coverage store", e); + } } else { + probes = THREAD_LOCAL_COVERAGE_STORE.get(); + } + + if (probes == null) { probes = getCurrentCoverageProbeStore(); - if (probes != null) { - probes.record(clazz, classId, probeId); - } + } + + if (probes != null) { + probes.record(clazz, classId, probeId); } } /* This method is referenced by name in bytecode added by coverage probes (see CoverageUtils) */ public static void currentCoverageProbeStoreRecord(Class clazz) { - CoverageStore probes = COVERAGE_PROBE_STORE.get(); - if (probes != null) { - probes.record(clazz); + CoverageStore probes; + if (COVERAGE_STORE_GETTER != null) { + try { + probes = + (CoverageStore) ((Object) COVERAGE_STORE_GETTER.invokeExact(Thread.currentThread())); + } catch (Throwable e) { + throw new RuntimeException("Could not get coverage store", e); + } } else { + probes = THREAD_LOCAL_COVERAGE_STORE.get(); + } + + if (probes == null) { probes = getCurrentCoverageProbeStore(); - if (probes != null) { - probes.record(clazz); - } + } + + if (probes != null) { + probes.record(clazz); } } public static void currentCoverageProbeStoreRecordNonCode(String absolutePath) { - CoverageStore probes = COVERAGE_PROBE_STORE.get(); - if (probes != null) { - probes.recordNonCodeResource(absolutePath); + CoverageStore probes; + if (COVERAGE_STORE_GETTER != null) { + try { + probes = + (CoverageStore) ((Object) COVERAGE_STORE_GETTER.invokeExact(Thread.currentThread())); + } catch (Throwable e) { + throw new RuntimeException("Could not get coverage store", e); + } } else { + probes = THREAD_LOCAL_COVERAGE_STORE.get(); + } + + if (probes == null) { probes = getCurrentCoverageProbeStore(); - if (probes != null) { - probes.recordNonCodeResource(absolutePath); - } + } + + if (probes != null) { + probes.recordNonCodeResource(absolutePath); } } /** * Gets coverage probe store associated with the active span. This is a fallback method for cases - * when the probe store could not be retrieved from the thread local. This can happen if the span - * is propagated from the original test thread to another thread. + * when the probe store could not be retrieved via faster methods. This can happen if the span is + * propagated from the original test thread to another thread. */ private static CoverageStore getCurrentCoverageProbeStore() { TestContext currentTest = InstrumentationTestBridge.getCurrentTestContext();