From 96e0020e5a811deb3caf1da5d93dbfaf66127fb5 Mon Sep 17 00:00:00 2001 From: Tudor-Stefan Magirescu Date: Thu, 17 Jul 2025 11:39:48 +0200 Subject: [PATCH 1/3] [GR-65623] Implement layer compatibility check for class and module-path entries. --- .../src/jdk/graal/compiler/util/Digest.java | 20 ++- .../com/oracle/svm/core/jdk/JRTSupport.java | 8 ++ .../hosted/NativeImageClassLoaderSupport.java | 119 +++++++++++++++++- .../hosted/driver/LayerOptionsSupport.java | 4 +- .../HostedImageLayerBuildingSupport.java | 12 +- .../imagelayer/LayerArchiveSupport.java | 65 ++++++++++ .../imagelayer/LoadLayerArchiveSupport.java | 43 +++++++ .../imagelayer/WriteLayerArchiveSupport.java | 12 ++ 8 files changed, 276 insertions(+), 7 deletions(-) diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java index 8ecc1ab2251b..c7950d729c12 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java @@ -102,8 +102,10 @@ public static String digest(byte[] bytes, int offset, int length) { * new byte array. */ public static byte[] digestAsByteArray(byte[] bytes, int offset, int length) { - LongLong hash = MurmurHash3_x64_128(bytes, offset, length, HASH_SEED); + return longLongToByteArray(MurmurHash3_x64_128(bytes, offset, length, HASH_SEED)); + } + private static byte[] longLongToByteArray(LongLong hash) { byte[] array = new byte[DIGEST_SIZE]; encodeBase62(hash.l1, array, 0); encodeBase62(hash.l2, array, BASE62_DIGITS_PER_LONG); @@ -281,4 +283,20 @@ private static long fmix64(long input) { return k; } + + public static final class DigestBuilder { + private LongLong digest; + + public DigestBuilder() { + digest = new LongLong(0L, 0L); + } + + public void update(byte[] input) { + digest = MurmurHash3_x64_128(input, 0, input.length, digest.l1 ^ digest.l2); + } + + public byte[] digest() { + return longLongToByteArray(digest); + } + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java index 712eff9d3cf9..7f3d2d5d17e1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java @@ -29,6 +29,7 @@ import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; +import java.nio.file.FileSystem; import java.nio.file.Path; import java.util.Arrays; import java.util.HashMap; @@ -181,6 +182,13 @@ protected URLConnection openConnection(URL url) throws IOException { final class Target_jdk_internal_jrtfs_JrtFileSystemProvider_JRTDisabled { } +@TargetClass(className = "jdk.internal.jrtfs.JrtFileSystemProvider", onlyWith = JRTEnabled.class) +final class Target_jdk_internal_jrtfs_JrtFileSystemProvider_BuildTime { + @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset)// + @Alias// + volatile FileSystem theFileSystem; +} + // endregion Disable jimage/jrtfs @TargetClass(className = "jdk.internal.jimage.BasicImageReader") diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index ccd1963e1637..75ee45b4e70a 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.hosted; +import static jdk.graal.compiler.util.Digest.DigestBuilder; + import java.io.File; import java.io.IOException; import java.lang.module.Configuration; @@ -36,9 +38,12 @@ import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.nio.channels.ClosedByInterruptException; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.FileSystems; +import java.nio.file.FileVisitOption; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; import java.nio.file.Files; @@ -52,6 +57,8 @@ import java.util.Collections; import java.util.Comparator; import java.util.Deque; +import java.util.EnumSet; +import java.util.Enumeration; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -67,9 +74,12 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; import java.util.stream.Collector; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.zip.ZipFile; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; @@ -116,6 +126,7 @@ public final class NativeImageClassLoaderSupport { private final List buildmp; private final Set imageProvidedJars; + private PathDigests pathDigests; private final EconomicMap> classes; private final EconomicMap> packages; @@ -300,6 +311,47 @@ public NativeImageClassLoader getClassLoader() { return classLoader; } + public Optional getPathDigests(boolean free) { + Optional res = Optional.ofNullable(pathDigests); + if (free) { + pathDigests = null; + } + return res; + } + + public void initializePathDigests(Path digestIgnoreRelativePath) { + List digestIgnorePaths = new ArrayList<>(); + if (digestIgnoreRelativePath != null) { + try { + Enumeration urls = classLoader.getResources(digestIgnoreRelativePath.toString()); + while (urls.hasMoreElements()) { + URI uri = urls.nextElement().toURI(); + if ("jar".equalsIgnoreCase(uri.getScheme())) { + String uriString = uri.toString(); + String fileUriString = uriString.substring("jar:".length(), uriString.indexOf("!/")); + uri = URI.create(fileUriString); + } + digestIgnorePaths.add(Path.of(uri)); + } + } catch (URISyntaxException | IOException e) { + throw UserError.abort("Error while looking for %s in %s", digestIgnoreRelativePath, classLoader); + } + } + + pathDigests = new PathDigests(filterIgnoredPathEntries(imagecp, digestIgnorePaths), filterIgnoredPathEntries(imagemp, digestIgnorePaths)); + } + + private List filterIgnoredPathEntries(List pathEntries, List digestIgnorePaths) { + return pathEntries.stream().filter(pathEntry -> { + for (Path p : digestIgnorePaths) { + if (p.startsWith(pathEntry)) { + return false; + } + } + return true; + }).toList(); + } + public LibGraalLoader getLibGraalLoader() { VMError.guarantee(libGraalLoader != null, "Invalid access to libGraalLoader before getting set up"); return libGraalLoader.orElse(null); @@ -932,6 +984,8 @@ private void initModule(ModuleReference moduleReference, boolean moduleRequiresI if (ModuleLayer.boot().equals(module.getLayer())) { builderURILocations.add(container); } + final boolean isInImageModulePathOfLayeredBuild = pathDigests != null && pathDigests.mpDigests.containsKey(container); + final boolean isJar = ClasspathUtils.isJar(Path.of(container)); moduleReader.list().forEach(moduleResource -> { char fileSystemSeparatorChar = '/'; String className = extractClassName(moduleResource, fileSystemSeparatorChar); @@ -939,6 +993,9 @@ private void initModule(ModuleReference moduleReference, boolean moduleRequiresI currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource; executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit, preserveModule)); } + if (isInImageModulePathOfLayeredBuild) { + executor.execute(() -> PathDigests.storePathFileDigest(container, moduleResource, isJar, pathDigests.mpDigests)); + } entriesProcessed.increment(); }); } catch (IOException e) { @@ -962,7 +1019,7 @@ private void loadClassesFromPath(Path path) { } if (probeJarFileSystem != null) { try (FileSystem jarFileSystem = probeJarFileSystem) { - loadClassesFromPath(container, jarFileSystem.getPath("/"), null, Collections.emptySet(), includeUnconditionally, includeAllMetadata); + loadClassesFromPath(container, jarFileSystem.getPath("/"), null, Collections.emptySet(), includeUnconditionally, includeAllMetadata, true); } } } catch (ClosedByInterruptException ignored) { @@ -973,13 +1030,13 @@ private void loadClassesFromPath(Path path) { } else { URI container = path.toUri(); loadClassesFromPath(container, path, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES, includeUnconditionally, - includeAllMetadata); + includeAllMetadata, false); } } private static final String CLASS_EXTENSION = ".class"; - private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set excludes, boolean includeUnconditionally, boolean includeAllMetadata) { + private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set excludes, boolean includeUnconditionally, boolean includeAllMetadata, boolean isJar) { boolean useFilter = root.equals(excludeRoot); if (useFilter) { String excludesStr = excludes.stream().map(Path::toString).collect(Collectors.joining(", ")); @@ -987,6 +1044,7 @@ private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set } FileVisitor visitor = new SimpleFileVisitor<>() { private final char fileSystemSeparatorChar = root.getFileSystem().getSeparator().charAt(0); + private final boolean isInImageClassPathOfLayeredBuild = pathDigests != null && pathDigests.getCpDigests().containsKey(container); @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { @@ -1007,6 +1065,9 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { currentlyProcessedEntry = file.toUri().toString(); executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally, true, includeAllMetadata)); } + if (isInImageClassPathOfLayeredBuild) { + executor.execute(() -> PathDigests.storePathFileDigest(container, fileName, isJar, pathDigests.cpDigests)); + } entriesProcessed.increment(); return FileVisitResult.CONTINUE; } @@ -1019,7 +1080,7 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) { }; try { - Files.walkFileTree(root, visitor); + Files.walkFileTree(root, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, visitor); } catch (IOException ex) { throw VMError.shouldNotReachHere(ex); } @@ -1372,4 +1433,54 @@ public Set packages() { return packages.keySet(); } } + + public static final class PathDigests { + private final EconomicMap> cpDigests = EconomicMap.create(); + private final EconomicMap> mpDigests = EconomicMap.create(); + + private PathDigests(List imagecp, List imagemp) { + imagecp.stream() + .map(Path::toUri) + .forEach(path -> cpDigests.put(path, new ArrayList<>())); + imagemp.stream() + .map(Path::toUri) + .forEach(path -> mpDigests.put(path, new ArrayList<>())); + } + + private static void storePathFileDigest(URI container, String resource, boolean isJar, EconomicMap> digests) { + byte[] fileContent; + try { + if (isJar) { + try (JarFile jarFile = new JarFile(new File(container), true, ZipFile.OPEN_READ, JarFile.runtimeVersion())) { + JarEntry jarEntry = jarFile.getJarEntry(resource); + fileContent = jarFile.getInputStream(jarEntry).readAllBytes(); + } + } else { + Path resourcePath = Path.of(container).resolve(resource); + if (!resourcePath.toFile().isFile()) { + return; + } + fileContent = Files.readAllBytes(resourcePath); + } + } catch (IOException e) { + throw UserError.abort("Image builder cannot read file: " + resource); + } + + DigestBuilder db = new DigestBuilder(); + db.update(fileContent); + db.update(resource.getBytes(StandardCharsets.UTF_8)); + List containerDigests = digests.get(container); + synchronized (containerDigests) { + containerDigests.add(new String(db.digest(), StandardCharsets.UTF_8)); + } + } + + public EconomicMap> getCpDigests() { + return cpDigests; + } + + public EconomicMap> getMpDigests() { + return mpDigests; + } + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java index ba75884a0228..5cb8e5427a61 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java @@ -35,7 +35,9 @@ public class LayerOptionsSupport extends IncludeOptionsSupport { public record LayerOption(Path fileName, ExtendedOption[] extendedOptions) { - /** Split a layer option into its components. */ + /** + * Split a layer option into its components. + */ public static LayerOption parse(String layerOptionValue) { VMError.guarantee(!layerOptionValue.isEmpty()); // Given an argument of form layer-file.nil,module=m1,package=p1 diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java index 31e45372ce8f..08b106333996 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java @@ -187,6 +187,7 @@ public Function, SingletonTrait[]> getSingletonTraitInjector() { */ public static void processLayerOptions(EconomicMap, Object> values, NativeImageClassLoaderSupport classLoaderSupport) { OptionValues hostedOptions = new OptionValues(values); + Path digestIgnorePath = null; if (isLayerCreateOptionEnabled(hostedOptions)) { ValueWithOrigin valueWithOrigin = getLayerCreateValueWithOrigin(hostedOptions); @@ -205,7 +206,12 @@ public static void processLayerOptions(EconomicMap, Object> values, classLoaderSupport.setLayerFile(layerFile); NativeImageClassLoaderSupport.IncludeSelectors layerSelectors = classLoaderSupport.getLayerSelectors(); + IncludeOptionsSupport.ExtendedOption digestIgnoreExtendedOption = new IncludeOptionsSupport.ExtendedOption("digest-ignore", null); for (IncludeOptionsSupport.ExtendedOption option : layerOption.extendedOptions()) { + if (option.equals(digestIgnoreExtendedOption)) { + digestIgnorePath = valueWithOrigin.origin().location(); + continue; + } IncludeOptionsSupport.parseIncludeSelector(layerCreateArg, valueWithOrigin, layerSelectors, option, layerCreatePossibleOptions()); } @@ -238,6 +244,10 @@ public static void processLayerOptions(EconomicMap, Object> values, SubstrateOptions.ApplicationLayerInitializedClasses.update(values, Module.class.getName()); setOptionIfHasNotBeenSet(values, SubstrateOptions.ConcealedOptions.RelativeCodePointers, true); } + + if (isLayerCreateOptionEnabled(hostedOptions) || isLayerUseOptionEnabled(hostedOptions)) { + classLoaderSupport.initializePathDigests(digestIgnorePath); + } } private static void setOptionIfHasNotBeenSet(EconomicMap, Object> values, HostedOptionKey option, boolean boxedValue) { @@ -285,7 +295,7 @@ private static String getLayerCreateValue(ValueWithOrigin valueWithOrigi return String.join(",", OptionUtils.resolveOptionValuesRedirection(SubstrateOptions.LayerCreate, valueWithOrigin)); } - private static boolean isLayerUseOptionEnabled(OptionValues values) { + public static boolean isLayerUseOptionEnabled(OptionValues values) { if (SubstrateOptions.LayerUse.hasBeenSet(values)) { return !getLayerUseValue(values).toString().isEmpty(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java index cb23f4560965..de0e9e95af68 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java @@ -25,9 +25,12 @@ package com.oracle.svm.hosted.imagelayer; import static com.oracle.svm.core.util.EnvVariableUtils.EnvironmentVariable; +import static jdk.graal.compiler.util.Digest.DigestBuilder; import java.io.IOException; import java.io.OutputStream; +import java.net.URI; +import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.ArrayList; @@ -36,14 +39,17 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Properties; +import org.graalvm.collections.EconomicMap; import org.graalvm.nativeimage.Platform; import com.oracle.svm.core.SharedConstants; import com.oracle.svm.core.util.ArchiveSupport; import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; +import com.oracle.svm.hosted.NativeImageClassLoaderSupport; import com.oracle.svm.util.LogUtils; public class LayerArchiveSupport { @@ -55,6 +61,7 @@ public class LayerArchiveSupport { private static final String ENV_VARIABLES_FILE_NAME = "env-variables.txt"; private static final String SNAPSHOT_FILE_NAME = "layer-snapshot.lsb"; private static final String SNAPSHOT_GRAPHS_FILE_NAME = "layer-snapshot-graphs.big"; + private static final String BUILD_PATH_DIGESTS_FILE_NAME = "build-path-digests.txt"; private static final String LAYER_INFO_MESSAGE_PREFIX = "Native Image Layers"; protected static final String LAYER_TEMP_DIR_PREFIX = "layerRoot_"; protected static final String SHARED_LIB_NAME_PREFIX = "lib"; @@ -62,6 +69,7 @@ public class LayerArchiveSupport { public static final String LAYER_FILE_EXTENSION = ".nil"; protected final List builderArguments; + protected final List buildPathDigests; protected final LayerProperties layerProperties; protected final Path layerFile; @@ -86,6 +94,7 @@ public LayerArchiveSupport(String layerName, Path layerFile, Path layerDir, Arch this.layerProperties = new LayerArchiveSupport.LayerProperties(layerName); this.builderArguments = new ArrayList<>(); + this.buildPathDigests = new ArrayList<>(); } protected void validateLayerFile() { @@ -129,6 +138,10 @@ protected Path getEnvVariablesFilePath() { return layerDir.resolve(ENV_VARIABLES_FILE_NAME); } + protected Path getBuildPathDigestsFilePath() { + return layerDir.resolve(BUILD_PATH_DIGESTS_FILE_NAME); + } + protected List parseEnvVariables() { return System.getenv().entrySet().stream() .map(EnvironmentVariable::of) @@ -262,6 +275,58 @@ public String layerName() { } } + protected record PathDigestEntry(PathType type, String digest, String path) { + + protected static List getPathDigestEntries(NativeImageClassLoaderSupport classLoaderSupport) { + Optional buildPathDigestsOpt = classLoaderSupport.getPathDigests(true); + assert buildPathDigestsOpt.isPresent() : "NativeImageClassLoaderSupport#pathDigests should not be empty for a layered build."; + return aggregatePathDigests(buildPathDigestsOpt.get()); + } + + private static List aggregatePathDigests(NativeImageClassLoaderSupport.PathDigests pathDigests) { + List aggregatedDigests = new ArrayList<>(); + aggregatedDigests.addAll(aggregatePathDigests(pathDigests.getCpDigests(), PathType.cp)); + aggregatedDigests.addAll(aggregatePathDigests(pathDigests.getMpDigests(), PathType.mp)); + return aggregatedDigests; + } + + private static List aggregatePathDigests(EconomicMap> pathDigests, PathType type) { + List aggregatedDigests = new ArrayList<>(); + var cursor = pathDigests.getEntries(); + while (cursor.advance()) { + aggregatedDigests.add(PathDigestEntry.of(type, cursor.getKey().getPath(), cursor.getValue())); + } + return aggregatedDigests; + } + + private static PathDigestEntry of(PathType type, String path, List digests) { + DigestBuilder db = new DigestBuilder(); + digests.stream() + .sorted() + .map(d -> d.getBytes(StandardCharsets.UTF_8)) + .forEach(db::update); + + String aggregatedDigest = new String(db.digest(), StandardCharsets.UTF_8); + return new PathDigestEntry(type, aggregatedDigest, path); + } + + protected static PathDigestEntry of(String digestEntry) { + String[] envVarArr = digestEntry.split(":", 3); + return new PathDigestEntry(PathType.valueOf(envVarArr[0]), envVarArr[1], envVarArr[2]); + } + + @Override + public String toString() { + return type + ":" + digest + ":" + path; + } + + protected enum PathType { + cp, + mp + } + + } + private static String asString(Platform val) { return (val.getOS() + "-" + val.getArchitecture()).toLowerCase(Locale.ROOT); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java index 855135c05c9b..9ed1d2852c62 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java @@ -65,6 +65,7 @@ public LoadLayerArchiveSupport(String layerName, Path layerFile, Path tempDir, A this.archiveSupport.expandJarToDir(layerFile, layerDir); layerProperties.loadAndVerify(current); loadBuilderArgumentsFile(); + loadBuildPathDigestsFile(); } private void loadBuilderArgumentsFile() { @@ -85,6 +86,14 @@ private List loadEnvironmentVariablesFile() { } } + private void loadBuildPathDigestsFile() { + try (Stream lines = Files.lines(getBuildPathDigestsFilePath())) { + lines.map(PathDigestEntry::of).forEach(buildPathDigests::add); + } catch (IOException e) { + throw UserError.abort("Unable to load build path digests from file " + getBuildPathDigestsFilePath()); + } + } + @Override protected void validateLayerFile() { super.validateLayerFile(); @@ -100,6 +109,7 @@ public void verifyCompatibility(NativeImageClassLoaderSupport classLoaderSupport violationsFound |= verifyBuilderArgumentsCompatibility(builderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, true); violationsFound |= verifyBuilderArgumentsCompatibility(builderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, false); violationsFound |= verifyEnvironmentVariablesCompatibility(loadEnvironmentVariablesFile(), parseEnvVariables(), strict, verbose); + violationsFound |= verifyBuildPathDigestsCompatibility(buildPathDigests, PathDigestEntry.getPathDigestEntries(classLoaderSupport), strict, verbose); if (violationsFound && verbose) { UserError.abort("Verbose LayerOptionVerification failed."); } @@ -255,6 +265,39 @@ private static boolean verifyEnvironmentVariablesCompatibility(List previousPathDigests, List currentPathDigests, boolean strict, boolean verbose) { + Set currentDigests = new HashSet<>(currentPathDigests.size()); + currentPathDigests.forEach(pathEntry -> currentDigests.add(pathEntry.digest())); + List previousRemainingPathDigests = previousPathDigests.stream() + .filter(pathEntry -> !currentDigests.contains(pathEntry.digest())) + .toList(); + + if (previousRemainingPathDigests.isEmpty()) { + return false; + } + + Set currentPaths = new HashSet<>(currentPathDigests.size()); + currentPathDigests.forEach(pathEntry -> currentPaths.add(pathEntry.path())); + for (var remainingDigest : previousRemainingPathDigests) { + boolean pathNotFound = !currentPaths.contains(remainingDigest.path()); + String pathType = remainingDigest.type().equals(PathDigestEntry.PathType.cp) ? "classpath" : "modulepath"; + + String message = "The entry " + remainingDigest.path() + " included in the " + pathType + " of the previous layered build was" + + (pathNotFound ? " not" : "") + " found in the " + pathType + " of the current layered build" + (pathNotFound ? "." : ", but its contents have been modified."); + + if (verbose) { + LogUtils.info("Error: ", message); + } else { + if (strict) { + throw UserError.abort(message); + } else { + LogUtils.warning(message); + } + } + } + return true; + } + record ArgumentOrigin(boolean booleanOption, String argument, String origin) { record NameValue(String name, String value) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java index 9a3d266464a8..7d8ace917cc6 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java @@ -53,6 +53,7 @@ public WriteLayerArchiveSupport(String layerName, NativeImageClassLoaderSupport "' needs to start with '" + SHARED_LIB_NAME_PREFIX + "'"); } builderArguments.addAll(classLoaderSupport.getHostedOptionParser().getArguments()); + buildPathDigests.addAll(PathDigestEntry.getPathDigestEntries(classLoaderSupport)); } @Override @@ -88,6 +89,14 @@ private void writeEnvVariablesFile() { } } + private void writeBuildPathDigestsFile() { + try { + Files.write(getBuildPathDigestsFilePath(), buildPathDigests.stream().map(PathDigestEntry::toString).toList()); + } catch (IOException e) { + throw UserError.abort("Unable to write build path digests to file " + getBuildPathDigestsFilePath()); + } + } + public void write(Platform current) { try (JarOutputStream jarOutStream = new JarOutputStream(Files.newOutputStream(layerFile), archiveSupport.createManifest())) { // disable compression for significant (un)archiving speedup at the cost of file size @@ -98,6 +107,9 @@ public void write(Platform current) { // write environment variables file and add to jar writeEnvVariablesFile(); archiveSupport.addFileToJar(layerDir, getEnvVariablesFilePath(), layerFile, jarOutStream); + // write image class and module paths file and add to jar + writeBuildPathDigestsFile(); + archiveSupport.addFileToJar(layerDir, getBuildPathDigestsFilePath(), layerFile, jarOutStream); // copy the layer snapshot file and its graphs file to the jar archiveSupport.addFileToJar(layerDir, getSnapshotPath(), layerFile, jarOutStream); archiveSupport.addFileToJar(layerDir, getSnapshotGraphsPath(), layerFile, jarOutStream); From 22ae1646401c5ef20a0cc51b9dd624edf4507b23 Mon Sep 17 00:00:00 2001 From: Tudor-Stefan Magirescu Date: Wed, 10 Sep 2025 18:25:41 +0200 Subject: [PATCH 2/3] [GR-65623] Extend java.lang.module.ModuleReader#list with the ability to follow symlinks for directory modulepath entries during digest computation. --- .../ci/ci_common/benchmark-suites.libsonnet | 4 +- .../svm/core/imagelayer/NativeImageLayers.md | 28 +++- .../hosted/NativeImageClassLoaderSupport.java | 145 +++++++++++++++--- .../HostedImageLayerBuildingSupport.java | 13 +- .../imagelayer/LayerArchiveSupport.java | 64 -------- .../imagelayer/LoadLayerArchiveSupport.java | 102 +++++------- .../imagelayer/WriteLayerArchiveSupport.java | 49 +++--- 7 files changed, 224 insertions(+), 181 deletions(-) diff --git a/compiler/ci/ci_common/benchmark-suites.libsonnet b/compiler/ci/ci_common/benchmark-suites.libsonnet index 2bdc4ba80ae8..529e49dea715 100644 --- a/compiler/ci/ci_common/benchmark-suites.libsonnet +++ b/compiler/ci/ci_common/benchmark-suites.libsonnet @@ -91,7 +91,7 @@ barista_template(suite_version=null, suite_name="barista", max_jdk_version=null, cmd_app_prefix=["hwloc-bind --cpubind node:0.core:0-3.pu:0 --membind node:0"], non_prefix_barista_args=[]):: cc.compiler_benchmark + { suite:: suite_name, - local barista_version = "v0.4.7", + local barista_version = "v0.4.8", local suite_version_args = if suite_version != null then ["--bench-suite-version=" + suite_version] else [], local prefix_barista_arg = if std.length(cmd_app_prefix) > 0 then [std.format("--cmd-app-prefix=%s", std.join(" ", cmd_app_prefix))] else [], local all_barista_args = prefix_barista_arg + non_prefix_barista_args, @@ -99,7 +99,7 @@ downloads+: { "WRK": { "name": "wrk", "version": "a211dd5", platformspecific: true}, "WRK2": { "name": "wrk2", "version": "2.1", platformspecific: true}, - "BARISTA_BENCHMARKS": { "name": "barista", "version": "0.4.7"} + "BARISTA_BENCHMARKS": { "name": "barista", "version": "0.4.8"} }, packages+: { maven: "==3.8.6", diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md index 9af004890514..edd7a63d512c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md @@ -115,6 +115,8 @@ Args = --layer-create=@layer-create.args The `layer-create.args` file-path is relative to the directory that contains the `native-image.properties` file and might look like this: ``` base-layer.nil +# ignore this classpath/modulepath entry during layer compatibility checks +digest-ignore module=java.base # micronaut and dependencies package=io.micronaut.* @@ -129,6 +131,13 @@ package=org.reactivestreams.* Each line corresponds to one entry in the list of comma-separated entries that can usually be found in a regular `--layer-create` argument. Lines starting with `#` are ignored and can therefore be used to provide comments in such an option argument file. +Note the use of the `digest-ignore` suboption in the `layer-create.args` example file above. +This suboption _only takes effect_ when the `--layer-create` option is specified _within a file_, and is ignored if provided via the command line. +When defining `--layer-create` in a file, always include this suboption (see [compatibility rules](#compatibility-rules)). + +Avoid placing other class or resource files in the same classpath/modulepath entry as the file(s) defining `--layer-create`. +These files would be excluded from compatibility checks, potentially leading to subtle, difficult-to-debug issues. + ### Option `--layer-use` consumes a shared layer, and can extend it or create a final executable: ``` @@ -195,7 +204,11 @@ native-image --module-path target/AwesomeLib-1.0-SNAPSHOT.jar --shared ### Compatibility rules -Currently, layer build compatibility checking is very limited and is only performed for the image builder arguments. +Layer build compatibility checks are performed to ensure the consistency of _image builder arguments_ and _classpath/modulepath entries_. +These will be covered in the following subsections. + +#### Image builder arguments compatibility + The list below gives a few examples of checks that are already implemented. - Module system options `--add-exports`, `--add-opens`, `--add-reads` that were passed in the previous image build also need to be passed in the current image build. @@ -203,8 +216,19 @@ The list below gives a few examples of checks that are already implemented. - Builder options of the form `-H:NeverInline=` follow the same logic as the module system options above. - If debug option `-g` was passed in the previous image build it must also be passed in the current image build at the same position. - Other options like `-H:EntryPointNamePrefix=...`, `-H:APIFunctionPrefix=...`, ... follow the same logic as the `-g` option. +- Environment variables provided using `-E` to the previous image build must be passed to the current image build, with their values remaining unchanged. Additional environment variables may be supplied to the current build if needed. + +#### Classpath & modulepath compatibility + +The classpath/modulepath entries used in the previous image layer build must also be included in the current image layer build. +These entries must retain the same content. If any of the shared entries are modified (.e.g, updated jar files), the previous image layer must be rebuilt with the updated versions. + +For example, assume the previous layer build used the classpath: `/path/to/foo.jar:/path/to/bar.jar`. +Then the current layer build must include both `/path/to/foo.jar` and `/path/to/bar.jar`, possibly alongside additional entries: `path/to/foo.jar:/path/to/bar.jar:/path/to/extra.jar`. +Additionally, the contents of `foo.jar` and `bar.jar` should remain unchanged between the two builds. -The number of checks is subject to change and will be further improved in the future. +**Exception**: Classpath or modulepath entries that include a `native-image.properties` file specifying the `--layer-create` option with the `digest-ignore` suboption are exempt from this rule. +These entries should be _excluded_ from subsequent layer builds. ### Limitations diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index 75ee45b4e70a..53b3c4171e6b 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -74,6 +74,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; +import java.util.function.Predicate; import java.util.jar.JarEntry; import java.util.jar.JarFile; import java.util.stream.Collector; @@ -81,6 +82,7 @@ import java.util.stream.Stream; import java.util.zip.ZipFile; +import jdk.internal.module.Resources; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; import org.graalvm.collections.MapCursor; @@ -126,7 +128,9 @@ public final class NativeImageClassLoaderSupport { private final List buildmp; private final Set imageProvidedJars; + /** Cleared by {@link #computePathEntryDigests()} on first call. */ private PathDigests pathDigests; + private final Class explodedModuleReaderClass; private final EconomicMap> classes; private final EconomicMap> packages; @@ -289,6 +293,8 @@ protected NativeImageClassLoaderSupport(ClassLoader defaultSystemClassLoader, St annotationExtractor = new SubstrateAnnotationExtractor(); includeConfigSealed = false; + + explodedModuleReaderClass = ReflectionUtil.lookupClass(false, "jdk.internal.module.ModuleReferences$ExplodedModuleReader"); } private static Stream toRealPath(Path p) { @@ -311,11 +317,14 @@ public NativeImageClassLoader getClassLoader() { return classLoader; } - public Optional getPathDigests(boolean free) { - Optional res = Optional.ofNullable(pathDigests); - if (free) { - pathDigests = null; - } + /** + * The temporary {@link PathDigestEntry} object is used to create the list of + * {@link PathDigestEntry} tuples. Note that the {@code pathDigests} field is cleared after the + * first time this method is called. + */ + public List computePathEntryDigests() { + List res = PathDigestEntry.aggregatePathDigests(pathDigests); + pathDigests = null; return res; } @@ -986,23 +995,49 @@ private void initModule(ModuleReference moduleReference, boolean moduleRequiresI } final boolean isInImageModulePathOfLayeredBuild = pathDigests != null && pathDigests.mpDigests.containsKey(container); final boolean isJar = ClasspathUtils.isJar(Path.of(container)); - moduleReader.list().forEach(moduleResource -> { - char fileSystemSeparatorChar = '/'; - String className = extractClassName(moduleResource, fileSystemSeparatorChar); - if (className != null) { - currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource; - executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit, preserveModule)); - } - if (isInImageModulePathOfLayeredBuild) { - executor.execute(() -> PathDigests.storePathFileDigest(container, moduleResource, isJar, pathDigests.mpDigests)); - } - entriesProcessed.increment(); - }); + try (Stream moduleResources = moduleReaderListFollowSymlinks(moduleReader, container)) { + moduleResources.forEach(moduleResource -> { + char fileSystemSeparatorChar = '/'; + String className = extractClassName(moduleResource, fileSystemSeparatorChar); + if (className != null) { + currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource; + executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit, preserveModule)); + } + if (isInImageModulePathOfLayeredBuild) { + executor.execute(() -> PathDigests.storePathFileDigest(container, moduleResource, isJar, pathDigests.mpDigests)); + } + entriesProcessed.increment(); + }); + } } catch (IOException e) { throw new RuntimeException("Unable get list of resources in module" + moduleReference.descriptor().name(), e); } } + /** + * This method extends {@link ModuleReader#list} with the ability to follow symlinks when + * the given {@link ModuleReader} corresponds to a + * {@code jdk.internal.module.ModuleReferences$ExplodedModuleReader} (i.e., + * {@code container} is a directory which contains the contents of the module). + * + * Classloaders (e.g., {@link NativeImageClassLoader#findResource(String, String)}) can load + * resources pointed to by a symlink present on the modulepath, meaning the extension below + * is necessary if we want to keep track of such resources (e.g., for digest computation in + * layered builds). + */ + @SuppressWarnings("resource") + private Stream moduleReaderListFollowSymlinks(ModuleReader reader, URI container) throws IOException { + if (explodedModuleReaderClass == null || !reader.getClass().isAssignableFrom(explodedModuleReaderClass)) { + return reader.list(); + } + + Path dir = Path.of(container); + // See {@code jdk.internal.module.ModuleReferences$ExplodedModuleReader#list}. + return Files.walk(dir, Integer.MAX_VALUE, FileVisitOption.FOLLOW_LINKS) + .map(f -> Resources.toResourceName(dir, f)) + .filter(Predicate.not(String::isEmpty)); + } + private void loadClassesFromPath(Path path) { final boolean includeUnconditionally = layerSelectors.classpathEntries().contains(path); final boolean includeAllMetadata = preserveSelectors.classpathEntries().contains(path); @@ -1434,7 +1469,16 @@ public Set packages() { } } - public static final class PathDigests { + /** + * Stores a temporary collection of individual class/resource file digests that is updated + * during class loading. In particular, {@code PathDigests} objects store and update two + * {@link EconomicMap}s, one for classpath entries and the other for modulepath entries. Each + * {@link EconomicMap} maps a path entry (i.e., directory/jar) with the list of individual + * digests for the files it contains. The order of the individual digests is non-deterministic. + * After class loading, the {@code PathDigests} should be aggregated into a list of + * {@link PathDigestEntry}. + */ + private static final class PathDigests { private final EconomicMap> cpDigests = EconomicMap.create(); private final EconomicMap> mpDigests = EconomicMap.create(); @@ -1475,12 +1519,73 @@ private static void storePathFileDigest(URI container, String resource, boolean } } - public EconomicMap> getCpDigests() { + private EconomicMap> getCpDigests() { return cpDigests; } - public EconomicMap> getMpDigests() { + private EconomicMap> getMpDigests() { return mpDigests; } } + + /** + * The record type {@link PathDigestEntry} encodes tuples of the form (type, digest, path), + * where: - type: is either {@code PathType.cp} or {@code PathType.mp} - digest: is a checksum + * which encodes information about every class/resource file part of that path (or reachable + * from a symlink part of that path) - path: is the absolute path ponting to the directory/jar + * included on the class/module-path. To obtain a list of all {@code PathDigestEntry} objects + * corresponding to a particular Native Image build, use the method + * {@link NativeImageClassLoaderSupport#computePathEntryDigests()}. + */ + public record PathDigestEntry(PathType type, String digest, String path) { + + /** + * Aggregate the contents of {@link PathDigests} to create a list of + * {@link PathDigestEntry}. The individual file digests of a path entry are aggregated into + * a final digest that encodes information about all the contents of that path entry. + */ + private static List aggregatePathDigests(PathDigests pathDigests) { + Objects.requireNonNull(pathDigests, "NativeImageClassLoaderSupport#pathDigests should not be empty for a layered build."); + + List aggregatedDigests = new ArrayList<>(); + aggregatedDigests.addAll(aggregatePathDigests(pathDigests.getCpDigests(), PathType.cp)); + aggregatedDigests.addAll(aggregatePathDigests(pathDigests.getMpDigests(), PathType.mp)); + return aggregatedDigests; + } + + private static List aggregatePathDigests(EconomicMap> pathDigests, PathType type) { + List aggregatedDigests = new ArrayList<>(); + var cursor = pathDigests.getEntries(); + while (cursor.advance()) { + aggregatedDigests.add(PathDigestEntry.of(type, cursor.getKey().getPath(), cursor.getValue())); + } + return aggregatedDigests; + } + + private static PathDigestEntry of(PathType type, String path, List digests) { + DigestBuilder db = new DigestBuilder(); + digests.stream() + .sorted() + .map(d -> d.getBytes(StandardCharsets.UTF_8)) + .forEach(db::update); + + String aggregatedDigest = new String(db.digest(), StandardCharsets.UTF_8); + return new PathDigestEntry(type, aggregatedDigest, path); + } + + public static PathDigestEntry of(String digestEntry) { + String[] envVarArr = digestEntry.split(":", 3); + return new PathDigestEntry(PathType.valueOf(envVarArr[0]), envVarArr[1], envVarArr[2]); + } + + @Override + public String toString() { + return type + ":" + digest + ":" + path; + } + + public enum PathType { + cp, + mp + } + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java index 08b106333996..42b229703ace 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java @@ -100,6 +100,15 @@ private static String layerCreatePossibleOptions() { * {@link #initialize}. */ private final Function, SingletonTrait[]> singletonTraitInjector; + /** + * Optional suboption of the {@link SubstrateOptions#LayerCreate} option. If the `LayerCreate` + * option is specified inside a `native-image.properties` file and this suboption is enabled, + * the classpath/modulepath entry containing the `native-image.properties` file will be excluded + * from the classpath/modulepath layered compatibility check. This suboption has no effect if + * it's specified from the command line. See + * {@link #processLayerOptions(EconomicMap, NativeImageClassLoaderSupport)} for more details. + */ + private static final String DIGEST_IGNORE = "digest-ignore"; private HostedImageLayerBuildingSupport(ImageClassLoader imageClassLoader, Reader snapshot, List graphsChannels, @@ -206,7 +215,7 @@ public static void processLayerOptions(EconomicMap, Object> values, classLoaderSupport.setLayerFile(layerFile); NativeImageClassLoaderSupport.IncludeSelectors layerSelectors = classLoaderSupport.getLayerSelectors(); - IncludeOptionsSupport.ExtendedOption digestIgnoreExtendedOption = new IncludeOptionsSupport.ExtendedOption("digest-ignore", null); + IncludeOptionsSupport.ExtendedOption digestIgnoreExtendedOption = new IncludeOptionsSupport.ExtendedOption(DIGEST_IGNORE, null); for (IncludeOptionsSupport.ExtendedOption option : layerOption.extendedOptions()) { if (option.equals(digestIgnoreExtendedOption)) { digestIgnorePath = valueWithOrigin.origin().location(); @@ -295,7 +304,7 @@ private static String getLayerCreateValue(ValueWithOrigin valueWithOrigi return String.join(",", OptionUtils.resolveOptionValuesRedirection(SubstrateOptions.LayerCreate, valueWithOrigin)); } - public static boolean isLayerUseOptionEnabled(OptionValues values) { + private static boolean isLayerUseOptionEnabled(OptionValues values) { if (SubstrateOptions.LayerUse.hasBeenSet(values)) { return !getLayerUseValue(values).toString().isEmpty(); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java index de0e9e95af68..9461a104ea99 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java @@ -25,31 +25,24 @@ package com.oracle.svm.hosted.imagelayer; import static com.oracle.svm.core.util.EnvVariableUtils.EnvironmentVariable; -import static jdk.graal.compiler.util.Digest.DigestBuilder; import java.io.IOException; import java.io.OutputStream; -import java.net.URI; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Objects; -import java.util.Optional; import java.util.Properties; -import org.graalvm.collections.EconomicMap; import org.graalvm.nativeimage.Platform; import com.oracle.svm.core.SharedConstants; import com.oracle.svm.core.util.ArchiveSupport; import com.oracle.svm.core.util.UserError; import com.oracle.svm.core.util.VMError; -import com.oracle.svm.hosted.NativeImageClassLoaderSupport; import com.oracle.svm.util.LogUtils; public class LayerArchiveSupport { @@ -68,9 +61,6 @@ public class LayerArchiveSupport { public static final String LAYER_FILE_EXTENSION = ".nil"; - protected final List builderArguments; - protected final List buildPathDigests; - protected final LayerProperties layerProperties; protected final Path layerFile; protected final ArchiveSupport archiveSupport; @@ -93,8 +83,6 @@ public LayerArchiveSupport(String layerName, Path layerFile, Path layerDir, Arch } this.layerProperties = new LayerArchiveSupport.LayerProperties(layerName); - this.builderArguments = new ArrayList<>(); - this.buildPathDigests = new ArrayList<>(); } protected void validateLayerFile() { @@ -275,58 +263,6 @@ public String layerName() { } } - protected record PathDigestEntry(PathType type, String digest, String path) { - - protected static List getPathDigestEntries(NativeImageClassLoaderSupport classLoaderSupport) { - Optional buildPathDigestsOpt = classLoaderSupport.getPathDigests(true); - assert buildPathDigestsOpt.isPresent() : "NativeImageClassLoaderSupport#pathDigests should not be empty for a layered build."; - return aggregatePathDigests(buildPathDigestsOpt.get()); - } - - private static List aggregatePathDigests(NativeImageClassLoaderSupport.PathDigests pathDigests) { - List aggregatedDigests = new ArrayList<>(); - aggregatedDigests.addAll(aggregatePathDigests(pathDigests.getCpDigests(), PathType.cp)); - aggregatedDigests.addAll(aggregatePathDigests(pathDigests.getMpDigests(), PathType.mp)); - return aggregatedDigests; - } - - private static List aggregatePathDigests(EconomicMap> pathDigests, PathType type) { - List aggregatedDigests = new ArrayList<>(); - var cursor = pathDigests.getEntries(); - while (cursor.advance()) { - aggregatedDigests.add(PathDigestEntry.of(type, cursor.getKey().getPath(), cursor.getValue())); - } - return aggregatedDigests; - } - - private static PathDigestEntry of(PathType type, String path, List digests) { - DigestBuilder db = new DigestBuilder(); - digests.stream() - .sorted() - .map(d -> d.getBytes(StandardCharsets.UTF_8)) - .forEach(db::update); - - String aggregatedDigest = new String(db.digest(), StandardCharsets.UTF_8); - return new PathDigestEntry(type, aggregatedDigest, path); - } - - protected static PathDigestEntry of(String digestEntry) { - String[] envVarArr = digestEntry.split(":", 3); - return new PathDigestEntry(PathType.valueOf(envVarArr[0]), envVarArr[1], envVarArr[2]); - } - - @Override - public String toString() { - return type + ":" + digest + ":" + path; - } - - protected enum PathType { - cp, - mp - } - - } - private static String asString(Platform val) { return (val.getOS() + "-" + val.getArchitecture()).toLowerCase(Locale.ROOT); } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java index 9ed1d2852c62..35fa8bb19c38 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java @@ -25,6 +25,7 @@ package com.oracle.svm.hosted.imagelayer; import static com.oracle.svm.core.util.EnvVariableUtils.EnvironmentVariable; +import static com.oracle.svm.hosted.NativeImageClassLoaderSupport.PathDigestEntry; import java.io.IOException; import java.nio.file.Files; @@ -64,33 +65,13 @@ public LoadLayerArchiveSupport(String layerName, Path layerFile, Path tempDir, A super(layerName, layerFile, tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "load"), archiveSupport); this.archiveSupport.expandJarToDir(layerFile, layerDir); layerProperties.loadAndVerify(current); - loadBuilderArgumentsFile(); - loadBuildPathDigestsFile(); } - private void loadBuilderArgumentsFile() { - try (Stream lines = Files.lines(getBuilderArgumentsFilePath())) { - lines.forEach(builderArguments::add); + private List loadBuildEntries(Path path, Function stringParser, String buildEntryTypeDescription) { + try (Stream lines = Files.lines(path)) { + return lines.map(stringParser).toList(); } catch (IOException e) { - throw UserError.abort("Unable to load builder arguments from file " + getBuilderArgumentsFilePath()); - } - } - - private List loadEnvironmentVariablesFile() { - List envVariables = new ArrayList<>(); - try (Stream lines = Files.lines(getEnvVariablesFilePath())) { - lines.map(EnvironmentVariable::of).forEach(envVariables::add); - return envVariables; - } catch (IOException e) { - throw UserError.abort("Unable to load environment variables from file " + getEnvVariablesFilePath()); - } - } - - private void loadBuildPathDigestsFile() { - try (Stream lines = Files.lines(getBuildPathDigestsFilePath())) { - lines.map(PathDigestEntry::of).forEach(buildPathDigests::add); - } catch (IOException e) { - throw UserError.abort("Unable to load build path digests from file " + getBuildPathDigestsFilePath()); + throw UserError.abort("Unable to load " + buildEntryTypeDescription + " from file " + path); } } @@ -106,13 +87,19 @@ protected void validateLayerFile() { public void verifyCompatibility(NativeImageClassLoaderSupport classLoaderSupport, Map allRequests, boolean strict, boolean verbose) { Function filterFunction = argument -> splitArgumentOrigin(argument).argument; boolean violationsFound = false; - violationsFound |= verifyBuilderArgumentsCompatibility(builderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, true); - violationsFound |= verifyBuilderArgumentsCompatibility(builderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, false); - violationsFound |= verifyEnvironmentVariablesCompatibility(loadEnvironmentVariablesFile(), parseEnvVariables(), strict, verbose); - violationsFound |= verifyBuildPathDigestsCompatibility(buildPathDigests, PathDigestEntry.getPathDigestEntries(classLoaderSupport), strict, verbose); + List previousBuilderArguments = loadBuildEntries(getBuilderArgumentsFilePath(), Function.identity(), "builder arguments"); + violationsFound |= verifyBuilderArgumentsCompatibility(previousBuilderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, true); + violationsFound |= verifyBuilderArgumentsCompatibility(previousBuilderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, + false); if (violationsFound && verbose) { UserError.abort("Verbose LayerOptionVerification failed."); } + + List previousEnvironmentVariables = loadBuildEntries(getEnvVariablesFilePath(), EnvironmentVariable::of, "environment variables"); + verifyEnvironmentVariablesCompatibility(previousEnvironmentVariables, parseEnvVariables(), strict); + + List previousPathDigestEntries = loadBuildEntries(getBuildPathDigestsFilePath(), PathDigestEntry::of, "build path digests"); + verifyBuildPathDigestsCompatibility(previousPathDigestEntries, classLoaderSupport.computePathEntryDigests(), strict); } private static boolean verifyBuilderArgumentsCompatibility(List previousArgs, List currentArgs, Function filterFunction, @@ -240,62 +227,53 @@ private static boolean verifyBuilderArgumentsCompatibility(List previous * but are not considered by this verification, since mismatches between these don't affect * layer compatibility. */ - private static boolean verifyEnvironmentVariablesCompatibility(List previousEnvVars, List currentEnvVars, boolean strict, boolean verbose) { + private static void verifyEnvironmentVariablesCompatibility(List previousEnvVars, List currentEnvVars, boolean strict) { Set currentEnvVarsSet = new HashSet<>(currentEnvVars); boolean violationsFound = false; + String messagePrefix = strict ? "Error" : "Warning"; for (EnvironmentVariable previousEnvVar : previousEnvVars) { if (currentEnvVarsSet.contains(previousEnvVar)) { continue; } - violationsFound = true; String message = "Current layered image build must set environment variable " + previousEnvVar + " as it was set in the previous layer build."; - if (verbose) { - LogUtils.info("Error: ", message); - } else { - if (strict) { - throw UserError.abort(message); - } else { - LogUtils.warning(message); - } - } + LogUtils.info(messagePrefix, message); } - return violationsFound; + if (violationsFound && strict) { + throw UserError.abort("Environment variable layer compatibility check failed."); + } } - private static boolean verifyBuildPathDigestsCompatibility(List previousPathDigests, List currentPathDigests, boolean strict, boolean verbose) { + private static void verifyBuildPathDigestsCompatibility(List previousPathDigests, List currentPathDigests, boolean strict) { Set currentDigests = new HashSet<>(currentPathDigests.size()); currentPathDigests.forEach(pathEntry -> currentDigests.add(pathEntry.digest())); - List previousRemainingPathDigests = previousPathDigests.stream() + + List previousUnmatchedPathDigests = previousPathDigests.stream() .filter(pathEntry -> !currentDigests.contains(pathEntry.digest())) .toList(); - - if (previousRemainingPathDigests.isEmpty()) { - return false; + if (previousUnmatchedPathDigests.isEmpty()) { + // All class/module-path entry digests from the previous layered build were present in + // the current layered build. + // Therefore, the compatibility check passes and we report no violations. + return; } - Set currentPaths = new HashSet<>(currentPathDigests.size()); - currentPathDigests.forEach(pathEntry -> currentPaths.add(pathEntry.path())); - for (var remainingDigest : previousRemainingPathDigests) { - boolean pathNotFound = !currentPaths.contains(remainingDigest.path()); - String pathType = remainingDigest.type().equals(PathDigestEntry.PathType.cp) ? "classpath" : "modulepath"; - - String message = "The entry " + remainingDigest.path() + " included in the " + pathType + " of the previous layered build was" + + Set currentPathNames = new HashSet<>(currentPathDigests.size()); + currentPathDigests.forEach(pathEntry -> currentPathNames.add(pathEntry.path())); + String messagePrefix = strict ? "Error" : "Warning"; + for (PathDigestEntry unmatchedDigestEntry : previousUnmatchedPathDigests) { + boolean pathNotFound = !currentPathNames.contains(unmatchedDigestEntry.path()); + String pathType = unmatchedDigestEntry.type().equals(PathDigestEntry.PathType.cp) ? "classpath" : "modulepath"; + String message = "The entry " + unmatchedDigestEntry.path() + " included in the " + pathType + " of the previous layered build was" + (pathNotFound ? " not" : "") + " found in the " + pathType + " of the current layered build" + (pathNotFound ? "." : ", but its contents have been modified."); + LogUtils.info(messagePrefix, message); + } - if (verbose) { - LogUtils.info("Error: ", message); - } else { - if (strict) { - throw UserError.abort(message); - } else { - LogUtils.warning(message); - } - } + if (strict) { + throw UserError.abort("Class/Module-path layer compatibility check failed."); } - return true; } record ArgumentOrigin(boolean booleanOption, String argument, String origin) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java index 7d8ace917cc6..7aaaef9562e5 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java @@ -24,11 +24,13 @@ */ package com.oracle.svm.hosted.imagelayer; -import static com.oracle.svm.core.util.EnvVariableUtils.EnvironmentVariable; +import static com.oracle.svm.hosted.NativeImageClassLoaderSupport.PathDigestEntry; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.jar.JarOutputStream; import org.graalvm.nativeimage.Platform; @@ -44,6 +46,8 @@ /* Builds an image layer, either initial or intermediate. */ public class WriteLayerArchiveSupport extends LayerArchiveSupport { + private final List builderArguments; + private final List buildPathDigests; public WriteLayerArchiveSupport(String layerName, NativeImageClassLoaderSupport classLoaderSupport, Path tempDir, ArchiveSupport archiveSupport) { super(layerName, classLoaderSupport.getLayerFile(), tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "write"), archiveSupport); @@ -52,8 +56,8 @@ public WriteLayerArchiveSupport(String layerName, NativeImageClassLoaderSupport SubstrateOptionsParser.commandArgument(SubstrateOptions.Name, layerName) + "' needs to start with '" + SHARED_LIB_NAME_PREFIX + "'"); } - builderArguments.addAll(classLoaderSupport.getHostedOptionParser().getArguments()); - buildPathDigests.addAll(PathDigestEntry.getPathDigestEntries(classLoaderSupport)); + builderArguments = new ArrayList<>(classLoaderSupport.getHostedOptionParser().getArguments()); + buildPathDigests = new ArrayList<>(classLoaderSupport.computePathEntryDigests()); } @Override @@ -73,27 +77,11 @@ protected void validateLayerFile() { } } - private void writeBuilderArgumentsFile() { + private void writeBuildEntries(Path path, List buildEntries, String buildEntryTypeDescription) { try { - Files.write(getBuilderArgumentsFilePath(), builderArguments); + Files.write(path, buildEntries.stream().map(T::toString).toList()); } catch (IOException e) { - throw UserError.abort("Unable to write builder arguments to file " + getBuilderArgumentsFilePath()); - } - } - - private void writeEnvVariablesFile() { - try { - Files.write(getEnvVariablesFilePath(), parseEnvVariables().stream().map(EnvironmentVariable::toString).toList()); - } catch (IOException e) { - throw UserError.abort("Unable to write environment variables to file " + getEnvVariablesFilePath()); - } - } - - private void writeBuildPathDigestsFile() { - try { - Files.write(getBuildPathDigestsFilePath(), buildPathDigests.stream().map(PathDigestEntry::toString).toList()); - } catch (IOException e) { - throw UserError.abort("Unable to write build path digests to file " + getBuildPathDigestsFilePath()); + throw UserError.abort("Unable to write " + buildEntryTypeDescription + " to file " + path); } } @@ -102,14 +90,17 @@ public void write(Platform current) { // disable compression for significant (un)archiving speedup at the cost of file size jarOutStream.setLevel(0); // write builder arguments file and add to jar - writeBuilderArgumentsFile(); - archiveSupport.addFileToJar(layerDir, getBuilderArgumentsFilePath(), layerFile, jarOutStream); + Path builderArgumentsFilePath = getBuilderArgumentsFilePath(); + writeBuildEntries(builderArgumentsFilePath, builderArguments, "builder arguments"); + archiveSupport.addFileToJar(layerDir, builderArgumentsFilePath, layerFile, jarOutStream); // write environment variables file and add to jar - writeEnvVariablesFile(); - archiveSupport.addFileToJar(layerDir, getEnvVariablesFilePath(), layerFile, jarOutStream); - // write image class and module paths file and add to jar - writeBuildPathDigestsFile(); - archiveSupport.addFileToJar(layerDir, getBuildPathDigestsFilePath(), layerFile, jarOutStream); + Path envVariablesFilePath = getEnvVariablesFilePath(); + writeBuildEntries(envVariablesFilePath, parseEnvVariables(), "environment variables"); + archiveSupport.addFileToJar(layerDir, envVariablesFilePath, layerFile, jarOutStream); + // write image class and module paths digests file and add to jar + Path buildPathDigestsFilePath = getBuildPathDigestsFilePath(); + writeBuildEntries(buildPathDigestsFilePath, buildPathDigests, "build path digests"); + archiveSupport.addFileToJar(layerDir, buildPathDigestsFilePath, layerFile, jarOutStream); // copy the layer snapshot file and its graphs file to the jar archiveSupport.addFileToJar(layerDir, getSnapshotPath(), layerFile, jarOutStream); archiveSupport.addFileToJar(layerDir, getSnapshotGraphsPath(), layerFile, jarOutStream); From 9185afd96a78c4e55da0574b851795bb4e6da3dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Paul=20W=C3=B6gerer?= Date: Tue, 30 Sep 2025 15:49:01 +0200 Subject: [PATCH 3/3] Style fixes --- .../com/oracle/svm/hosted/NativeImageClassLoaderSupport.java | 4 ++-- .../oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java | 2 +- .../svm/hosted/imagelayer/WriteLayerArchiveSupport.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index 53b3c4171e6b..b255031a1d39 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -82,7 +82,6 @@ import java.util.stream.Stream; import java.util.zip.ZipFile; -import jdk.internal.module.Resources; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; import org.graalvm.collections.MapCursor; @@ -117,6 +116,7 @@ import jdk.graal.compiler.options.OptionKey; import jdk.graal.compiler.options.OptionValues; import jdk.internal.module.Modules; +import jdk.internal.module.Resources; public final class NativeImageClassLoaderSupport { @@ -350,7 +350,7 @@ public void initializePathDigests(Path digestIgnoreRelativePath) { pathDigests = new PathDigests(filterIgnoredPathEntries(imagecp, digestIgnorePaths), filterIgnoredPathEntries(imagemp, digestIgnorePaths)); } - private List filterIgnoredPathEntries(List pathEntries, List digestIgnorePaths) { + private static List filterIgnoredPathEntries(List pathEntries, List digestIgnorePaths) { return pathEntries.stream().filter(pathEntry -> { for (Path p : digestIgnorePaths) { if (p.startsWith(pathEntry)) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java index 35fa8bb19c38..ac2dc55c3876 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java @@ -67,7 +67,7 @@ public LoadLayerArchiveSupport(String layerName, Path layerFile, Path tempDir, A layerProperties.loadAndVerify(current); } - private List loadBuildEntries(Path path, Function stringParser, String buildEntryTypeDescription) { + private static List loadBuildEntries(Path path, Function stringParser, String buildEntryTypeDescription) { try (Stream lines = Files.lines(path)) { return lines.map(stringParser).toList(); } catch (IOException e) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java index 7aaaef9562e5..9e52a800e85d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java @@ -77,7 +77,7 @@ protected void validateLayerFile() { } } - private void writeBuildEntries(Path path, List buildEntries, String buildEntryTypeDescription) { + private static void writeBuildEntries(Path path, List buildEntries, String buildEntryTypeDescription) { try { Files.write(path, buildEntries.stream().map(T::toString).toList()); } catch (IOException e) {