diff --git a/compiler/ci/ci_common/benchmark-suites.libsonnet b/compiler/ci/ci_common/benchmark-suites.libsonnet index 2bdc4ba80ae8..529e49dea715 100644 --- a/compiler/ci/ci_common/benchmark-suites.libsonnet +++ b/compiler/ci/ci_common/benchmark-suites.libsonnet @@ -91,7 +91,7 @@ barista_template(suite_version=null, suite_name="barista", max_jdk_version=null, cmd_app_prefix=["hwloc-bind --cpubind node:0.core:0-3.pu:0 --membind node:0"], non_prefix_barista_args=[]):: cc.compiler_benchmark + { suite:: suite_name, - local barista_version = "v0.4.7", + local barista_version = "v0.4.8", local suite_version_args = if suite_version != null then ["--bench-suite-version=" + suite_version] else [], local prefix_barista_arg = if std.length(cmd_app_prefix) > 0 then [std.format("--cmd-app-prefix=%s", std.join(" ", cmd_app_prefix))] else [], local all_barista_args = prefix_barista_arg + non_prefix_barista_args, @@ -99,7 +99,7 @@ downloads+: { "WRK": { "name": "wrk", "version": "a211dd5", platformspecific: true}, "WRK2": { "name": "wrk2", "version": "2.1", platformspecific: true}, - "BARISTA_BENCHMARKS": { "name": "barista", "version": "0.4.7"} + "BARISTA_BENCHMARKS": { "name": "barista", "version": "0.4.8"} }, packages+: { maven: "==3.8.6", diff --git a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java index 8ecc1ab2251b..c7950d729c12 100644 --- a/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java +++ b/compiler/src/jdk.graal.compiler/src/jdk/graal/compiler/util/Digest.java @@ -102,8 +102,10 @@ public static String digest(byte[] bytes, int offset, int length) { * new byte array. */ public static byte[] digestAsByteArray(byte[] bytes, int offset, int length) { - LongLong hash = MurmurHash3_x64_128(bytes, offset, length, HASH_SEED); + return longLongToByteArray(MurmurHash3_x64_128(bytes, offset, length, HASH_SEED)); + } + private static byte[] longLongToByteArray(LongLong hash) { byte[] array = new byte[DIGEST_SIZE]; encodeBase62(hash.l1, array, 0); encodeBase62(hash.l2, array, BASE62_DIGITS_PER_LONG); @@ -281,4 +283,20 @@ private static long fmix64(long input) { return k; } + + public static final class DigestBuilder { + private LongLong digest; + + public DigestBuilder() { + digest = new LongLong(0L, 0L); + } + + public void update(byte[] input) { + digest = MurmurHash3_x64_128(input, 0, input.length, digest.l1 ^ digest.l2); + } + + public byte[] digest() { + return longLongToByteArray(digest); + } + } } diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md index 9af004890514..edd7a63d512c 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/imagelayer/NativeImageLayers.md @@ -115,6 +115,8 @@ Args = --layer-create=@layer-create.args The `layer-create.args` file-path is relative to the directory that contains the `native-image.properties` file and might look like this: ``` base-layer.nil +# ignore this classpath/modulepath entry during layer compatibility checks +digest-ignore module=java.base # micronaut and dependencies package=io.micronaut.* @@ -129,6 +131,13 @@ package=org.reactivestreams.* Each line corresponds to one entry in the list of comma-separated entries that can usually be found in a regular `--layer-create` argument. Lines starting with `#` are ignored and can therefore be used to provide comments in such an option argument file. +Note the use of the `digest-ignore` suboption in the `layer-create.args` example file above. +This suboption _only takes effect_ when the `--layer-create` option is specified _within a file_, and is ignored if provided via the command line. +When defining `--layer-create` in a file, always include this suboption (see [compatibility rules](#compatibility-rules)). + +Avoid placing other class or resource files in the same classpath/modulepath entry as the file(s) defining `--layer-create`. +These files would be excluded from compatibility checks, potentially leading to subtle, difficult-to-debug issues. + ### Option `--layer-use` consumes a shared layer, and can extend it or create a final executable: ``` @@ -195,7 +204,11 @@ native-image --module-path target/AwesomeLib-1.0-SNAPSHOT.jar --shared ### Compatibility rules -Currently, layer build compatibility checking is very limited and is only performed for the image builder arguments. +Layer build compatibility checks are performed to ensure the consistency of _image builder arguments_ and _classpath/modulepath entries_. +These will be covered in the following subsections. + +#### Image builder arguments compatibility + The list below gives a few examples of checks that are already implemented. - Module system options `--add-exports`, `--add-opens`, `--add-reads` that were passed in the previous image build also need to be passed in the current image build. @@ -203,8 +216,19 @@ The list below gives a few examples of checks that are already implemented. - Builder options of the form `-H:NeverInline=` follow the same logic as the module system options above. - If debug option `-g` was passed in the previous image build it must also be passed in the current image build at the same position. - Other options like `-H:EntryPointNamePrefix=...`, `-H:APIFunctionPrefix=...`, ... follow the same logic as the `-g` option. +- Environment variables provided using `-E` to the previous image build must be passed to the current image build, with their values remaining unchanged. Additional environment variables may be supplied to the current build if needed. + +#### Classpath & modulepath compatibility + +The classpath/modulepath entries used in the previous image layer build must also be included in the current image layer build. +These entries must retain the same content. If any of the shared entries are modified (.e.g, updated jar files), the previous image layer must be rebuilt with the updated versions. + +For example, assume the previous layer build used the classpath: `/path/to/foo.jar:/path/to/bar.jar`. +Then the current layer build must include both `/path/to/foo.jar` and `/path/to/bar.jar`, possibly alongside additional entries: `path/to/foo.jar:/path/to/bar.jar:/path/to/extra.jar`. +Additionally, the contents of `foo.jar` and `bar.jar` should remain unchanged between the two builds. -The number of checks is subject to change and will be further improved in the future. +**Exception**: Classpath or modulepath entries that include a `native-image.properties` file specifying the `--layer-create` option with the `digest-ignore` suboption are exempt from this rule. +These entries should be _excluded_ from subsequent layer builds. ### Limitations diff --git a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java index 712eff9d3cf9..7f3d2d5d17e1 100644 --- a/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java +++ b/substratevm/src/com.oracle.svm.core/src/com/oracle/svm/core/jdk/JRTSupport.java @@ -29,6 +29,7 @@ import java.net.URL; import java.net.URLConnection; import java.nio.ByteBuffer; +import java.nio.file.FileSystem; import java.nio.file.Path; import java.util.Arrays; import java.util.HashMap; @@ -181,6 +182,13 @@ protected URLConnection openConnection(URL url) throws IOException { final class Target_jdk_internal_jrtfs_JrtFileSystemProvider_JRTDisabled { } +@TargetClass(className = "jdk.internal.jrtfs.JrtFileSystemProvider", onlyWith = JRTEnabled.class) +final class Target_jdk_internal_jrtfs_JrtFileSystemProvider_BuildTime { + @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Reset)// + @Alias// + volatile FileSystem theFileSystem; +} + // endregion Disable jimage/jrtfs @TargetClass(className = "jdk.internal.jimage.BasicImageReader") diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java index ccd1963e1637..b255031a1d39 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/NativeImageClassLoaderSupport.java @@ -24,6 +24,8 @@ */ package com.oracle.svm.hosted; +import static jdk.graal.compiler.util.Digest.DigestBuilder; + import java.io.File; import java.io.IOException; import java.lang.module.Configuration; @@ -36,9 +38,12 @@ import java.lang.reflect.Method; import java.net.URI; import java.net.URISyntaxException; +import java.net.URL; import java.nio.channels.ClosedByInterruptException; +import java.nio.charset.StandardCharsets; import java.nio.file.FileSystem; import java.nio.file.FileSystems; +import java.nio.file.FileVisitOption; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; import java.nio.file.Files; @@ -52,6 +57,8 @@ import java.util.Collections; import java.util.Comparator; import java.util.Deque; +import java.util.EnumSet; +import java.util.Enumeration; import java.util.HashSet; import java.util.LinkedHashMap; import java.util.LinkedHashSet; @@ -67,9 +74,13 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.LongAdder; import java.util.function.BiConsumer; +import java.util.function.Predicate; +import java.util.jar.JarEntry; +import java.util.jar.JarFile; import java.util.stream.Collector; import java.util.stream.Collectors; import java.util.stream.Stream; +import java.util.zip.ZipFile; import org.graalvm.collections.EconomicMap; import org.graalvm.collections.EconomicSet; @@ -105,6 +116,7 @@ import jdk.graal.compiler.options.OptionKey; import jdk.graal.compiler.options.OptionValues; import jdk.internal.module.Modules; +import jdk.internal.module.Resources; public final class NativeImageClassLoaderSupport { @@ -116,6 +128,9 @@ public final class NativeImageClassLoaderSupport { private final List buildmp; private final Set imageProvidedJars; + /** Cleared by {@link #computePathEntryDigests()} on first call. */ + private PathDigests pathDigests; + private final Class explodedModuleReaderClass; private final EconomicMap> classes; private final EconomicMap> packages; @@ -278,6 +293,8 @@ protected NativeImageClassLoaderSupport(ClassLoader defaultSystemClassLoader, St annotationExtractor = new SubstrateAnnotationExtractor(); includeConfigSealed = false; + + explodedModuleReaderClass = ReflectionUtil.lookupClass(false, "jdk.internal.module.ModuleReferences$ExplodedModuleReader"); } private static Stream toRealPath(Path p) { @@ -300,6 +317,50 @@ public NativeImageClassLoader getClassLoader() { return classLoader; } + /** + * The temporary {@link PathDigestEntry} object is used to create the list of + * {@link PathDigestEntry} tuples. Note that the {@code pathDigests} field is cleared after the + * first time this method is called. + */ + public List computePathEntryDigests() { + List res = PathDigestEntry.aggregatePathDigests(pathDigests); + pathDigests = null; + return res; + } + + public void initializePathDigests(Path digestIgnoreRelativePath) { + List digestIgnorePaths = new ArrayList<>(); + if (digestIgnoreRelativePath != null) { + try { + Enumeration urls = classLoader.getResources(digestIgnoreRelativePath.toString()); + while (urls.hasMoreElements()) { + URI uri = urls.nextElement().toURI(); + if ("jar".equalsIgnoreCase(uri.getScheme())) { + String uriString = uri.toString(); + String fileUriString = uriString.substring("jar:".length(), uriString.indexOf("!/")); + uri = URI.create(fileUriString); + } + digestIgnorePaths.add(Path.of(uri)); + } + } catch (URISyntaxException | IOException e) { + throw UserError.abort("Error while looking for %s in %s", digestIgnoreRelativePath, classLoader); + } + } + + pathDigests = new PathDigests(filterIgnoredPathEntries(imagecp, digestIgnorePaths), filterIgnoredPathEntries(imagemp, digestIgnorePaths)); + } + + private static List filterIgnoredPathEntries(List pathEntries, List digestIgnorePaths) { + return pathEntries.stream().filter(pathEntry -> { + for (Path p : digestIgnorePaths) { + if (p.startsWith(pathEntry)) { + return false; + } + } + return true; + }).toList(); + } + public LibGraalLoader getLibGraalLoader() { VMError.guarantee(libGraalLoader != null, "Invalid access to libGraalLoader before getting set up"); return libGraalLoader.orElse(null); @@ -932,20 +993,51 @@ private void initModule(ModuleReference moduleReference, boolean moduleRequiresI if (ModuleLayer.boot().equals(module.getLayer())) { builderURILocations.add(container); } - moduleReader.list().forEach(moduleResource -> { - char fileSystemSeparatorChar = '/'; - String className = extractClassName(moduleResource, fileSystemSeparatorChar); - if (className != null) { - currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource; - executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit, preserveModule)); - } - entriesProcessed.increment(); - }); + final boolean isInImageModulePathOfLayeredBuild = pathDigests != null && pathDigests.mpDigests.containsKey(container); + final boolean isJar = ClasspathUtils.isJar(Path.of(container)); + try (Stream moduleResources = moduleReaderListFollowSymlinks(moduleReader, container)) { + moduleResources.forEach(moduleResource -> { + char fileSystemSeparatorChar = '/'; + String className = extractClassName(moduleResource, fileSystemSeparatorChar); + if (className != null) { + currentlyProcessedEntry = moduleReferenceLocation + fileSystemSeparatorChar + moduleResource; + executor.execute(() -> handleClassFileName(container, module, className, includeUnconditionally, moduleRequiresInit, preserveModule)); + } + if (isInImageModulePathOfLayeredBuild) { + executor.execute(() -> PathDigests.storePathFileDigest(container, moduleResource, isJar, pathDigests.mpDigests)); + } + entriesProcessed.increment(); + }); + } } catch (IOException e) { throw new RuntimeException("Unable get list of resources in module" + moduleReference.descriptor().name(), e); } } + /** + * This method extends {@link ModuleReader#list} with the ability to follow symlinks when + * the given {@link ModuleReader} corresponds to a + * {@code jdk.internal.module.ModuleReferences$ExplodedModuleReader} (i.e., + * {@code container} is a directory which contains the contents of the module). + * + * Classloaders (e.g., {@link NativeImageClassLoader#findResource(String, String)}) can load + * resources pointed to by a symlink present on the modulepath, meaning the extension below + * is necessary if we want to keep track of such resources (e.g., for digest computation in + * layered builds). + */ + @SuppressWarnings("resource") + private Stream moduleReaderListFollowSymlinks(ModuleReader reader, URI container) throws IOException { + if (explodedModuleReaderClass == null || !reader.getClass().isAssignableFrom(explodedModuleReaderClass)) { + return reader.list(); + } + + Path dir = Path.of(container); + // See {@code jdk.internal.module.ModuleReferences$ExplodedModuleReader#list}. + return Files.walk(dir, Integer.MAX_VALUE, FileVisitOption.FOLLOW_LINKS) + .map(f -> Resources.toResourceName(dir, f)) + .filter(Predicate.not(String::isEmpty)); + } + private void loadClassesFromPath(Path path) { final boolean includeUnconditionally = layerSelectors.classpathEntries().contains(path); final boolean includeAllMetadata = preserveSelectors.classpathEntries().contains(path); @@ -962,7 +1054,7 @@ private void loadClassesFromPath(Path path) { } if (probeJarFileSystem != null) { try (FileSystem jarFileSystem = probeJarFileSystem) { - loadClassesFromPath(container, jarFileSystem.getPath("/"), null, Collections.emptySet(), includeUnconditionally, includeAllMetadata); + loadClassesFromPath(container, jarFileSystem.getPath("/"), null, Collections.emptySet(), includeUnconditionally, includeAllMetadata, true); } } } catch (ClosedByInterruptException ignored) { @@ -973,13 +1065,13 @@ private void loadClassesFromPath(Path path) { } else { URI container = path.toUri(); loadClassesFromPath(container, path, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES_ROOT, ClassUtil.CLASS_MODULE_PATH_EXCLUDE_DIRECTORIES, includeUnconditionally, - includeAllMetadata); + includeAllMetadata, false); } } private static final String CLASS_EXTENSION = ".class"; - private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set excludes, boolean includeUnconditionally, boolean includeAllMetadata) { + private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set excludes, boolean includeUnconditionally, boolean includeAllMetadata, boolean isJar) { boolean useFilter = root.equals(excludeRoot); if (useFilter) { String excludesStr = excludes.stream().map(Path::toString).collect(Collectors.joining(", ")); @@ -987,6 +1079,7 @@ private void loadClassesFromPath(URI container, Path root, Path excludeRoot, Set } FileVisitor visitor = new SimpleFileVisitor<>() { private final char fileSystemSeparatorChar = root.getFileSystem().getSeparator().charAt(0); + private final boolean isInImageClassPathOfLayeredBuild = pathDigests != null && pathDigests.getCpDigests().containsKey(container); @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { @@ -1007,6 +1100,9 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) { currentlyProcessedEntry = file.toUri().toString(); executor.execute(() -> handleClassFileName(container, null, className, includeUnconditionally, true, includeAllMetadata)); } + if (isInImageClassPathOfLayeredBuild) { + executor.execute(() -> PathDigests.storePathFileDigest(container, fileName, isJar, pathDigests.cpDigests)); + } entriesProcessed.increment(); return FileVisitResult.CONTINUE; } @@ -1019,7 +1115,7 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) { }; try { - Files.walkFileTree(root, visitor); + Files.walkFileTree(root, EnumSet.of(FileVisitOption.FOLLOW_LINKS), Integer.MAX_VALUE, visitor); } catch (IOException ex) { throw VMError.shouldNotReachHere(ex); } @@ -1372,4 +1468,124 @@ public Set packages() { return packages.keySet(); } } + + /** + * Stores a temporary collection of individual class/resource file digests that is updated + * during class loading. In particular, {@code PathDigests} objects store and update two + * {@link EconomicMap}s, one for classpath entries and the other for modulepath entries. Each + * {@link EconomicMap} maps a path entry (i.e., directory/jar) with the list of individual + * digests for the files it contains. The order of the individual digests is non-deterministic. + * After class loading, the {@code PathDigests} should be aggregated into a list of + * {@link PathDigestEntry}. + */ + private static final class PathDigests { + private final EconomicMap> cpDigests = EconomicMap.create(); + private final EconomicMap> mpDigests = EconomicMap.create(); + + private PathDigests(List imagecp, List imagemp) { + imagecp.stream() + .map(Path::toUri) + .forEach(path -> cpDigests.put(path, new ArrayList<>())); + imagemp.stream() + .map(Path::toUri) + .forEach(path -> mpDigests.put(path, new ArrayList<>())); + } + + private static void storePathFileDigest(URI container, String resource, boolean isJar, EconomicMap> digests) { + byte[] fileContent; + try { + if (isJar) { + try (JarFile jarFile = new JarFile(new File(container), true, ZipFile.OPEN_READ, JarFile.runtimeVersion())) { + JarEntry jarEntry = jarFile.getJarEntry(resource); + fileContent = jarFile.getInputStream(jarEntry).readAllBytes(); + } + } else { + Path resourcePath = Path.of(container).resolve(resource); + if (!resourcePath.toFile().isFile()) { + return; + } + fileContent = Files.readAllBytes(resourcePath); + } + } catch (IOException e) { + throw UserError.abort("Image builder cannot read file: " + resource); + } + + DigestBuilder db = new DigestBuilder(); + db.update(fileContent); + db.update(resource.getBytes(StandardCharsets.UTF_8)); + List containerDigests = digests.get(container); + synchronized (containerDigests) { + containerDigests.add(new String(db.digest(), StandardCharsets.UTF_8)); + } + } + + private EconomicMap> getCpDigests() { + return cpDigests; + } + + private EconomicMap> getMpDigests() { + return mpDigests; + } + } + + /** + * The record type {@link PathDigestEntry} encodes tuples of the form (type, digest, path), + * where: - type: is either {@code PathType.cp} or {@code PathType.mp} - digest: is a checksum + * which encodes information about every class/resource file part of that path (or reachable + * from a symlink part of that path) - path: is the absolute path ponting to the directory/jar + * included on the class/module-path. To obtain a list of all {@code PathDigestEntry} objects + * corresponding to a particular Native Image build, use the method + * {@link NativeImageClassLoaderSupport#computePathEntryDigests()}. + */ + public record PathDigestEntry(PathType type, String digest, String path) { + + /** + * Aggregate the contents of {@link PathDigests} to create a list of + * {@link PathDigestEntry}. The individual file digests of a path entry are aggregated into + * a final digest that encodes information about all the contents of that path entry. + */ + private static List aggregatePathDigests(PathDigests pathDigests) { + Objects.requireNonNull(pathDigests, "NativeImageClassLoaderSupport#pathDigests should not be empty for a layered build."); + + List aggregatedDigests = new ArrayList<>(); + aggregatedDigests.addAll(aggregatePathDigests(pathDigests.getCpDigests(), PathType.cp)); + aggregatedDigests.addAll(aggregatePathDigests(pathDigests.getMpDigests(), PathType.mp)); + return aggregatedDigests; + } + + private static List aggregatePathDigests(EconomicMap> pathDigests, PathType type) { + List aggregatedDigests = new ArrayList<>(); + var cursor = pathDigests.getEntries(); + while (cursor.advance()) { + aggregatedDigests.add(PathDigestEntry.of(type, cursor.getKey().getPath(), cursor.getValue())); + } + return aggregatedDigests; + } + + private static PathDigestEntry of(PathType type, String path, List digests) { + DigestBuilder db = new DigestBuilder(); + digests.stream() + .sorted() + .map(d -> d.getBytes(StandardCharsets.UTF_8)) + .forEach(db::update); + + String aggregatedDigest = new String(db.digest(), StandardCharsets.UTF_8); + return new PathDigestEntry(type, aggregatedDigest, path); + } + + public static PathDigestEntry of(String digestEntry) { + String[] envVarArr = digestEntry.split(":", 3); + return new PathDigestEntry(PathType.valueOf(envVarArr[0]), envVarArr[1], envVarArr[2]); + } + + @Override + public String toString() { + return type + ":" + digest + ":" + path; + } + + public enum PathType { + cp, + mp + } + } } diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java index ba75884a0228..5cb8e5427a61 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/driver/LayerOptionsSupport.java @@ -35,7 +35,9 @@ public class LayerOptionsSupport extends IncludeOptionsSupport { public record LayerOption(Path fileName, ExtendedOption[] extendedOptions) { - /** Split a layer option into its components. */ + /** + * Split a layer option into its components. + */ public static LayerOption parse(String layerOptionValue) { VMError.guarantee(!layerOptionValue.isEmpty()); // Given an argument of form layer-file.nil,module=m1,package=p1 diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java index 31e45372ce8f..42b229703ace 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/HostedImageLayerBuildingSupport.java @@ -100,6 +100,15 @@ private static String layerCreatePossibleOptions() { * {@link #initialize}. */ private final Function, SingletonTrait[]> singletonTraitInjector; + /** + * Optional suboption of the {@link SubstrateOptions#LayerCreate} option. If the `LayerCreate` + * option is specified inside a `native-image.properties` file and this suboption is enabled, + * the classpath/modulepath entry containing the `native-image.properties` file will be excluded + * from the classpath/modulepath layered compatibility check. This suboption has no effect if + * it's specified from the command line. See + * {@link #processLayerOptions(EconomicMap, NativeImageClassLoaderSupport)} for more details. + */ + private static final String DIGEST_IGNORE = "digest-ignore"; private HostedImageLayerBuildingSupport(ImageClassLoader imageClassLoader, Reader snapshot, List graphsChannels, @@ -187,6 +196,7 @@ public Function, SingletonTrait[]> getSingletonTraitInjector() { */ public static void processLayerOptions(EconomicMap, Object> values, NativeImageClassLoaderSupport classLoaderSupport) { OptionValues hostedOptions = new OptionValues(values); + Path digestIgnorePath = null; if (isLayerCreateOptionEnabled(hostedOptions)) { ValueWithOrigin valueWithOrigin = getLayerCreateValueWithOrigin(hostedOptions); @@ -205,7 +215,12 @@ public static void processLayerOptions(EconomicMap, Object> values, classLoaderSupport.setLayerFile(layerFile); NativeImageClassLoaderSupport.IncludeSelectors layerSelectors = classLoaderSupport.getLayerSelectors(); + IncludeOptionsSupport.ExtendedOption digestIgnoreExtendedOption = new IncludeOptionsSupport.ExtendedOption(DIGEST_IGNORE, null); for (IncludeOptionsSupport.ExtendedOption option : layerOption.extendedOptions()) { + if (option.equals(digestIgnoreExtendedOption)) { + digestIgnorePath = valueWithOrigin.origin().location(); + continue; + } IncludeOptionsSupport.parseIncludeSelector(layerCreateArg, valueWithOrigin, layerSelectors, option, layerCreatePossibleOptions()); } @@ -238,6 +253,10 @@ public static void processLayerOptions(EconomicMap, Object> values, SubstrateOptions.ApplicationLayerInitializedClasses.update(values, Module.class.getName()); setOptionIfHasNotBeenSet(values, SubstrateOptions.ConcealedOptions.RelativeCodePointers, true); } + + if (isLayerCreateOptionEnabled(hostedOptions) || isLayerUseOptionEnabled(hostedOptions)) { + classLoaderSupport.initializePathDigests(digestIgnorePath); + } } private static void setOptionIfHasNotBeenSet(EconomicMap, Object> values, HostedOptionKey option, boolean boxedValue) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java index cb23f4560965..9461a104ea99 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LayerArchiveSupport.java @@ -30,7 +30,6 @@ import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -55,14 +54,13 @@ public class LayerArchiveSupport { private static final String ENV_VARIABLES_FILE_NAME = "env-variables.txt"; private static final String SNAPSHOT_FILE_NAME = "layer-snapshot.lsb"; private static final String SNAPSHOT_GRAPHS_FILE_NAME = "layer-snapshot-graphs.big"; + private static final String BUILD_PATH_DIGESTS_FILE_NAME = "build-path-digests.txt"; private static final String LAYER_INFO_MESSAGE_PREFIX = "Native Image Layers"; protected static final String LAYER_TEMP_DIR_PREFIX = "layerRoot_"; protected static final String SHARED_LIB_NAME_PREFIX = "lib"; public static final String LAYER_FILE_EXTENSION = ".nil"; - protected final List builderArguments; - protected final LayerProperties layerProperties; protected final Path layerFile; protected final ArchiveSupport archiveSupport; @@ -85,7 +83,6 @@ public LayerArchiveSupport(String layerName, Path layerFile, Path layerDir, Arch } this.layerProperties = new LayerArchiveSupport.LayerProperties(layerName); - this.builderArguments = new ArrayList<>(); } protected void validateLayerFile() { @@ -129,6 +126,10 @@ protected Path getEnvVariablesFilePath() { return layerDir.resolve(ENV_VARIABLES_FILE_NAME); } + protected Path getBuildPathDigestsFilePath() { + return layerDir.resolve(BUILD_PATH_DIGESTS_FILE_NAME); + } + protected List parseEnvVariables() { return System.getenv().entrySet().stream() .map(EnvironmentVariable::of) diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java index 855135c05c9b..ac2dc55c3876 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/LoadLayerArchiveSupport.java @@ -25,6 +25,7 @@ package com.oracle.svm.hosted.imagelayer; import static com.oracle.svm.core.util.EnvVariableUtils.EnvironmentVariable; +import static com.oracle.svm.hosted.NativeImageClassLoaderSupport.PathDigestEntry; import java.io.IOException; import java.nio.file.Files; @@ -64,24 +65,13 @@ public LoadLayerArchiveSupport(String layerName, Path layerFile, Path tempDir, A super(layerName, layerFile, tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "load"), archiveSupport); this.archiveSupport.expandJarToDir(layerFile, layerDir); layerProperties.loadAndVerify(current); - loadBuilderArgumentsFile(); } - private void loadBuilderArgumentsFile() { - try (Stream lines = Files.lines(getBuilderArgumentsFilePath())) { - lines.forEach(builderArguments::add); + private static List loadBuildEntries(Path path, Function stringParser, String buildEntryTypeDescription) { + try (Stream lines = Files.lines(path)) { + return lines.map(stringParser).toList(); } catch (IOException e) { - throw UserError.abort("Unable to load builder arguments from file " + getBuilderArgumentsFilePath()); - } - } - - private List loadEnvironmentVariablesFile() { - List envVariables = new ArrayList<>(); - try (Stream lines = Files.lines(getEnvVariablesFilePath())) { - lines.map(EnvironmentVariable::of).forEach(envVariables::add); - return envVariables; - } catch (IOException e) { - throw UserError.abort("Unable to load environment variables from file " + getEnvVariablesFilePath()); + throw UserError.abort("Unable to load " + buildEntryTypeDescription + " from file " + path); } } @@ -97,12 +87,19 @@ protected void validateLayerFile() { public void verifyCompatibility(NativeImageClassLoaderSupport classLoaderSupport, Map allRequests, boolean strict, boolean verbose) { Function filterFunction = argument -> splitArgumentOrigin(argument).argument; boolean violationsFound = false; - violationsFound |= verifyBuilderArgumentsCompatibility(builderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, true); - violationsFound |= verifyBuilderArgumentsCompatibility(builderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, false); - violationsFound |= verifyEnvironmentVariablesCompatibility(loadEnvironmentVariablesFile(), parseEnvVariables(), strict, verbose); + List previousBuilderArguments = loadBuildEntries(getBuilderArgumentsFilePath(), Function.identity(), "builder arguments"); + violationsFound |= verifyBuilderArgumentsCompatibility(previousBuilderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, true); + violationsFound |= verifyBuilderArgumentsCompatibility(previousBuilderArguments, classLoaderSupport.getHostedOptionParser().getArguments(), filterFunction, allRequests, strict, verbose, + false); if (violationsFound && verbose) { UserError.abort("Verbose LayerOptionVerification failed."); } + + List previousEnvironmentVariables = loadBuildEntries(getEnvVariablesFilePath(), EnvironmentVariable::of, "environment variables"); + verifyEnvironmentVariablesCompatibility(previousEnvironmentVariables, parseEnvVariables(), strict); + + List previousPathDigestEntries = loadBuildEntries(getBuildPathDigestsFilePath(), PathDigestEntry::of, "build path digests"); + verifyBuildPathDigestsCompatibility(previousPathDigestEntries, classLoaderSupport.computePathEntryDigests(), strict); } private static boolean verifyBuilderArgumentsCompatibility(List previousArgs, List currentArgs, Function filterFunction, @@ -230,29 +227,53 @@ private static boolean verifyBuilderArgumentsCompatibility(List previous * but are not considered by this verification, since mismatches between these don't affect * layer compatibility. */ - private static boolean verifyEnvironmentVariablesCompatibility(List previousEnvVars, List currentEnvVars, boolean strict, boolean verbose) { + private static void verifyEnvironmentVariablesCompatibility(List previousEnvVars, List currentEnvVars, boolean strict) { Set currentEnvVarsSet = new HashSet<>(currentEnvVars); boolean violationsFound = false; + String messagePrefix = strict ? "Error" : "Warning"; for (EnvironmentVariable previousEnvVar : previousEnvVars) { if (currentEnvVarsSet.contains(previousEnvVar)) { continue; } - violationsFound = true; String message = "Current layered image build must set environment variable " + previousEnvVar + " as it was set in the previous layer build."; - if (verbose) { - LogUtils.info("Error: ", message); - } else { - if (strict) { - throw UserError.abort(message); - } else { - LogUtils.warning(message); - } - } + LogUtils.info(messagePrefix, message); } - return violationsFound; + if (violationsFound && strict) { + throw UserError.abort("Environment variable layer compatibility check failed."); + } + } + + private static void verifyBuildPathDigestsCompatibility(List previousPathDigests, List currentPathDigests, boolean strict) { + Set currentDigests = new HashSet<>(currentPathDigests.size()); + currentPathDigests.forEach(pathEntry -> currentDigests.add(pathEntry.digest())); + + List previousUnmatchedPathDigests = previousPathDigests.stream() + .filter(pathEntry -> !currentDigests.contains(pathEntry.digest())) + .toList(); + if (previousUnmatchedPathDigests.isEmpty()) { + // All class/module-path entry digests from the previous layered build were present in + // the current layered build. + // Therefore, the compatibility check passes and we report no violations. + return; + } + + Set currentPathNames = new HashSet<>(currentPathDigests.size()); + currentPathDigests.forEach(pathEntry -> currentPathNames.add(pathEntry.path())); + String messagePrefix = strict ? "Error" : "Warning"; + for (PathDigestEntry unmatchedDigestEntry : previousUnmatchedPathDigests) { + boolean pathNotFound = !currentPathNames.contains(unmatchedDigestEntry.path()); + String pathType = unmatchedDigestEntry.type().equals(PathDigestEntry.PathType.cp) ? "classpath" : "modulepath"; + String message = "The entry " + unmatchedDigestEntry.path() + " included in the " + pathType + " of the previous layered build was" + + (pathNotFound ? " not" : "") + " found in the " + pathType + " of the current layered build" + (pathNotFound ? "." : ", but its contents have been modified."); + LogUtils.info(messagePrefix, message); + } + + if (strict) { + throw UserError.abort("Class/Module-path layer compatibility check failed."); + } } record ArgumentOrigin(boolean booleanOption, String argument, String origin) { diff --git a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java index 9a3d266464a8..9e52a800e85d 100644 --- a/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java +++ b/substratevm/src/com.oracle.svm.hosted/src/com/oracle/svm/hosted/imagelayer/WriteLayerArchiveSupport.java @@ -24,11 +24,13 @@ */ package com.oracle.svm.hosted.imagelayer; -import static com.oracle.svm.core.util.EnvVariableUtils.EnvironmentVariable; +import static com.oracle.svm.hosted.NativeImageClassLoaderSupport.PathDigestEntry; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; import java.util.jar.JarOutputStream; import org.graalvm.nativeimage.Platform; @@ -44,6 +46,8 @@ /* Builds an image layer, either initial or intermediate. */ public class WriteLayerArchiveSupport extends LayerArchiveSupport { + private final List builderArguments; + private final List buildPathDigests; public WriteLayerArchiveSupport(String layerName, NativeImageClassLoaderSupport classLoaderSupport, Path tempDir, ArchiveSupport archiveSupport) { super(layerName, classLoaderSupport.getLayerFile(), tempDir.resolve(LAYER_TEMP_DIR_PREFIX + "write"), archiveSupport); @@ -52,7 +56,8 @@ public WriteLayerArchiveSupport(String layerName, NativeImageClassLoaderSupport SubstrateOptionsParser.commandArgument(SubstrateOptions.Name, layerName) + "' needs to start with '" + SHARED_LIB_NAME_PREFIX + "'"); } - builderArguments.addAll(classLoaderSupport.getHostedOptionParser().getArguments()); + builderArguments = new ArrayList<>(classLoaderSupport.getHostedOptionParser().getArguments()); + buildPathDigests = new ArrayList<>(classLoaderSupport.computePathEntryDigests()); } @Override @@ -72,19 +77,11 @@ protected void validateLayerFile() { } } - private void writeBuilderArgumentsFile() { + private static void writeBuildEntries(Path path, List buildEntries, String buildEntryTypeDescription) { try { - Files.write(getBuilderArgumentsFilePath(), builderArguments); + Files.write(path, buildEntries.stream().map(T::toString).toList()); } catch (IOException e) { - throw UserError.abort("Unable to write builder arguments to file " + getBuilderArgumentsFilePath()); - } - } - - private void writeEnvVariablesFile() { - try { - Files.write(getEnvVariablesFilePath(), parseEnvVariables().stream().map(EnvironmentVariable::toString).toList()); - } catch (IOException e) { - throw UserError.abort("Unable to write environment variables to file " + getEnvVariablesFilePath()); + throw UserError.abort("Unable to write " + buildEntryTypeDescription + " to file " + path); } } @@ -93,11 +90,17 @@ public void write(Platform current) { // disable compression for significant (un)archiving speedup at the cost of file size jarOutStream.setLevel(0); // write builder arguments file and add to jar - writeBuilderArgumentsFile(); - archiveSupport.addFileToJar(layerDir, getBuilderArgumentsFilePath(), layerFile, jarOutStream); + Path builderArgumentsFilePath = getBuilderArgumentsFilePath(); + writeBuildEntries(builderArgumentsFilePath, builderArguments, "builder arguments"); + archiveSupport.addFileToJar(layerDir, builderArgumentsFilePath, layerFile, jarOutStream); // write environment variables file and add to jar - writeEnvVariablesFile(); - archiveSupport.addFileToJar(layerDir, getEnvVariablesFilePath(), layerFile, jarOutStream); + Path envVariablesFilePath = getEnvVariablesFilePath(); + writeBuildEntries(envVariablesFilePath, parseEnvVariables(), "environment variables"); + archiveSupport.addFileToJar(layerDir, envVariablesFilePath, layerFile, jarOutStream); + // write image class and module paths digests file and add to jar + Path buildPathDigestsFilePath = getBuildPathDigestsFilePath(); + writeBuildEntries(buildPathDigestsFilePath, buildPathDigests, "build path digests"); + archiveSupport.addFileToJar(layerDir, buildPathDigestsFilePath, layerFile, jarOutStream); // copy the layer snapshot file and its graphs file to the jar archiveSupport.addFileToJar(layerDir, getSnapshotPath(), layerFile, jarOutStream); archiveSupport.addFileToJar(layerDir, getSnapshotGraphsPath(), layerFile, jarOutStream);