8281089: JavaFX built with VS2019 and jlinked into JDK 11.x fails to start

Reviewed-by: arapte, sykora

Author: kevinrushforth

build.gradle

@@ -5343,13 +5343,6 @@ compileTargets { t ->
         def standaloneSdkDir = "${rootProject.buildDir}/${standaloneSdkDirName}"
         def standaloneLegalDir = "${standaloneSdkDir}/legal"
 
-        def excludeNativeLibs = []
-        if (IS_WINDOWS) {
-            // List of duplicate Microsoft DLLs to exclude
-            excludeNativeLibs += targetProperties.VS2017DLLNames
-            excludeNativeLibs += targetProperties.WinSDKDLLNames
-        }
-
         moduleProjList.each { project ->
             def moduleName = project.ext.moduleName
             def buildDir = project.buildDir
@@ -5358,9 +5351,36 @@ compileTargets { t ->
             def srcLibDir = "${buildDir}/${platformPrefix}module-lib"
             def srcLegalDir = "${standaloneLegalDir}/${moduleName}"
 
+            def jmodLibDir = srcLibDir
+            if (IS_WINDOWS) {
+                jmodLibDir = "${srcLibDir}-jmod"
+            }
+
             def jmodName = "${moduleName}.jmod"
             def jmodFile = "${jmodsDir}/${jmodName}"
-            def jmodTask = project.task("jmod$t.capital", group: "Build", dependsOn: sdk) {
+
+            // On Windows, copy the native libraries in the jmod image
+            // to a "javafx" subdir to avoid conflicting with the Microsoft
+            // DLLs that are shipped with the JDK
+            def jmodCopyLibTask = project.task("jmodCopyLib$t.capital", type: Copy, dependsOn: sdk) {
+                enabled = IS_WINDOWS
+
+                group = "Basic"
+                description = "copied Windows DLLs into javafx subdir for jmods"
+
+                into jmodLibDir
+
+                from (srcLibDir) {
+                    exclude("*.dll")
+                }
+
+                from (srcLibDir) {
+                    include("*.dll")
+                    into("javafx")
+                }
+            }
+
+            def jmodTask = project.task("jmod$t.capital", group: "Build", dependsOn: [sdk, jmodCopyLibTask]) {
                 doLast {
                     mkdir jmodsDir
                     delete(jmodFile);
@@ -5371,16 +5391,9 @@ compileTargets { t ->
                         args(srcClassesDir)
                         args("--module-version", "$RELEASE_VERSION_SHORT")
                         // Not all modules have a "lib" dir
-                        if (file(srcLibDir).isDirectory()) {
+                        if (file(jmodLibDir).isDirectory()) {
                             args("--libs")
-                            args(srcLibDir)
-                        }
-                        // Exclude duplicate native libs from javafx.graphics.jmod
-                        if (moduleName == "javafx.graphics") {
-                            excludeNativeLibs.each { name ->
-                                args("--exclude")
-                                args(name)
-                            }
+                            args(jmodLibDir)
                         }
                         args("--legal-notices")
                         args(srcLegalDir)

modules/javafx.graphics/src/main/java/com/sun/glass/utils/NativeLibLoader.java

@@ -69,7 +69,6 @@ public static synchronized void loadLibrary(String libname, List<String> depende
 
     private static boolean verbose = false;
 
-    private static boolean usingModules = false;
     private static File libDir = null;
     private static String libPrefix = "";
     private static String libSuffix = "";
@@ -116,8 +115,9 @@ private static String[] initializePath(String propname) {
 
     private static void loadLibraryInternal(String libraryName, List<String> dependencies, Class caller) {
         // The search order for native library loading is:
-        // - try to load the native library from the same folder as this jar
-        //   (only on non-modular builds)
+        // - try to load the native library from either ${java.home}
+        //   (for jlinked javafx modules) or from the same folder as
+        //   this jar (if using modular jars)
         // - if the native library comes bundled as a resource it is extracted
         //   and loaded
         // - the java.library.path is searched for the library in definition
@@ -130,7 +130,7 @@ private static void loadLibraryInternal(String libraryName, List<String> depende
             // since it isn't applicable to Jigsaw.
             loadLibraryFullPath(libraryName);
         } catch (UnsatisfiedLinkError ex) {
-            if (verbose && !usingModules) {
+            if (verbose) {
                 System.err.println("WARNING: " + ex);
             }
 
@@ -326,51 +326,73 @@ static byte[] calculateCheckSum(File file) {
     }
 
 
+    private static File libDirForJRT() {
+        String javaHome = System.getProperty("java.home");
+
+        if (javaHome == null || javaHome.isEmpty()) {
+            throw new UnsatisfiedLinkError("Cannot find java.home");
+        }
+
+        // Set the native directory based on the OS
+        String osName = System.getProperty("os.name");
+        String relativeDir = null;
+        if (osName.startsWith("Windows")) {
+            relativeDir = "bin/javafx";
+        } else if (osName.startsWith("Mac")) {
+            relativeDir = "lib";
+        } else if (osName.startsWith("Linux")) {
+            relativeDir = "lib";
+        }
+
+        // Location of native libraries relative to java.home
+        return new File(javaHome + "/" + relativeDir);
+    }
+
+    private static File libDirForJarFile(String classUrlString) throws Exception {
+        // Strip out the "jar:" and everything after and including the "!"
+        String tmpStr = classUrlString.substring(4, classUrlString.lastIndexOf('!'));
+        // Strip everything after the last "/" or "\" to get rid of the jar filename
+        int lastIndexOfSlash = Math.max(tmpStr.lastIndexOf('/'), tmpStr.lastIndexOf('\\'));
+
+        // Set the native directory based on the OS
+        String osName = System.getProperty("os.name");
+        String relativeDir = null;
+        if (osName.startsWith("Windows")) {
+            relativeDir = "../bin";
+        } else if (osName.startsWith("Mac")) {
+            relativeDir = ".";
+        } else if (osName.startsWith("Linux")) {
+            relativeDir = ".";
+        }
+
+        // Location of native libraries relative to jar file
+        String libDirUrlString = tmpStr.substring(0, lastIndexOfSlash)
+                + "/" + relativeDir;
+        return new File(new URI(libDirUrlString).getPath());
+    }
+
     /**
-     * Load the native library from the same directory as the jar file
-     * containing this class.
+     * Load the native library either from the same directory as the jar file
+     * containing this class, or from the Java runtime.
      */
     private static void loadLibraryFullPath(String libraryName) {
         try {
-            if (usingModules) {
-                throw new UnsatisfiedLinkError("ignored");
-            }
             if (libDir == null) {
                 // Get the URL for this class, if it is a jar URL, then get the
                 // filename associated with it.
                 String theClassFile = "NativeLibLoader.class";
                 Class theClass = NativeLibLoader.class;
                 String classUrlString = theClass.getResource(theClassFile).toString();
                 if (classUrlString.startsWith("jrt:")) {
-                    // Suppress warning messages
-                    usingModules = true;
-                    throw new UnsatisfiedLinkError("ignored");
-                }
-                if (!classUrlString.startsWith("jar:file:") || classUrlString.indexOf('!') == -1) {
+                    libDir = libDirForJRT();
+                } else if (classUrlString.startsWith("jar:file:") && classUrlString.indexOf('!') > 0) {
+                    libDir = libDirForJarFile(classUrlString);
+                } else {
                     throw new UnsatisfiedLinkError("Invalid URL for class: " + classUrlString);
                 }
-                // Strip out the "jar:" and everything after and including the "!"
-                String tmpStr = classUrlString.substring(4, classUrlString.lastIndexOf('!'));
-                // Strip everything after the last "/" or "\" to get rid of the jar filename
-                int lastIndexOfSlash = Math.max(tmpStr.lastIndexOf('/'), tmpStr.lastIndexOf('\\'));
-
-                // Set the native directory based on the OS
-                String osName = System.getProperty("os.name");
-                String relativeDir = null;
-                if (osName.startsWith("Windows")) {
-                    relativeDir = "../bin";
-                } else if (osName.startsWith("Mac")) {
-                    relativeDir = ".";
-                } else if (osName.startsWith("Linux")) {
-                    relativeDir = ".";
-                }
-
-                // Location of native libraries relative to jar file
-                String libDirUrlString = tmpStr.substring(0, lastIndexOfSlash)
-                        + "/" + relativeDir;
-                libDir = new File(new URI(libDirUrlString).getPath());
 
                 // Set the lib prefix and suffix based on the OS
+                String osName = System.getProperty("os.name");
                 if (osName.startsWith("Windows")) {
                     libPrefix = "";
                     libSuffix = ".dll";