8266814: Improve library loading with SymbolLookup abstraction

Reviewed-by: jvernee, sundar

make/modules/jdk.incubator.foreign/Lib.gmk

@@ -0,0 +1,59 @@
+#
+# Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# This code is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License version 2 only, as
+# published by the Free Software Foundation.  Oracle designates this
+# particular file as subject to the "Classpath" exception as provided
+# by Oracle in the LICENSE file that accompanied this code.
+#
+# This code is distributed in the hope that it will be useful, but WITHOUT
+# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+# version 2 for more details (a copy is included in the LICENSE file that
+# accompanied this code).
+#
+# You should have received a copy of the GNU General Public License version
+# 2 along with this work; if not, write to the Free Software Foundation,
+# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+#
+# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+# or visit www.oracle.com if you need additional information or have any
+# questions.
+#
+
+include LibCommon.gmk
+
+ifeq ($(call isTargetOs, linux), true)
+
+$(eval $(call SetupJdkLibrary, BUILD_LIBCSTDLIB, \
+    NAME := syslookup, \
+    OPTIMIZATION := HIGH, \
+    DISABLED_WARNINGS_gcc := sign-compare pointer-arith, \
+    DISABLED_WARNINGS_clang := sign-compare pointer-arith format-nonliteral, \
+    CFLAGS := $(CFLAGS_JDKLIB), \
+    CXXFLAGS := $(CXXFLAGS_JDKLIB), \
+    LDFLAGS := -Wl$(COMMA)--no-as-needed -lc -lm $(LDFLAGS_JDKLIB) $(call SET_SHARED_LIBRARY_ORIGIN), \
+    LIBS := $(LIBCXX), \
+))
+
+else ifeq ($(call isTargetOs, windows), false)
+
+$(eval $(call SetupJdkLibrary, BUILD_LIBCSTDLIB, \
+    NAME := syslookup, \
+    OPTIMIZATION := HIGH, \
+    DISABLED_WARNINGS_gcc := sign-compare pointer-arith, \
+    DISABLED_WARNINGS_clang := sign-compare pointer-arith format-nonliteral, \
+    CFLAGS := $(CFLAGS_JDKLIB), \
+    CXXFLAGS := $(CXXFLAGS_JDKLIB), \
+    LDFLAGS := $(LDFLAGS_JDKLIB) $(call SET_SHARED_LIBRARY_ORIGIN), \
+    LIBS := $(LIBCXX), \
+))
+
+
+endif
+
+TARGETS += $(BUILD_LIBCSTDLIB)
+
+################################################################################

src/jdk.incubator.foreign/share/classes/jdk/incubator/foreign/CLinker.java

@@ -29,6 +29,7 @@
 import jdk.internal.access.SharedSecrets;
 import jdk.internal.foreign.NativeMemorySegmentImpl;
 import jdk.internal.foreign.PlatformLayouts;
+import jdk.internal.foreign.SystemLookup;
 import jdk.internal.foreign.abi.SharedUtils;
 import jdk.internal.reflect.CallerSensitive;
 import jdk.internal.reflect.Reflection;
@@ -129,29 +130,20 @@ static CLinker getInstance() {
         return SharedUtils.getSystemLinker();
     }
 
-
     /**
-     * Finds the address of a symbol with given name in one of the native libraries associated with the caller's
-     * classloader (that is, libraries loaded using {@link System#loadLibrary} or {@link System#load}).
-     *
+     * Obtains a system lookup which is suitable to find symbols in the standard C libraries. The set of symbols
+     * available for lookup is unspecified, as it depends on the platform and on the operating system.
      * <p>
      * This method is <a href="package-summary.html#restricted"><em>restricted</em></a>.
      * Restricted method are unsafe, and, if used incorrectly, their use might crash
      * the JVM or, worse, silently result in memory corruption. Thus, clients should refrain from depending on
      * restricted methods, and use safe and supported functionalities, where possible.
-     *
-     * @param name the name of the symbol to be searched.
-     * @return the address of a symbol with given name in one of the native libraries associated with the caller's
-     *         classloader (if any).
+     * @return a system-specific library lookup which is suitable to find symbols in the standard C libraries.
      */
     @CallerSensitive
-    public static Optional<MemoryAddress> findNative(String name) {
-         Reflection.ensureNativeAccess(Reflection.getCallerClass());
-         ClassLoader loader = Reflection.getCallerClass().getClassLoader();
-         Objects.requireNonNull(name);
-         JavaLangAccess javaLangAccess = SharedSecrets.getJavaLangAccess();
-         MemoryAddress addr = MemoryAddress.ofLong(javaLangAccess.findNative(loader, name));
-         return addr == MemoryAddress.NULL? Optional.empty() : Optional.of(addr);
+    static SymbolLookup systemLookup() {
+        Reflection.ensureNativeAccess(Reflection.getCallerClass());
+        return SystemLookup.getInstance();
     }
 
     /**
@@ -167,7 +159,7 @@ public static Optional<MemoryAddress> findNative(String name) {
      * the JVM or, worse, silently result in memory corruption. Thus, clients should refrain from depending on
      * restricted methods, and use safe and supported functionalities, where possible.
      *
-     * @see CLinker#findNative(String)
+     * @see SymbolLookup
      *
      * @param symbol   downcall symbol.
      * @param type     the method type.
@@ -189,7 +181,7 @@ public static Optional<MemoryAddress> findNative(String name) {
      * the JVM or, worse, silently result in memory corruption. Thus, clients should refrain from depending on
      * restricted methods, and use safe and supported functionalities, where possible.
      *
-     * @see CLinker#findNative(String)
+     * @see SymbolLookup
      *
      * @param symbol    downcall symbol.
      * @param allocator the segment allocator.
@@ -219,7 +211,7 @@ public static Optional<MemoryAddress> findNative(String name) {
      * the JVM or, worse, silently result in memory corruption. Thus, clients should refrain from depending on
      * restricted methods, and use safe and supported functionalities, where possible.
      **
-     * @see CLinker#findNative(String)
+     * @see SymbolLookup
      *
      * @param type     the method type.
      * @param function the function descriptor.

src/jdk.incubator.foreign/share/classes/jdk/incubator/foreign/SymbolLookup.java

@@ -0,0 +1,77 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package jdk.incubator.foreign;
+
+import jdk.internal.access.JavaLangAccess;
+import jdk.internal.access.SharedSecrets;
+import jdk.internal.reflect.CallerSensitive;
+import jdk.internal.reflect.Reflection;
+
+import java.util.Objects;
+import java.util.Optional;
+
+/**
+ * A symbol lookup. Exposes a lookup operation for searching symbols, see {@link SymbolLookup#lookup(String)}.
+ * <p> Unless otherwise specified, passing a {@code null} argument, or an array argument containing one or more {@code null}
+ * elements to a method in this class causes a {@link NullPointerException NullPointerException} to be thrown. </p>
+ */
+@FunctionalInterface
+public interface SymbolLookup {
+
+    /**
+     * Looks up a symbol with given name in this lookup.
+     *
+     * @param name the symbol name.
+     * @return the memory address associated with the symbol (if any).
+     */
+    Optional<MemoryAddress> lookup(String name);
+
+    /**
+     * Obtains a symbol lookup suitable to find symbols in native libraries associated with the caller's classloader
+     * (that is, libraries loaded using {@link System#loadLibrary} or {@link System#load}).
+     * <p>
+     * This method is <a href="package-summary.html#restricted"><em>restricted</em></a>.
+     * Restricted method are unsafe, and, if used incorrectly, their use might crash
+     * the JVM or, worse, silently result in memory corruption. Thus, clients should refrain from depending on
+     * restricted methods, and use safe and supported functionalities, where possible.
+     *
+     * @return a symbol lookup suitable to find symbols in libraries loaded by the caller's classloader.
+     * @throws IllegalCallerException if access to this method occurs from a module {@code M} and the command line option
+     * {@code --enable-native-access} is either absent, or does not mention the module name {@code M}, or
+     * {@code ALL-UNNAMED} in case {@code M} is an unnamed module.
+     */
+    @CallerSensitive
+    static SymbolLookup loaderLookup() {
+        Class<?> caller = Reflection.getCallerClass();
+        Reflection.ensureNativeAccess(caller);
+        ClassLoader loader = Objects.requireNonNull(caller.getClassLoader());
+        return name -> {
+            Objects.requireNonNull(name);
+            JavaLangAccess javaLangAccess = SharedSecrets.getJavaLangAccess();
+            MemoryAddress addr = MemoryAddress.ofLong(javaLangAccess.findNative(loader, name));
+            return addr == MemoryAddress.NULL? Optional.empty() : Optional.of(addr);
+        };
+    }
+}

src/jdk.incubator.foreign/share/classes/jdk/incubator/foreign/package-info.java

@@ -96,8 +96,9 @@
  * operation either succeeds - and accesses a valid memory location - or fails.
  *
  * <h2>Foreign function access</h2>
- * The key abstractions introduced to support foreign function access is {@link jdk.incubator.foreign.CLinker}.
- * CLinker provides linking capabilities which allow to model foreign functions as {@link java.lang.invoke.MethodHandle} instances,
+ * The key abstractions introduced to support foreign function access are {@link jdk.incubator.foreign.SymbolLookup} and {@link jdk.incubator.foreign.CLinker}.
+ * The former is used to lookup symbols inside native libraries; the latter
+ * provides linking capabilities which allow to model foreign functions as {@link java.lang.invoke.MethodHandle} instances,
  * so that clients can perform foreign function calls directly in Java, without the need for intermediate layers of native
  * code (as it's the case with the <a href="{@docRoot}/../specs/jni/index.html">Java Native Interface (JNI)</a>).
  * <p>
@@ -106,7 +107,7 @@
  *
  * <pre>{@code
       MethodHandle strlen = CLinker.getInstance().downcallHandle(
-        CLinker.findNative("strlen").get(),
+        CLinker.systemLookup().lookup("strlen").get(),
         MethodType.methodType(long.class, MemoryAddress.class),
         FunctionDescriptor.of(CLinker.C_LONG, CLinker.C_POINTER)
       );
@@ -117,9 +118,7 @@
       }
  * }</pre>
  *
- * Here, we lookup the {@code strlen} symbol; this assumes that a library containing the {@code strlen} symbol has been loaded
- * earlier, e.g. using {@link System#loadLibrary(String)}.
- *
+ * Here, we lookup the {@code strlen} symbol in the {@linkplain jdk.incubator.foreign.CLinker#systemLookup() system lookup}.
  * Then, we obtain a linker instance (see {@link jdk.incubator.foreign.CLinker#getInstance()}) and we use it to
  * obtain a method handle which targets the {@code strlen} library symbol. To complete the linking successfully,
  * we must provide (i) a {@link java.lang.invoke.MethodType} instance, describing the type of the resulting method handle

src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/SystemLookup.java

@@ -0,0 +1,64 @@
+/*
+ * Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.  Oracle designates this
+ * particular file as subject to the "Classpath" exception as provided
+ * by Oracle in the LICENSE file that accompanied this code.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package jdk.internal.foreign;
+
+import jdk.incubator.foreign.SymbolLookup;
+import jdk.incubator.foreign.MemoryAddress;
+import jdk.internal.loader.NativeLibraries;
+import jdk.internal.loader.NativeLibrary;
+
+import java.nio.file.Path;
+import java.util.Objects;
+import java.util.Optional;
+
+public class SystemLookup implements SymbolLookup {
+
+    private SystemLookup() { }
+
+    final static SystemLookup INSTANCE = new SystemLookup();
+
+    /*
+     * On POSIX systems, dlsym will allow us to lookup symbol in library dependencies; the same trick doesn't work
+     * on Windows. For this reason, on Windows we do not generate any side-library, and load msvcrt.dll directly instead.
+     */
+    final NativeLibrary syslookup = switch (CABI.current()) {
+        case SysV, AArch64 -> NativeLibraries.rawNativeLibraries(SystemLookup.class, false).loadLibrary("syslookup");
+        case Win64 -> NativeLibraries.rawNativeLibraries(SystemLookup.class, false)
+                .loadLibrary(null, Path.of(System.getenv("SystemRoot"), "System32", "msvcrt.dll").toFile());
+    };
+
+    @Override
+    public Optional<MemoryAddress> lookup(String name) {
+        Objects.requireNonNull(name);
+        long addr = syslookup.find(name);
+        return addr == 0 ?
+                Optional.empty() : Optional.of(MemoryAddress.ofLong(addr));
+    }
+
+    public static SystemLookup getInstance() {
+        return INSTANCE;
+    }
+}

src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/abi/SharedUtils.java

@@ -412,6 +412,20 @@ static MethodHandle wrapWithAllocator(MethodHandle specializedHandle,
         return specializedHandle;
     }
 
+    // lazy init MH_ALLOC and MH_FREE handles
+    private static class AllocHolder {
+
+        private static final CLinker linker = getSystemLinker();
+
+        static final MethodHandle MH_MALLOC = linker.downcallHandle(CLinker.systemLookup().lookup("malloc").get(),
+                        MethodType.methodType(MemoryAddress.class, long.class),
+                FunctionDescriptor.of(C_POINTER, C_LONG_LONG));
+
+        static final MethodHandle MH_FREE = linker.downcallHandle(CLinker.systemLookup().lookup("free").get(),
+                        MethodType.methodType(void.class, MemoryAddress.class),
+                FunctionDescriptor.ofVoid(C_POINTER));
+    }
+
     public static MemoryAddress checkSymbol(Addressable symbol) {
         Objects.requireNonNull(symbol);
         MemoryAddress symbolAddr = symbol.address();
@@ -422,15 +436,15 @@ public static MemoryAddress checkSymbol(Addressable symbol) {
 
     public static MemoryAddress allocateMemoryInternal(long size) {
         try {
-            return (MemoryAddress) VMFunctions.MH_MALLOC.invokeExact(size);
+            return (MemoryAddress) AllocHolder.MH_MALLOC.invokeExact(size);
         } catch (Throwable th) {
             throw new RuntimeException(th);
         }
     }
 
     public static void freeMemoryInternal(MemoryAddress addr) {
         try {
-            VMFunctions.MH_FREE.invokeExact(addr);
+            AllocHolder.MH_FREE.invokeExact(addr);
         } catch (Throwable th) {
             throw new RuntimeException(th);
         }