8267989: Exceptions thrown during upcalls should be handled

src/java.base/share/classes/java/lang/System.java

@@ -2352,6 +2352,11 @@ public boolean isEnableNativeAccess(Module m) {
             public long findNative(ClassLoader loader, String entry) {
                 return ClassLoader.findNative(loader, entry);
             }
+
+            @Override
+            public void exit(int statusCode) {
+                Shutdown.exit(statusCode);
+            }
         });
     }
 }

src/java.base/share/classes/java/lang/invoke/MethodHandleImpl.java

@@ -1488,6 +1488,11 @@ public VarHandle collectCoordinates(VarHandle target, int pos, MethodHandle filt
             public VarHandle insertCoordinates(VarHandle target, int pos, Object... values) {
                 return VarHandles.insertCoordinates(target, pos, values);
             }
+
+            @Override
+            public Class<?>[] exceptionTypes(MethodHandle handle) {
+                return VarHandles.exceptionTypes(handle);
+            }
         });
     }
 
@@ -1959,15 +1964,16 @@ private static LambdaForm makeCollectorForm(MethodType basicType, Class<?> array
 
     // Indexes into constant method handles:
     static final int
-            MH_cast                  = 0,
-            MH_selectAlternative     = 1,
-            MH_countedLoopPred       = 2,
-            MH_countedLoopStep       = 3,
-            MH_initIterator          = 4,
-            MH_iteratePred           = 5,
-            MH_iterateNext           = 6,
-            MH_Array_newInstance     = 7,
-            MH_LIMIT                 = 8;
+            MH_cast                               = 0,
+            MH_selectAlternative                  = 1,
+            MH_countedLoopPred                    = 2,
+            MH_countedLoopStep                    = 3,
+            MH_initIterator                       = 4,
+            MH_iteratePred                        = 5,
+            MH_iterateNext                        = 6,
+            MH_Array_newInstance                  = 7,
+            MH_VarHandles_handleCheckedExceptions = 8,
+            MH_LIMIT                              = 9;
 
     static MethodHandle getConstantHandle(int idx) {
         MethodHandle handle = HANDLES[idx];
@@ -2017,6 +2023,9 @@ private static MethodHandle makeConstantHandle(int idx) {
                 case MH_Array_newInstance:
                     return IMPL_LOOKUP.findStatic(Array.class, "newInstance",
                             MethodType.methodType(Object.class, Class.class, int.class));
+                case MH_VarHandles_handleCheckedExceptions:
+                    return IMPL_LOOKUP.findStatic(VarHandles.class, "handleCheckedExceptions",
+                            MethodType.methodType(void.class, Throwable.class));
             }
         } catch (ReflectiveOperationException ex) {
             throw newInternalError(ex);

src/java.base/share/classes/java/lang/invoke/VarHandles.java

@@ -31,12 +31,9 @@
 import java.lang.reflect.Field;
 import java.lang.reflect.Method;
 import java.lang.reflect.Modifier;
-import java.lang.reflect.Parameter;
 import java.nio.ByteOrder;
 import java.util.ArrayList;
-import java.util.LinkedHashMap;
 import java.util.List;
-import java.util.Map;
 import java.util.Objects;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentMap;
@@ -45,8 +42,6 @@
 import static java.lang.invoke.MethodHandleStatics.UNSAFE;
 import static java.lang.invoke.MethodHandleStatics.VAR_HANDLE_IDENTITY_ADAPT;
 import static java.lang.invoke.MethodHandleStatics.newIllegalArgumentException;
-import static java.util.stream.Collectors.joining;
-import static java.util.stream.Collectors.toList;
 
 final class VarHandles {
 
@@ -359,13 +354,13 @@ private static VarHandle maybeAdapt(VarHandle target) {
         return target;
     }
 
-    public static VarHandle filterValue(VarHandle target, MethodHandle filterToTarget, MethodHandle filterFromTarget) {
+    public static VarHandle filterValue(VarHandle target, MethodHandle pFilterToTarget, MethodHandle pFilterFromTarget) {
         Objects.requireNonNull(target);
-        Objects.requireNonNull(filterToTarget);
-        Objects.requireNonNull(filterFromTarget);
+        Objects.requireNonNull(pFilterToTarget);
+        Objects.requireNonNull(pFilterFromTarget);
         //check that from/to filters do not throw checked exceptions
-        noCheckedExceptions(filterToTarget);
-        noCheckedExceptions(filterFromTarget);
+        MethodHandle filterToTarget = adaptForCheckedExceptions(pFilterToTarget);
+        MethodHandle filterFromTarget = adaptForCheckedExceptions(pFilterFromTarget);
 
         List<Class<?>> newCoordinates = new ArrayList<>();
         List<Class<?>> additionalCoordinates = new ArrayList<>();
@@ -473,8 +468,9 @@ public static VarHandle filterCoordinates(VarHandle target, int pos, MethodHandl
 
         List<Class<?>> newCoordinates = new ArrayList<>(targetCoordinates);
         for (int i = 0 ; i < filters.length ; i++) {
-            noCheckedExceptions(filters[i]);
-            MethodType filterType = filters[i].type();
+            MethodHandle filter = Objects.requireNonNull(filters[i]);
+            filter = adaptForCheckedExceptions(filter);
+            MethodType filterType = filter.type();
             if (filterType.parameterCount() != 1) {
                 throw newIllegalArgumentException("Invalid filter type " + filterType);
             } else if (newCoordinates.get(pos + i) != filterType.returnType()) {
@@ -564,10 +560,10 @@ private static MethodType methodTypeFor(VarHandle.AccessType at, MethodType oldT
         return adjustedType;
     }
 
-    public static VarHandle collectCoordinates(VarHandle target, int pos, MethodHandle filter) {
+    public static VarHandle collectCoordinates(VarHandle target, int pos, MethodHandle pFilter) {
         Objects.requireNonNull(target);
-        Objects.requireNonNull(filter);
-        noCheckedExceptions(filter);
+        Objects.requireNonNull(pFilter);
+        MethodHandle filter = adaptForCheckedExceptions(pFilter);
 
         List<Class<?>> targetCoordinates = target.coordinateTypes();
         if (pos < 0 || pos >= targetCoordinates.size()) {
@@ -604,42 +600,55 @@ public static VarHandle dropCoordinates(VarHandle target, int pos, Class<?>... v
                 (mode, modeHandle) -> MethodHandles.dropArguments(modeHandle, 1 + pos, valueTypes));
     }
 
-    private static void noCheckedExceptions(MethodHandle handle) {
+    private static MethodHandle adaptForCheckedExceptions(MethodHandle target) {
+        Class<?>[] exceptionTypes = exceptionTypes(target);
+        if (exceptionTypes != null) { // exceptions known
+            if (Stream.of(exceptionTypes).anyMatch(VarHandles::isCheckedException)) {
+                throw newIllegalArgumentException("Cannot adapt a var handle with a method handle which throws checked exceptions");
+            }
+            return target; // no adaptation needed
+        } else {
+            MethodHandle handler = MethodHandleImpl.getConstantHandle(MethodHandleImpl.MH_VarHandles_handleCheckedExceptions);
+            MethodHandle zero = MethodHandles.zero(target.type().returnType()); // dead branch
+            handler = MethodHandles.collectArguments(zero, 0, handler);
+            return MethodHandles.catchException(target, Throwable.class, handler);
+        }
+    }
+
+    static void handleCheckedExceptions(Throwable throwable) throws Throwable {
+        if (isCheckedException(throwable.getClass())) {
+            throw new IllegalStateException("Adapter handle threw checked exception", throwable);
+        }
+        throw throwable;
+    }
+
+    static Class<?>[] exceptionTypes(MethodHandle handle) {
         if (handle instanceof DirectMethodHandle directHandle) {
             byte refKind = directHandle.member.getReferenceKind();
             MethodHandleInfo info = new InfoFromMemberName(
                     MethodHandles.Lookup.IMPL_LOOKUP,
                     directHandle.member,
                     refKind);
-            final Class<?>[] exceptionTypes;
             if (MethodHandleNatives.refKindIsMethod(refKind)) {
-                exceptionTypes = info.reflectAs(Method.class, MethodHandles.Lookup.IMPL_LOOKUP)
+                return info.reflectAs(Method.class, MethodHandles.Lookup.IMPL_LOOKUP)
                         .getExceptionTypes();
             } else if (MethodHandleNatives.refKindIsField(refKind)) {
-                exceptionTypes = null;
+                return new Class<?>[0];
             } else if (MethodHandleNatives.refKindIsConstructor(refKind)) {
-                exceptionTypes = info.reflectAs(Constructor.class, MethodHandles.Lookup.IMPL_LOOKUP)
+                return info.reflectAs(Constructor.class, MethodHandles.Lookup.IMPL_LOOKUP)
                         .getExceptionTypes();
             } else {
                 throw new AssertionError("Cannot get here");
             }
-            if (exceptionTypes != null) {
-                if (Stream.of(exceptionTypes).anyMatch(VarHandles::isCheckedException)) {
-                    throw newIllegalArgumentException("Cannot adapt a var handle with a method handle which throws checked exceptions");
-                }
-            }
         } else if (handle instanceof DelegatingMethodHandle) {
-            noCheckedExceptions(((DelegatingMethodHandle)handle).getTarget());
-        } else {
-            //bound
-            BoundMethodHandle boundHandle = (BoundMethodHandle)handle;
-            for (int i = 0 ; i < boundHandle.fieldCount() ; i++) {
-                Object arg = boundHandle.arg(i);
-                if (arg instanceof MethodHandle){
-                    noCheckedExceptions((MethodHandle) arg);
-                }
-            }
+            return exceptionTypes(((DelegatingMethodHandle)handle).getTarget());
+        } else if (handle instanceof NativeMethodHandle) {
+            return new Class<?>[0];
         }
+
+        assert handle instanceof BoundMethodHandle : "Unexpected handle type: " + handle;
+        // unknown
+        return null;
     }
 
     private static boolean isCheckedException(Class<?> clazz) {

src/java.base/share/classes/jdk/internal/access/JavaLangAccess.java

@@ -384,4 +384,10 @@ public interface JavaLangAccess {
     boolean isEnableNativeAccess(Module m);
 
     long findNative(ClassLoader loader, String entry);
+
+    /**
+     * Direct access to Shutdown.exit to avoid security manager checks
+     * @param statusCode the status code
+     */
+    void exit(int statusCode);
 }

src/java.base/share/classes/jdk/internal/access/JavaLangInvokeAccess.java

@@ -139,4 +139,11 @@ VarHandle memoryAccessVarHandle(Class<?> carrier, boolean skipAlignmentMaskCheck
      * @param mh the method handle
      */
     void ensureCustomized(MethodHandle mh);
+
+    /**
+     * A best-effort method that tries to find any exceptions thrown by the given method handle.
+     * @param handle the handle to check
+     * @return an array of exceptions, or {@code null}.
+     */
+    Class<?>[] exceptionTypes(MethodHandle handle);
 }

src/jdk.incubator.foreign/share/classes/jdk/incubator/foreign/CLinker.java

@@ -25,8 +25,6 @@
  */
 package jdk.incubator.foreign;
 
-import jdk.internal.access.JavaLangAccess;
-import jdk.internal.access.SharedSecrets;
 import jdk.internal.foreign.NativeMemorySegmentImpl;
 import jdk.internal.foreign.PlatformLayouts;
 import jdk.internal.foreign.SystemLookup;
@@ -39,7 +37,6 @@
 import java.lang.invoke.MethodType;
 import java.nio.charset.Charset;
 import java.util.Objects;
-import java.util.Optional;
 import java.util.function.Consumer;
 
 import static jdk.internal.foreign.PlatformLayouts.*;
@@ -224,9 +221,13 @@ static SymbolLookup systemLookup() {
      * Allocates a native stub with given scope which can be passed to other foreign functions (as a function pointer);
      * calling such a function pointer from native code will result in the execution of the provided method handle.
      *
-     * <p>The returned memory address is associated with the provided scope. When such scope is closed,
+     * <p>
+     * The returned memory address is associated with the provided scope. When such scope is closed,
      * the corresponding native stub will be deallocated.
      * <p>
+     * The target method handle should not throw any exceptions. If the target method handle does throw an exception,
+     * the VM will exit with a non-zero exit code.
+     * <p>
      * This method is <a href="package-summary.html#restricted"><em>restricted</em></a>.
      * Restricted method are unsafe, and, if used incorrectly, their use might crash
      * the JVM or, worse, silently result in memory corruption. Thus, clients should refrain from depending on
@@ -236,7 +237,8 @@ static SymbolLookup systemLookup() {
      * @param function the function descriptor.
      * @param scope the upcall stub scope.
      * @return the native stub segment.
-     * @throws IllegalArgumentException if the target's method type and the function descriptor mismatch.
+     * @throws IllegalArgumentException if the target's method type and the function descriptor mismatch, or
+     *         if it is determined that the target method handle can throw an exception.
      */
     MemoryAddress upcallStub(MethodHandle target, FunctionDescriptor function, ResourceScope scope);
 

src/jdk.incubator.foreign/share/classes/jdk/incubator/foreign/MemoryHandles.java

@@ -329,6 +329,9 @@ public static VarHandle asUnsigned(VarHandle target, final Class<?> adaptedType)
      * the resulting var handle will have type {@code S} and will feature the additional coordinates {@code A...} (which
      * will be appended to the coordinates of the target var handle).
      * <p>
+     * If the boxing and unboxing filters throw any checked exceptions when invoked, the resulting var handle will
+     * throw an {@link IllegalStateException}.
+     * <p>
      * The resulting var handle will feature the same access modes (see {@link java.lang.invoke.VarHandle.AccessMode} and
      * atomic access guarantees as those featured by the target var handle.
      *
@@ -338,7 +341,7 @@ public static VarHandle asUnsigned(VarHandle target, final Class<?> adaptedType)
      * @return an adapter var handle which accepts a new type, performing the provided boxing/unboxing conversions.
      * @throws IllegalArgumentException if {@code filterFromTarget} and {@code filterToTarget} are not well-formed, that is, they have types
      * other than {@code (A... , S) -> T} and {@code (A... , T) -> S}, respectively, where {@code T} is the type of the target var handle,
-     * or if either {@code filterFromTarget} or {@code filterToTarget} throws any checked exceptions.
+     * or if it's determined that either {@code filterFromTarget} or {@code filterToTarget} throws any checked exceptions.
      */
     public static VarHandle filterValue(VarHandle target, MethodHandle filterToTarget, MethodHandle filterFromTarget) {
         return JLI.filterValue(target, filterToTarget, filterFromTarget);
@@ -356,6 +359,9 @@ public static VarHandle filterValue(VarHandle target, MethodHandle filterToTarge
      * For the coordinate filters to be well formed, their types must be of the form {@code S1 -> T1, S2 -> T1 ... Sn -> Tn},
      * where {@code T1, T2 ... Tn} are the coordinate types starting at position {@code pos} of the target var handle.
      * <p>
+     * If any of the filters throws a checked exception when invoked, the resulting var handle will
+     * throw an {@link IllegalStateException}.
+     * <p>
      * The resulting var handle will feature the same access modes (see {@link java.lang.invoke.VarHandle.AccessMode}) and
      * atomic access guarantees as those featured by the target var handle.
      *
@@ -368,7 +374,7 @@ public static VarHandle filterValue(VarHandle target, MethodHandle filterToTarge
      * other than {@code S1 -> T1, S2 -> T2, ... Sn -> Tn} where {@code T1, T2 ... Tn} are the coordinate types starting
      * at position {@code pos} of the target var handle, if {@code pos} is not between 0 and the target var handle coordinate arity, inclusive,
      * or if more filters are provided than the actual number of coordinate types available starting at {@code pos},
-     * or if any of the filters throws any checked exceptions.
+     * or if it's determined that any of the filters throws any checked exceptions.
      */
     public static VarHandle filterCoordinates(VarHandle target, int pos, MethodHandle... filters) {
         return JLI.filterCoordinates(target, pos, filters);
@@ -464,6 +470,9 @@ public static VarHandle permuteCoordinates(VarHandle target, List<Class<?>> newC
      * coordinate type of the target var handle at position {@code pos}, and that target var handle
      * coordinate is supplied by the return value of the filter.
      * <p>
+     * If any of the filters throws a checked exception when invoked, the resulting var handle will
+     * throw an {@link IllegalStateException}.
+     * <p>
      * The resulting var handle will feature the same access modes (see {@link java.lang.invoke.VarHandle.AccessMode}) and
      * atomic access guarantees as those featured by the target var handle.
      *
@@ -476,7 +485,7 @@ public static VarHandle permuteCoordinates(VarHandle target, List<Class<?>> newC
      * is void, or it is not the same as the {@code pos} coordinate of the target var handle,
      * if {@code pos} is not between 0 and the target var handle coordinate arity, inclusive,
      * if the resulting var handle's type would have <a href="MethodHandle.html#maxarity">too many coordinates</a>,
-     * or if {@code filter} throws any checked exceptions.
+     * or if it's determined that {@code filter} throws any checked exceptions.
      */
     public static VarHandle collectCoordinates(VarHandle target, int pos, MethodHandle filter) {
         return JLI.collectCoordinates(target, pos, filter);

src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/CABI.java

@@ -25,9 +25,10 @@
  */
 package jdk.internal.foreign;
 
-import jdk.internal.foreign.abi.SharedUtils;
+import sun.security.action.GetPropertyAction;
 
 import static jdk.incubator.foreign.MemoryLayouts.ADDRESS;
+import static sun.security.action.GetPropertyAction.privilegedGetProperty;
 
 public enum CABI {
     SysV,
@@ -37,8 +38,8 @@ public enum CABI {
     private static final CABI current;
 
     static {
-        String arch = System.getProperty("os.arch");
-        String os = System.getProperty("os.name");
+        String arch = privilegedGetProperty("os.arch");
+        String os = privilegedGetProperty("os.name");
         long addressSize = ADDRESS.bitSize();
         // might be running in a 32-bit VM on a 64-bit platform.
         // addressSize will be correctly 32

src/jdk.incubator.foreign/share/classes/jdk/internal/foreign/abi/ProgrammableUpcallHandler.java

@@ -295,6 +295,9 @@ private static Object invokeInterpBindings(Object[] moves, MethodHandle leaf,
             } else {
                 return returnMoves;
             }
+        } catch(Throwable t) {
+            SharedUtils.handleUncaughtException(t);
+            return null;
         }
     }