8335553: [Graal] Compiler thread calls into jdk.internal.vm.VMSupport.decodeAndThrowThrowable and crashes in OOM situation

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -757,6 +757,35 @@ C2V_VMENTRY_NULL(jobject, lookupConstantInPool, (JNIEnv* env, jobject, ARGUMENT_
       return JVMCIENV->get_jobject(result);
     }
   }
+#ifdef ASSERT
+  // Support for testing an OOME raised in a context where the current thread cannot call Java
+  // 1. Put -Dtest.jvmci.oome_in_lookupConstantInPool=<trace> on the command line to
+  //    discover possible values for step 2.
+  //    Example output:
+  //
+  //      CompilerToVM.lookupConstantInPool: "Overflow: String length out of range"{0x00000007ffeb2960}
+  //      CompilerToVM.lookupConstantInPool: "null"{0x00000007ffebdfe8}
+  //      CompilerToVM.lookupConstantInPool: "Maximum lock count exceeded"{0x00000007ffec4f90}
+  //      CompilerToVM.lookupConstantInPool: "Negative length"{0x00000007ffec4468}
+  //
+  // 2. Choose a value shown in step 1.
+  //    Example: -Dtest.jvmci.oome_in_lookupConstantInPool=Negative
+  const char* val = Arguments::PropertyList_get_value(Arguments::system_properties(), "test.jvmci.oome_in_lookupConstantInPool");
+  if (val != nullptr) {
+    const char* str = obj->print_value_string();
+    if (strstr(val, "<trace>") != nullptr) {
+      tty->print_cr("CompilerToVM.lookupConstantInPool: %s", str);
+    } else if (strstr(str, val) != nullptr) {
+      Handle garbage;
+      while (true) {
+        // Trigger an OutOfMemoryError
+        objArrayOop next = oopFactory::new_objectArray(0x7FFFFFFF, CHECK_NULL);
+        next->obj_at_put(0, garbage());
+        garbage = Handle(THREAD, next);
+      }
+    }
+  }
+#endif
   return JVMCIENV->get_jobject(JVMCIENV->get_object_constant(obj));
 C2V_END
 

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -399,9 +399,11 @@ class ExceptionTranslation: public StackObj {
     _encoded_ok        = 0, // exception was successfully encoded into buffer
     _buffer_alloc_fail = 1, // native memory for buffer could not be allocated
     _encode_oome_fail  = 2, // OutOfMemoryError thrown during encoding
-    _encode_fail       = 3  // some other problem occured during encoding. If buffer != 0,
+    _encode_fail       = 3, // some other problem occured during encoding. If buffer != 0,
                             // buffer contains a `struct { u4 len; char[len] desc}`
                             // describing the problem
+    _encode_oome_in_vm = 4  // an OutOfMemoryError thrown from within VM code on a
+                            // thread that cannot call Java (OOME has no stack trace)
   };
 
   JVMCIEnv*  _from_env; // Source of translation. Can be null.
@@ -488,6 +490,12 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
 
   int encode(JavaThread* THREAD, jlong buffer, int buffer_size) {
     if (!THREAD->can_call_java()) {
+      Symbol *ex_name = _throwable->klass()->name();
+      if (ex_name == vmSymbols::java_lang_OutOfMemoryError()) {
+        JVMCI_event_1("translating exception: OutOfMemoryError within VM code");
+        decode(THREAD, _encode_oome_in_vm, 0L);
+        return 0;
+      }
       char* char_buffer = print_throwable_to_buffer(_throwable, buffer, buffer_size);
       const char* detail = log_is_enabled(Info, exceptions) ? "" : " (-Xlog:exceptions may give more detail)";
       JVMCI_event_1("cannot call Java to translate exception%s: %s", detail, char_buffer);

src/hotspot/share/utilities/exceptions.cpp

@@ -111,11 +111,9 @@ bool Exceptions::special_exception(JavaThread* thread, const char* file, int lin
   }
 #endif // ASSERT
 
-  if (!thread->can_call_java()) {
+  if (h_exception.is_null() && !thread->can_call_java()) {
     ResourceMark rm(thread);
-    const char* exc_value = h_exception.not_null() ? h_exception->print_value_string() :
-                      h_name != nullptr ? h_name->as_C_string() :
-                      "null";
+    const char* exc_value = h_name != nullptr ? h_name->as_C_string() : "null";
     log_info(exceptions)("Thread cannot call Java so instead of throwing exception <%s%s%s> (" PTR_FORMAT ") \n"
                         "at [%s, line %d]\nfor thread " PTR_FORMAT ",\n"
                         "throwing pre-allocated exception: %s",
@@ -205,7 +203,7 @@ void Exceptions::_throw_msg_cause(JavaThread* thread, const char* file, int line
 void Exceptions::_throw_cause(JavaThread* thread, const char* file, int line, Symbol* name, Handle h_cause,
                               Handle h_loader, Handle h_protection_domain) {
   // Check for special boot-strapping/compiler-thread handling
-  if (special_exception(thread, file, line, h_cause)) return;
+  if (special_exception(thread, file, line, Handle(), name)) return;
   // Create and throw exception
   Handle h_exception = new_exception(thread, name, h_cause, h_loader, h_protection_domain);
   _throw(thread, file, line, h_exception, nullptr);

src/java.base/share/classes/jdk/internal/vm/TranslatedException.java

@@ -42,9 +42,13 @@
 
 /**
  * Support for translating exceptions between the HotSpot heap and libjvmci heap.
+ *
+ * Successfully translated exceptions are wrapped in a TranslatedException instance.
+ * This allows callers to distiguish between a translated exception and an error
+ * that arose during translation.
  */
 @SuppressWarnings("serial")
-final class TranslatedException extends Exception {
+public final class TranslatedException extends Exception {
 
     /**
      * The value returned by {@link #encodeThrowable(Throwable)} when encoding
@@ -61,15 +65,18 @@ final class TranslatedException extends Exception {
         maybeFailClinit();
         try {
             FALLBACK_ENCODED_THROWABLE_BYTES =
-                encodeThrowable(new TranslatedException("error during encoding",
-                                                        "<unknown>"), false);
+                encodeThrowable(translationFailure("error during encoding"), false);
             FALLBACK_ENCODED_OUTOFMEMORYERROR_BYTES =
-                encodeThrowable(new OutOfMemoryError(), false);
+                encodeThrowable(translationFailure("OutOfMemoryError during encoding"), false);
         } catch (IOException e) {
             throw new InternalError(e);
         }
     }
 
+    private static InternalError translationFailure(String messageFormat, Object... messageArgs) {
+        return new InternalError(messageFormat.formatted(messageArgs));
+    }
+
     /**
      * Helper to test exception translation.
      */
@@ -86,14 +93,8 @@ private static void maybeFailClinit() {
         }
     }
 
-    /**
-     * Class name of exception that could not be instantiated.
-     */
-    private String originalExceptionClassName;
-
-    private TranslatedException(String message, String originalExceptionClassName) {
-        super(message);
-        this.originalExceptionClassName = originalExceptionClassName;
+    TranslatedException(Throwable translated) {
+        super(translated);
     }
 
     /**
@@ -106,18 +107,6 @@ public Throwable fillInStackTrace() {
         return this;
     }
 
-    @Override
-    public String toString() {
-        String s;
-        if (originalExceptionClassName.equals(TranslatedException.class.getName())) {
-            s = getClass().getName();
-        } else {
-            s = getClass().getName() + "[" + originalExceptionClassName + "]";
-        }
-        String message = getMessage();
-        return (message != null) ? (s + ": " + message) : s;
-    }
-
     /**
      * Prints a stack trace for {@code throwable} if the system property
      * {@code "jdk.internal.vm.TranslatedException.debug"} is true.
@@ -163,7 +152,7 @@ private static Throwable create(String className, String message, Throwable caus
             return initCause((Throwable) cons.newInstance(message), cause, debug);
         } catch (Throwable translationFailure) {
             debugPrintStackTrace(translationFailure, debug);
-            return initCause(new TranslatedException(message, className), cause, debug);
+            return initCause(translationFailure("%s [%s]", message, className), cause, debug);
         }
     }
 
@@ -308,11 +297,10 @@ static Throwable decodeThrowable(byte[] encodedThrowable, boolean debug) {
                 throwable.setStackTrace(stackTrace);
                 cause = throwable;
             }
-            return throwable;
+            return new TranslatedException(throwable);
         } catch (Throwable translationFailure) {
             debugPrintStackTrace(translationFailure, debug);
-            return new TranslatedException("Error decoding exception: " + encodedThrowable,
-                                           translationFailure.getClass().getName());
+            return translationFailure("error decoding exception: %s", encodedThrowable);
         }
     }
 }

src/java.base/share/classes/jdk/internal/vm/VMSupport.java

@@ -122,20 +122,25 @@ public static byte[] serializeAgentPropertiesToByteArray() throws IOException {
      *             2: an OutOfMemoryError was thrown while encoding the exception
      *             3: some other problem occured while encoding the exception. If {@code buffer != 0},
      *                it contains a {@code struct { u4 len; char[len] desc}} where {@code desc} describes the problem
+     *             4: an OutOfMemoryError thrown from within VM code on a
+     *                thread that cannot call Java (OOME has no stack trace)
      *            </pre>
      * @param buffer encoded info about the exception to throw (depends on {@code format})
      * @param inJVMHeap [@code true} if executing in the JVM heap, {@code false} otherwise
      * @param debug specifies whether debug stack traces should be enabled in case of translation failure
      */
     public static void decodeAndThrowThrowable(int format, long buffer, boolean inJVMHeap, boolean debug) throws Throwable {
         if (format != 0) {
+            if (format == 4) {
+                throw new TranslatedException(new OutOfMemoryError("in VM code and current thread cannot call Java"));
+            }
             String context = String.format("while encoding an exception to translate it %s the JVM heap",
                     inJVMHeap ? "to" : "from");
             if (format == 1) {
                 throw new InternalError("native buffer could not be allocated " + context);
             }
             if (format == 2) {
-                throw new OutOfMemoryError("OutOfMemoryError occurred " + context);
+                throw new OutOfMemoryError(context);
             }
             if (format == 3 && buffer != 0L) {
                 byte[] bytes = bufferToBytes(buffer);

test/jdk/jdk/internal/vm/TestTranslatedException.java

@@ -40,6 +40,7 @@
 import org.testng.annotations.Test;
 
 import jdk.internal.misc.Unsafe;
+import jdk.internal.vm.TranslatedException;
 import jdk.internal.vm.VMSupport;
 
 public class TestTranslatedException {
@@ -100,6 +101,15 @@ public void encodeDecodeTest() throws Exception {
         try {
             VMSupport.decodeAndThrowThrowable(4, 0L, true, false);
             throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
+        } catch (TranslatedException decoded) {
+            Assert.assertEquals(decoded.getCause().getClass(), OutOfMemoryError.class);
+        } catch (Throwable decoded) {
+            throw new AssertionError("unexpected exception: " + decoded);
+        }
+
+        try {
+            VMSupport.decodeAndThrowThrowable(5, 0L, true, false);
+            throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
         } catch (InternalError decoded) {
             // Expected
         } catch (Throwable decoded) {
@@ -142,7 +152,7 @@ private void encodeDecode(Throwable throwable) throws Exception {
                         VMSupport.decodeAndThrowThrowable(format, buffer, true, false);
                         throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
                     } catch (Throwable decoded) {
-                        assertThrowableEquals(throwable, decoded);
+                        assertThrowableEquals(throwable, decoded.getCause());
                     }
                     return;
                 }
@@ -152,13 +162,15 @@ private void encodeDecode(Throwable throwable) throws Exception {
         }
     }
 
-    private static void assertThrowableEquals(Throwable original, Throwable decoded) {
+    private static void assertThrowableEquals(Throwable originalIn, Throwable decodedIn) {
+        Throwable original = originalIn;
+        Throwable decoded = decodedIn;
         try {
             Assert.assertEquals(original == null, decoded == null);
             while (original != null) {
                 if (Untranslatable.class.equals(original.getClass())) {
-                    Assert.assertEquals(decoded.getClass().getName(), "jdk.internal.vm.TranslatedException");
-                    Assert.assertEquals(decoded.toString(), "jdk.internal.vm.TranslatedException[jdk.internal.vm.test.TestTranslatedException$Untranslatable]: test exception");
+                    Assert.assertEquals(decoded.getClass().getName(), "java.lang.InternalError");
+                    Assert.assertEquals(decoded.toString(), "java.lang.InternalError: test exception [jdk.internal.vm.test.TestTranslatedException$Untranslatable]");
                     Assert.assertEquals(original.getMessage(), "test exception");
                 } else {
                     Assert.assertEquals(decoded.getClass().getName(), original.getClass().getName());
@@ -182,10 +194,10 @@ private static void assertThrowableEquals(Throwable original, Throwable decoded)
             }
         } catch (AssertionError e) {
             System.err.println("original:[");
-            original.printStackTrace(System.err);
+            originalIn.printStackTrace(System.err);
             System.err.println("]");
             System.err.println("decoded:[");
-            original.printStackTrace(System.err);
+            decodedIn.printStackTrace(System.err);
             System.err.println("]");
             throw e;
         }