8310829: guarantee(!HAS_PENDING_EXCEPTION) failed in ExceptionTranslation::doit

Reviewed-by: chagedorn
Backport-of: f6bdccb45caca0f69918a773a9ad9b2ad91b702f

Author: TobiHartmann

src/hotspot/share/classfile/vmClassMacros.hpp

@@ -31,7 +31,7 @@
 #define VM_CLASS_ID(kname)      vmClassID::_VM_CLASS_ENUM(kname)
 
 // VM_CLASSES_DO iterates the classes that are directly referenced
-// by the VM, suhch as java.lang.Object and java.lang.String. These
+// by the VM, such as java.lang.Object and java.lang.String. These
 // classes are resolved at VM bootstrap, before any Java code is executed,
 // so no class loader is able to provide a different definition.
 //

src/hotspot/share/classfile/vmSymbols.hpp

@@ -760,7 +760,7 @@
   template(decodeAndThrowThrowable_name,               "decodeAndThrowThrowable")                                 \
   template(encodeAnnotations_name,                     "encodeAnnotations")                                       \
   template(encodeAnnotations_signature,                "([BLjava/lang/Class;Ljdk/internal/reflect/ConstantPool;Z[Ljava/lang/Class;)[B")\
-  template(decodeAndThrowThrowable_signature,          "(JZ)V")                                                   \
+  template(decodeAndThrowThrowable_signature,          "(IJZ)V")                                                  \
   template(classRedefinedCount_name,                   "classRedefinedCount")                                     \
   template(classLoader_name,                           "classLoader")                                             \
   template(componentType_name,                         "componentType")                                           \

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -582,7 +582,7 @@ C2V_VMENTRY_NULL(jobject, lookupType, (JNIEnv* env, jobject, jstring jname, ARGU
   TempNewSymbol class_name = SymbolTable::new_symbol(str);
 
   if (class_name->utf8_length() <= 1) {
-    JVMCI_THROW_MSG_0(InternalError, err_msg("Primitive type %s should be handled in Java code", class_name->as_C_string()));
+    JVMCI_THROW_MSG_0(InternalError, err_msg("Primitive type %s should be handled in Java code", str));
   }
 
   JVMCIKlassHandle resolved_klass(THREAD);

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -446,6 +446,15 @@ bool JVMCIEnv::pending_exception_as_string(const char** to_string, const char**
 // Shared code for translating an exception from HotSpot to libjvmci or vice versa.
 class ExceptionTranslation: public StackObj {
  protected:
+  enum DecodeFormat {
+    _encoded_ok        = 0, // exception was successfully encoded into buffer
+    _buffer_alloc_fail = 1, // native memory for buffer could not be allocated
+    _encode_oome_fail  = 2, // OutOfMemoryError thrown during encoding
+    _encode_fail       = 3  // some other problem occured during encoding. If buffer != 0,
+                            // buffer contains a `struct { u4 len; char[len] desc}`
+                            // describing the problem
+  };
+
   JVMCIEnv*  _from_env; // Source of translation. Can be null.
   JVMCIEnv*  _to_env;   // Destination of translation. Never null.
 
@@ -454,49 +463,34 @@ class ExceptionTranslation: public StackObj {
   // Encodes the exception in `_from_env` into `buffer`.
   // Where N is the number of bytes needed for the encoding, returns N if N <= `buffer_size`
   // and the encoding was written to `buffer` otherwise returns -N.
-  virtual int encode(JavaThread* THREAD, Klass* vmSupport, jlong buffer, int buffer_size) = 0;
+  virtual int encode(JavaThread* THREAD, jlong buffer, int buffer_size) = 0;
 
   // Decodes the exception in `buffer` in `_to_env` and throws it.
-  virtual void decode(JavaThread* THREAD, Klass* vmSupport, jlong buffer) = 0;
+  virtual void decode(JavaThread* THREAD, DecodeFormat format, jlong buffer) = 0;
 
  public:
   void doit(JavaThread* THREAD) {
-    // Resolve VMSupport class explicitly as HotSpotJVMCI::compute_offsets
-    // may not have been called.
-    Klass* vmSupport = SystemDictionary::resolve_or_fail(vmSymbols::jdk_internal_vm_VMSupport(), true, THREAD);
-    guarantee(!HAS_PENDING_EXCEPTION, "");
-
     int buffer_size = 2048;
     while (true) {
       ResourceMark rm;
       jlong buffer = (jlong) NEW_RESOURCE_ARRAY_IN_THREAD_RETURN_NULL(THREAD, jbyte, buffer_size);
       if (buffer == 0L) {
-        decode(THREAD, vmSupport, 0L);
+        JVMCI_event_1("error translating exception: translation buffer allocation failed");
+        decode(THREAD, _buffer_alloc_fail, 0L);
         return;
       }
-      int res = encode(THREAD, vmSupport, buffer, buffer_size);
-      if (_from_env != nullptr && !_from_env->is_hotspot() && _from_env->has_pending_exception()) {
-        // Cannot get name of exception thrown by `encode` as that involves
-        // calling into libjvmci which in turn can raise another exception.
-        _from_env->clear_pending_exception();
-        decode(THREAD, vmSupport, -2L);
-        return;
-      } else if (HAS_PENDING_EXCEPTION) {
-        Symbol *ex_name = PENDING_EXCEPTION->klass()->name();
-        CLEAR_PENDING_EXCEPTION;
-        if (ex_name == vmSymbols::java_lang_OutOfMemoryError()) {
-          decode(THREAD, vmSupport, -1L);
-        } else {
-          decode(THREAD, vmSupport, -2L);
-        }
+      int res = encode(THREAD, buffer, buffer_size);
+      if (_to_env->has_pending_exception()) {
+        // Propagate pending exception
         return;
-      } else if (res < 0) {
+      }
+      if (res < 0) {
         int required_buffer_size = -res;
         if (required_buffer_size > buffer_size) {
           buffer_size = required_buffer_size;
         }
       } else {
-        decode(THREAD, vmSupport, buffer);
+        decode(THREAD, _encoded_ok, buffer);
         if (!_to_env->has_pending_exception()) {
           _to_env->throw_InternalError("decodeAndThrowThrowable should have thrown an exception");
         }
@@ -511,7 +505,26 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
  private:
   const Handle& _throwable;
 
-  int encode(JavaThread* THREAD, Klass* vmSupport, jlong buffer, int buffer_size) {
+  int encode(JavaThread* THREAD, jlong buffer, int buffer_size) {
+    Klass* vmSupport = SystemDictionary::resolve_or_fail(vmSymbols::jdk_internal_vm_VMSupport(), true, THREAD);
+    if (HAS_PENDING_EXCEPTION) {
+      Handle throwable = Handle(THREAD, PENDING_EXCEPTION);
+      Symbol *ex_name = throwable->klass()->name();
+      CLEAR_PENDING_EXCEPTION;
+      if (ex_name == vmSymbols::java_lang_OutOfMemoryError()) {
+        JVMCI_event_1("error translating exception: OutOfMemoryError");
+        decode(THREAD, _encode_oome_fail, 0L);
+      } else {
+        char* char_buffer = (char*) buffer + 4;
+        stringStream st(char_buffer, (size_t) buffer_size - 4);
+        java_lang_Throwable::print_stack_trace(throwable, &st);
+        u4 len = (u4) st.size();
+        *((u4*) buffer) = len;
+        JVMCI_event_1("error translating exception: %s", char_buffer);
+        decode(THREAD, _encode_fail, buffer);
+      }
+      return 0;
+    }
     JavaCallArguments jargs;
     jargs.push_oop(_throwable);
     jargs.push_long(buffer);
@@ -524,11 +537,11 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
     return result.get_jint();
   }
 
-  void decode(JavaThread* THREAD, Klass* vmSupport, jlong buffer) {
+  void decode(JavaThread* THREAD, DecodeFormat format, jlong buffer) {
     JNIAccessMark jni(_to_env, THREAD);
     jni()->CallStaticVoidMethod(JNIJVMCI::VMSupport::clazz(),
                                 JNIJVMCI::VMSupport::decodeAndThrowThrowable_method(),
-                                buffer, false);
+                                format, buffer, false);
   }
  public:
   HotSpotToSharedLibraryExceptionTranslation(JVMCIEnv* hotspot_env, JVMCIEnv* jni_env, const Handle& throwable) :
@@ -540,15 +553,25 @@ class SharedLibraryToHotSpotExceptionTranslation : public ExceptionTranslation {
  private:
   jthrowable _throwable;
 
-  int encode(JavaThread* THREAD, Klass* vmSupport, jlong buffer, int buffer_size) {
+  int encode(JavaThread* THREAD, jlong buffer, int buffer_size) {
     JNIAccessMark jni(_from_env, THREAD);
-    return jni()->CallStaticIntMethod(JNIJVMCI::VMSupport::clazz(),
+    int res = jni()->CallStaticIntMethod(JNIJVMCI::VMSupport::clazz(),
                                       JNIJVMCI::VMSupport::encodeThrowable_method(),
                                       _throwable, buffer, buffer_size);
+    if (jni()->ExceptionCheck()) {
+      // Cannot get name of exception thrown as that can raise another exception.
+      jni()->ExceptionClear();
+      JVMCI_event_1("error translating exception: unknown error");
+      decode(THREAD, _encode_fail, 0L);
+      return 0;
+    }
+    return res;
   }
 
-  void decode(JavaThread* THREAD, Klass* vmSupport, jlong buffer) {
+  void decode(JavaThread* THREAD, DecodeFormat format, jlong buffer) {
+    Klass* vmSupport = SystemDictionary::resolve_or_fail(vmSymbols::jdk_internal_vm_VMSupport(), true, CHECK);
     JavaCallArguments jargs;
+    jargs.push_int(format);
     jargs.push_long(buffer);
     jargs.push_int(true);
     JavaValue result(T_VOID);

src/java.base/share/classes/jdk/internal/vm/VMSupport.java

@@ -40,6 +40,7 @@
 import java.io.IOException;
 import java.lang.annotation.Annotation;
 import java.lang.annotation.IncompleteAnnotationException;
+import java.nio.charset.StandardCharsets;
 import java.util.Collection;
 import java.util.LinkedHashMap;
 import java.util.Map;
@@ -125,36 +126,46 @@ public static byte[] serializeSavedPropertiesToByteArray() throws IOException {
     public static native String getVMTemporaryDirectory();
 
     /**
-     * Decodes the exception encoded in {@code errorOrBuffer} and throws it.
-     *
-     * @param errorOrBuffer an error code or a native byte errorOrBuffer containing an exception encoded by
-     *            {@link #encodeThrowable}. Error code values and their meanings are:
+     * Decodes the exception described by {@code format} and {@code buffer} and throws it.
      *
+     * @param format specifies how to interpret {@code buffer}:
      *            <pre>
-     *             0: native memory for the errorOrBuffer could not be allocated
-     *            -1: an OutOfMemoryError was thrown while encoding the exception
-     *            -2: some other throwable was thrown while encoding the exception
+     *             0: {@code buffer} was created by {@link #encodeThrowable}
+     *             1: native memory for {@code buffer} could not be allocated
+     *             2: an OutOfMemoryError was thrown while encoding the exception
+     *             3: some other problem occured while encoding the exception. If {@code buffer != 0},
+     *                it contains a {@code struct { u4 len; char[len] desc}} where {@code desc} describes the problem
      *            </pre>
-     * @param errorOrBuffer a native byte errorOrBuffer containing an exception encoded by
-     *            {@link #encodeThrowable}
+     * @param buffer encoded info about the exception to throw (depends on {@code format})
      * @param inJVMHeap [@code true} if executing in the JVM heap, {@code false} otherwise
      */
-    public static void decodeAndThrowThrowable(long errorOrBuffer, boolean inJVMHeap) throws Throwable {
-        if (errorOrBuffer >= -2L && errorOrBuffer <= 0) {
+    public static void decodeAndThrowThrowable(int format, long buffer, boolean inJVMHeap) throws Throwable {
+        if (format != 0) {
             String context = String.format("while encoding an exception to translate it %s the JVM heap",
                     inJVMHeap ? "to" : "from");
-            if (errorOrBuffer == 0) {
-                throw new InternalError("native errorOrBuffer could not be allocated " + context);
+            if (format == 1) {
+                throw new InternalError("native buffer could not be allocated " + context);
             }
-            if (errorOrBuffer == -1L) {
+            if (format == 2) {
                 throw new OutOfMemoryError("OutOfMemoryError occurred " + context);
             }
+            if (format == 3 && buffer != 0L) {
+                byte[] bytes = bufferToBytes(buffer);
+                throw new InternalError("unexpected problem occurred " + context + ": " + new String(bytes, StandardCharsets.UTF_8));
+            }
             throw new InternalError("unexpected problem occurred " + context);
         }
-        int encodingLength = U.getInt(errorOrBuffer);
-        byte[] encoding = new byte[encodingLength];
-        U.copyMemory(null, errorOrBuffer + 4, encoding, Unsafe.ARRAY_BYTE_BASE_OFFSET, encodingLength);
-        throw TranslatedException.decodeThrowable(encoding);
+        throw TranslatedException.decodeThrowable(bufferToBytes(buffer));
+    }
+
+    private static byte[] bufferToBytes(long buffer) {
+        if (buffer == 0) {
+            return null;
+        }
+        int len = U.getInt(buffer);
+        byte[] bytes = new byte[len];
+        U.copyMemory(null, buffer + 4, bytes, Unsafe.ARRAY_BYTE_BASE_OFFSET, len);
+        return bytes;
     }
 
     /**

test/jdk/jdk/internal/vm/TestTranslatedException.java

@@ -34,6 +34,7 @@
 import java.io.PrintStream;
 import java.lang.reflect.InvocationTargetException;
 import java.lang.reflect.Method;
+import java.nio.charset.StandardCharsets;
 
 import org.testng.Assert;
 import org.testng.annotations.Test;
@@ -57,11 +58,78 @@ public void encodeDecodeTest() throws Exception {
             throwable = new ExceptionInInitializerError(new InvocationTargetException(new RuntimeException(String.valueOf(i), throwable), "invoke"));
         }
         encodeDecode(throwable);
+
+        try {
+            VMSupport.decodeAndThrowThrowable(0, 0L, true);
+            throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
+        } catch (NullPointerException decoded) {
+            // Expected
+        } catch (Throwable decoded) {
+            throw new AssertionError("unexpected exception: " + decoded);
+        }
+
+        try {
+            VMSupport.decodeAndThrowThrowable(1, 0L, true);
+            throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
+        } catch (InternalError decoded) {
+            if (!decoded.getMessage().startsWith("native buffer could not be allocated")) {
+                throw new AssertionError("unexpected exception: " + decoded);
+            }
+        } catch (Throwable decoded) {
+            throw new AssertionError("unexpected exception: " + decoded);
+        }
+
+        try {
+            VMSupport.decodeAndThrowThrowable(2, 0L, true);
+            throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
+        } catch (OutOfMemoryError decoded) {
+            // Expected
+        } catch (Throwable decoded) {
+            throw new AssertionError("unexpected exception: " + decoded);
+        }
+
+        try {
+            VMSupport.decodeAndThrowThrowable(3, 0L, true);
+            throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
+        } catch (InternalError decoded) {
+            // Expected
+        } catch (Throwable decoded) {
+            throw new AssertionError("unexpected exception: " + decoded);
+        }
+
+        try {
+            VMSupport.decodeAndThrowThrowable(4, 0L, true);
+            throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
+        } catch (InternalError decoded) {
+            // Expected
+        } catch (Throwable decoded) {
+            throw new AssertionError("unexpected exception: " + decoded);
+        }
+
+        Unsafe unsafe = Unsafe.getUnsafe();
+        byte[] problem = "very unlikely problem".getBytes(StandardCharsets.UTF_8);
+        long buffer = unsafe.allocateMemory(problem.length + 4);
+        try {
+            unsafe.putInt(buffer, problem.length);
+            unsafe.copyMemory(problem, Unsafe.ARRAY_BYTE_BASE_OFFSET, null, buffer + 4, problem.length);
+            VMSupport.decodeAndThrowThrowable(3, buffer, true);
+            throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
+        } catch (InternalError decoded) {
+            String msg = decoded.getMessage();
+            if (!msg.endsWith("very unlikely problem")) {
+                throw new AssertionError("unexpected exception: " + decoded);
+            }
+        } catch (Throwable decoded) {
+            throw new AssertionError("unexpected exception: " + decoded);
+        } finally {
+            unsafe.freeMemory(buffer);
+        }
     }
 
     private void encodeDecode(Throwable throwable) throws Exception {
         Unsafe unsafe = Unsafe.getUnsafe();
         int bufferSize = 512;
+        int format = 0;
         long buffer = 0L;
         while (true) {
             buffer = unsafe.allocateMemory(bufferSize);
@@ -71,7 +139,7 @@ private void encodeDecode(Throwable throwable) throws Exception {
                     bufferSize = -res;
                 } else {
                     try {
-                        VMSupport.decodeAndThrowThrowable(buffer, true);
+                        VMSupport.decodeAndThrowThrowable(format, buffer, true);
                         throw new AssertionError("expected decodeAndThrowThrowable to throw an exception");
                     } catch (Throwable decoded) {
                         assertThrowableEquals(throwable, decoded);