8313899: JVMCI exception Translation can fail in TranslatedException.<clinit>

src/hotspot/share/jvmci/jvmciCompilerToVM.cpp

@@ -52,6 +52,7 @@
 #include "prims/jvmtiExport.hpp"
 #include "prims/methodHandles.hpp"
 #include "prims/nativeLookup.hpp"
+#include "runtime/arguments.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/deoptimization.hpp"
 #include "runtime/fieldDescriptor.inline.hpp"
@@ -585,6 +586,18 @@ C2V_VMENTRY_NULL(jobject, lookupType, (JNIEnv* env, jobject, jstring jname, ARGU
     JVMCI_THROW_MSG_0(InternalError, err_msg("Primitive type %s should be handled in Java code", str));
   }
 
+#ifdef ASSERT
+  const char* val = Arguments::PropertyList_get_value(Arguments::system_properties(), "test.jvmci.lookupTypeException");
+  if (val != nullptr) {
+    if (strstr(val, "<trace>") != nullptr) {
+      tty->print_cr("CompilerToVM.lookupType: %s", str);
+    } else if (strstr(val, str) != nullptr) {
+      THROW_MSG_0(vmSymbols::java_lang_Exception(),
+                  err_msg("lookupTypeException: %s", str));
+    }
+  }
+#endif
+
   JVMCIKlassHandle resolved_klass(THREAD);
   Klass* accessing_klass = UNPACK_PAIR(Klass, accessing_klass);
   Handle class_loader;

src/hotspot/share/jvmci/jvmciEnv.cpp

@@ -433,8 +433,7 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
  private:
   const Handle& _throwable;
 
-  int encode(JavaThread* THREAD, jlong buffer, int buffer_size) {
-    Klass* vmSupport = SystemDictionary::resolve_or_fail(vmSymbols::jdk_internal_vm_VMSupport(), true, THREAD);
+  bool handle_pending_exception(JavaThread* THREAD, jlong buffer, int buffer_size) {
     if (HAS_PENDING_EXCEPTION) {
       Handle throwable = Handle(THREAD, PENDING_EXCEPTION);
       Symbol *ex_name = throwable->klass()->name();
@@ -451,6 +450,14 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
         JVMCI_event_1("error translating exception: %s", char_buffer);
         decode(THREAD, _encode_fail, buffer);
       }
+      return true;
+    }
+    return false;
+  }
+
+  int encode(JavaThread* THREAD, jlong buffer, int buffer_size) {
+    Klass* vmSupport = SystemDictionary::resolve_or_fail(vmSymbols::jdk_internal_vm_VMSupport(), true, THREAD);
+    if (handle_pending_exception(THREAD, buffer, buffer_size)) {
       return 0;
     }
     JavaCallArguments jargs;
@@ -462,6 +469,9 @@ class HotSpotToSharedLibraryExceptionTranslation : public ExceptionTranslation {
                             vmSupport,
                             vmSymbols::encodeThrowable_name(),
                             vmSymbols::encodeThrowable_signature(), &jargs, THREAD);
+    if (handle_pending_exception(THREAD, buffer, buffer_size)) {
+      return 0;
+    }
     return result.get_jint();
   }
 

src/hotspot/share/jvmci/jvmciRuntime.cpp

@@ -1246,11 +1246,13 @@ JNIEnv* JVMCIRuntime::init_shared_library_javavm(int* create_JavaVM_err) {
   MutexLocker locker(_lock);
   JavaVM* javaVM = _shared_library_javavm;
   if (javaVM == nullptr) {
+#ifdef ASSERT
     const char* val = Arguments::PropertyList_get_value(Arguments::system_properties(), "test.jvmci.forceEnomemOnLibjvmciInit");
     if (val != nullptr && strcmp(val, "true") == 0) {
       *create_JavaVM_err = JNI_ENOMEM;
       return nullptr;
     }
+#endif
 
     char* sl_path;
     void* sl_handle = JVMCI::get_shared_library(sl_path, true);
@@ -2061,12 +2063,14 @@ void JVMCIRuntime::compile_method(JVMCIEnv* JVMCIENV, JVMCICompiler* compiler, c
 
   JVMCIObject result_object = JVMCIENV->call_HotSpotJVMCIRuntime_compileMethod(receiver, jvmci_method, entry_bci,
                                                                      (jlong) compile_state, compile_state->task()->compile_id());
+#ifdef ASSERT
   if (JVMCIENV->has_pending_exception()) {
     const char* val = Arguments::PropertyList_get_value(Arguments::system_properties(), "test.jvmci.compileMethodExceptionIsFatal");
     if (val != nullptr && strcmp(val, "true") == 0) {
       fatal_exception(JVMCIENV, "testing JVMCI fatal exception handling");
     }
   }
+#endif
 
   if (after_compiler_upcall(JVMCIENV, compiler, method, "call_HotSpotJVMCIRuntime_compileMethod")) {
     return;

src/java.base/share/classes/jdk/internal/vm/TranslatedException.java

@@ -24,6 +24,8 @@
  */
 package jdk.internal.vm;
 
+import jdk.internal.misc.VM;
+
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.DataInputStream;
@@ -56,6 +58,7 @@ final class TranslatedException extends Exception {
      */
     private static final byte[] FALLBACK_ENCODED_THROWABLE_BYTES;
     static {
+        maybeFailClinit();
         try {
             FALLBACK_ENCODED_THROWABLE_BYTES =
                 encodeThrowable(new TranslatedException("error during encoding",
@@ -67,6 +70,22 @@ final class TranslatedException extends Exception {
         }
     }
 
+    /**
+     * Helper to test exception translation.
+     */
+    private static void maybeFailClinit() {
+        String className = VM.getSavedProperty("test.jvmci.TranslatedException.clinit.throw");
+        if (className != null) {
+            try {
+                throw (Throwable) Class.forName(className).getDeclaredConstructor().newInstance();
+            } catch (RuntimeException | Error e) {
+                throw e;
+            } catch (Throwable e) {
+                throw new InternalError(e);
+            }
+        }
+    }
+
     /**
      * Class name of exception that could not be instantiated.
      */

test/hotspot/jtreg/compiler/jvmci/TestUncaughtErrorInCompileMethod.java

@@ -24,7 +24,10 @@
 /*
  * @test
  * @summary Tests handling of an exception thrown by HotSpotJVMCIRuntime.compileMethod.
+ *          Requires a debug VM as it uses test.jvmci.compileMethodExceptionIsFatal
+ *          which is only read in a debug VM.
  * @requires vm.jvmci
+ * @requires vm.debug
  * @library /test/lib /
  * @modules jdk.internal.vm.ci/jdk.vm.ci.hotspot
  *          jdk.internal.vm.ci/jdk.vm.ci.code

test/hotspot/jtreg/compiler/jvmci/jdk.vm.ci.hotspot.test/src/jdk/vm/ci/hotspot/test/TestHotSpotJVMCIRuntime.java

@@ -32,6 +32,7 @@
  *          jdk.internal.vm.ci/jdk.vm.ci.common
  * @library /compiler/jvmci/jdk.vm.ci.hotspot.test/src
  *          /compiler/jvmci/jdk.vm.ci.code.test/src
+ * @library /test/lib
  * @run testng/othervm
  *      -XX:+UnlockExperimentalVMOptions -XX:+EnableJVMCI -XX:-UseJVMCICompiler
  *      jdk.vm.ci.hotspot.test.TestHotSpotJVMCIRuntime
@@ -53,6 +54,8 @@
 import jdk.vm.ci.meta.MetaAccessProvider;
 import jdk.vm.ci.meta.ResolvedJavaType;
 
+import jdk.test.lib.Platform;
+
 public class TestHotSpotJVMCIRuntime {
 
     @Test
@@ -157,6 +160,11 @@ public static void main(String[] args) {
 
     @Test
     public void jniEnomemTest() throws Exception {
+        if (!Platform.isDebugBuild()) {
+            // The test.jvmci.forceEnomemOnLibjvmciInit property is only
+            // read in a debug VM.
+            return;
+        }
         String[] names = {"translate", "attachCurrentThread", "registerNativeMethods"};
         for (String name : names) {
             ProcessBuilder pb = ProcessTools.createJavaProcessBuilder(