8330146: assert(!_thread->is_in_any_VTMS_transition()) failed

Backport-of: c4ff58b9bcfd08eae0623a648a837e08f25b3f9b

Author: Jiawei Tang

src/hotspot/share/prims/jvmtiExport.cpp

@@ -929,7 +929,9 @@ class JvmtiClassFileLoadHookPoster : public StackObj {
     _cached_class_file_ptr = cache_ptr;
     _has_been_modified = false;
 
-    assert(!_thread->is_in_any_VTMS_transition(), "CFLH events are not allowed in any VTMS transition");
+    if (_thread->is_in_any_VTMS_transition()) {
+      return; // no events should be posted if thread is in any VTMS transition
+    }
     _state = JvmtiExport::get_jvmti_thread_state(_thread);
     if (_state != nullptr) {
       _class_being_redefined = _state->get_class_being_redefined();
@@ -1363,10 +1365,9 @@ void JvmtiExport::post_class_load(JavaThread *thread, Klass* klass) {
   if (state == nullptr) {
     return;
   }
-  if (thread->is_in_tmp_VTMS_transition()) {
-    return; // skip ClassLoad events in tmp VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
-  assert(!thread->is_in_any_VTMS_transition(), "class load events are not allowed in any VTMS transition");
 
   EVT_TRIG_TRACE(JVMTI_EVENT_CLASS_LOAD, ("[%s] Trg Class Load triggered",
                       JvmtiTrace::safe_get_thread_name(thread)));
@@ -1401,10 +1402,9 @@ void JvmtiExport::post_class_prepare(JavaThread *thread, Klass* klass) {
   if (state == nullptr) {
     return;
   }
-  if (thread->is_in_tmp_VTMS_transition()) {
-    return; // skip ClassPrepare events in tmp VTMS transition
+  if (thread->is_in_any_VTMS_transition()) {
+    return; // no events should be posted if thread is in any VTMS transition
   }
-  assert(!thread->is_in_any_VTMS_transition(), "class prepare events are not allowed in any VTMS transition");
 
   EVT_TRIG_TRACE(JVMTI_EVENT_CLASS_PREPARE, ("[%s] Trg Class Prepare triggered",
                       JvmtiTrace::safe_get_thread_name(thread)));