[vm] Fix JumpToFrame execution state transition

Instead of handling FFI related execution state and safepoint
in assembly handle it in runtime code.

The transition needs to be done before JumpToFrame unwinds
stack because unwinding destroys exit frame and this can't
be done at safepoint as GC might be traversing the stack.

An incorrect order of operation was manifesting as crashes in
GC when one isolate in a group was encountering a lot of
exceptions thrown from an FFI call and another isolate is
triggering GCs.

To catch this in the future added a bit of validation to
ExitSafepoint runtime call which triggers when --use-slow-path
is enabled. Though after refactoring this code does not
trigger this code path anymore because it was completely
removed - but it is better than nothing.

This CL also removes a lot of unnecessary complexity which
was associated with handling this transition in the stub
itself.

TEST=ffi/vmspecific_handle_test

Bug: b/408377905
Cq-Include-Trybots: dart/try:vm-aot-android-release-arm64c-try,vm-aot-android-release-arm_x64-try,vm-aot-asan-linux-release-x64-try,vm-aot-linux-debug-x64-try,vm-aot-linux-debug-x64c-try,vm-aot-mac-release-arm64-try,vm-aot-msan-linux-release-x64-try,vm-aot-obfuscate-linux-release-x64-try,vm-aot-optimization-level-linux-release-x64-try,vm-aot-tsan-linux-release-x64-try,vm-aot-ubsan-linux-release-x64-try,vm-aot-win-debug-x64-try,vm-aot-win-debug-x64c-try,vm-appjit-linux-debug-x64-try,vm-asan-linux-release-arm64-try,vm-asan-linux-release-x64-try,vm-checked-mac-release-arm64-try,vm-eager-optimization-linux-release-ia32-try,vm-eager-optimization-linux-release-x64-try,vm-ffi-android-debug-arm-try,vm-ffi-android-debug-arm64c-try,vm-ffi-mac-debug-simarm64_arm64-try,vm-ffi-qemu-linux-release-arm-try,vm-ffi-qemu-linux-release-riscv64-try,vm-fuchsia-release-x64-try,vm-linux-debug-ia32-try,vm-linux-debug-x64-try,vm-linux-debug-x64c-try,vm-mac-debug-arm64-try,vm-mac-debug-x64-try,vm-msan-linux-release-arm64-try,vm-msan-linux-release-x64-try,vm-reload-linux-debug-x64-try,vm-reload-rollback-linux-debug-x64-try,vm-tsan-linux-release-arm64-try,vm-tsan-linux-release-x64-try,vm-ubsan-linux-release-arm64-try,vm-ubsan-linux-release-x64-try,vm-win-debug-x64-try,vm-win-debug-x64c-try,vm-win-release-ia32-try
Change-Id: Ia073cb6bb9e1b5a0ea8514c7e048cee6019b84d6
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/420324
Commit-Queue: Slava Egorov <vegorov@google.com>
Reviewed-by: Daco Harkes <dacoharkes@google.com>

Author: mraleph

runtime/vm/compiler/assembler/assembler_arm.cc

@@ -634,9 +634,7 @@ void Assembler::TransitionGeneratedToNative(Register destination_address,
   }
 }
 
-void Assembler::ExitFullSafepoint(Register tmp1,
-                                  Register tmp2,
-                                  bool ignore_unwind_in_progress) {
+void Assembler::ExitFullSafepoint(Register tmp1, Register tmp2) {
   Register addr = tmp1;
   Register state = tmp2;
 
@@ -664,14 +662,7 @@ void Assembler::ExitFullSafepoint(Register tmp1,
   }
 
   Bind(&slow_path);
-  if (ignore_unwind_in_progress) {
-    ldr(TMP,
-        Address(THR,
-                target::Thread::
-                    exit_safepoint_ignore_unwind_in_progress_stub_offset()));
-  } else {
-    ldr(TMP, Address(THR, target::Thread::exit_safepoint_stub_offset()));
-  }
+  ldr(TMP, Address(THR, target::Thread::exit_safepoint_stub_offset()));
   ldr(TMP, FieldAddress(TMP, target::Code::entry_point_offset()));
   blx(TMP);
 
@@ -681,13 +672,10 @@ void Assembler::ExitFullSafepoint(Register tmp1,
 void Assembler::TransitionNativeToGenerated(Register addr,
                                             Register state,
                                             bool exit_safepoint,
-                                            bool ignore_unwind_in_progress,
                                             bool set_tag) {
   if (exit_safepoint) {
-    ExitFullSafepoint(addr, state, ignore_unwind_in_progress);
+    ExitFullSafepoint(addr, state);
   } else {
-    // flag only makes sense if we are leaving safepoint
-    ASSERT(!ignore_unwind_in_progress);
 #if defined(DEBUG)
     // Ensure we've already left the safepoint.
     ASSERT(target::Thread::native_safepoint_state_acquired() != 0);

runtime/vm/compiler/assembler/assembler_arm.h

@@ -621,14 +621,11 @@ class Assembler : public AssemblerBase {
   void TransitionNativeToGenerated(Register scratch0,
                                    Register scratch1,
                                    bool exit_safepoint,
-                                   bool ignore_unwind_in_progress = false,
                                    bool set_tag = true);
   void VerifyInGenerated(Register scratch);
   void VerifyNotInGenerated(Register scratch);
   void EnterFullSafepoint(Register scratch0, Register scratch1);
-  void ExitFullSafepoint(Register scratch0,
-                         Register scratch1,
-                         bool ignore_unwind_in_progress);
+  void ExitFullSafepoint(Register scratch0, Register scratch1);
 
   // Miscellaneous instructions.
   void clrex();

runtime/vm/compiler/assembler/assembler_arm64.cc

@@ -1619,8 +1619,7 @@ void Assembler::TransitionGeneratedToNative(Register destination,
   }
 }
 
-void Assembler::ExitFullSafepoint(Register state,
-                                  bool ignore_unwind_in_progress) {
+void Assembler::ExitFullSafepoint(Register state) {
   // We generate the same number of instructions whether or not the slow-path is
   // forced, for consistency with EnterFullSafepoint.
   // For TSAN, we always go to the runtime so TSAN is aware of the acquire
@@ -1650,14 +1649,7 @@ void Assembler::ExitFullSafepoint(Register state,
   }
 
   Bind(&slow_path);
-  if (ignore_unwind_in_progress) {
-    ldr(addr,
-        Address(THR,
-                target::Thread::
-                    exit_safepoint_ignore_unwind_in_progress_stub_offset()));
-  } else {
-    ldr(addr, Address(THR, target::Thread::exit_safepoint_stub_offset()));
-  }
+  ldr(addr, Address(THR, target::Thread::exit_safepoint_stub_offset()));
   ldr(addr, FieldAddress(addr, target::Code::entry_point_offset()));
   blr(addr);
 
@@ -1666,13 +1658,10 @@ void Assembler::ExitFullSafepoint(Register state,
 
 void Assembler::TransitionNativeToGenerated(Register state,
                                             bool exit_safepoint,
-                                            bool ignore_unwind_in_progress,
                                             bool set_tag) {
   if (exit_safepoint) {
-    ExitFullSafepoint(state, ignore_unwind_in_progress);
+    ExitFullSafepoint(state);
   } else {
-    // flag only makes sense if we are leaving safepoint
-    ASSERT(!ignore_unwind_in_progress);
 #if defined(DEBUG)
     // Ensure we've already left the safepoint.
     ASSERT(target::Thread::native_safepoint_state_acquired() != 0);

runtime/vm/compiler/assembler/assembler_arm64.h

@@ -2111,12 +2111,11 @@ class Assembler : public AssemblerBase {
                                    bool enter_safepoint);
   void TransitionNativeToGenerated(Register scratch,
                                    bool exit_safepoint,
-                                   bool ignore_unwind_in_progress = false,
                                    bool set_tag = true);
   void VerifyInGenerated(Register scratch);
   void VerifyNotInGenerated(Register scratch);
   void EnterFullSafepoint(Register scratch);
-  void ExitFullSafepoint(Register scratch, bool ignore_unwind_in_progress);
+  void ExitFullSafepoint(Register scratch);
 
   void CheckCodePointer();
   void RestoreCodePointer();

runtime/vm/compiler/assembler/assembler_ia32.cc

@@ -2537,8 +2537,7 @@ void Assembler::TransitionGeneratedToNative(Register destination_address,
   }
 }
 
-void Assembler::ExitFullSafepoint(Register scratch,
-                                  bool ignore_unwind_in_progress) {
+void Assembler::ExitFullSafepoint(Register scratch) {
   ASSERT(scratch != EAX);
   // We generate the same number of instructions whether or not the slow-path is
   // forced, for consistency with EnterFullSafepoint.
@@ -2563,14 +2562,7 @@ void Assembler::ExitFullSafepoint(Register scratch,
   }
 
   Bind(&slow_path);
-  if (ignore_unwind_in_progress) {
-    movl(scratch,
-         Address(THR,
-                 target::Thread::
-                     exit_safepoint_ignore_unwind_in_progress_stub_offset()));
-  } else {
-    movl(scratch, Address(THR, target::Thread::exit_safepoint_stub_offset()));
-  }
+  movl(scratch, Address(THR, target::Thread::exit_safepoint_stub_offset()));
   movl(scratch, FieldAddress(scratch, target::Code::entry_point_offset()));
   call(scratch);
 
@@ -2579,13 +2571,10 @@ void Assembler::ExitFullSafepoint(Register scratch,
 
 void Assembler::TransitionNativeToGenerated(Register scratch,
                                             bool exit_safepoint,
-                                            bool ignore_unwind_in_progress,
                                             bool set_tag) {
   if (exit_safepoint) {
-    ExitFullSafepoint(scratch, ignore_unwind_in_progress);
+    ExitFullSafepoint(scratch);
   } else {
-    // flag only makes sense if we are leaving safepoint
-    ASSERT(!ignore_unwind_in_progress);
 #if defined(DEBUG)
     // Ensure we've already left the safepoint.
     movl(scratch, Address(THR, target::Thread::safepoint_state_offset()));

runtime/vm/compiler/assembler/assembler_ia32.h

@@ -901,10 +901,9 @@ class Assembler : public AssemblerBase {
                                    bool enter_safepoint);
   void TransitionNativeToGenerated(Register scratch,
                                    bool exit_safepoint,
-                                   bool ignore_unwind_in_progress = false,
                                    bool set_tag = true);
   void EnterFullSafepoint(Register scratch);
-  void ExitFullSafepoint(Register scratch, bool ignore_unwind_in_progress);
+  void ExitFullSafepoint(Register scratch);
 
   // For non-leaf runtime calls. For leaf runtime calls, use LeafRuntimeScope,
   void CallRuntime(const RuntimeEntry& entry, intptr_t argument_count);

runtime/vm/compiler/assembler/assembler_riscv.cc

@@ -4399,13 +4399,10 @@ void Assembler::TransitionGeneratedToNative(Register destination,
 
 void Assembler::TransitionNativeToGenerated(Register state,
                                             bool exit_safepoint,
-                                            bool ignore_unwind_in_progress,
                                             bool set_tag) {
   if (exit_safepoint) {
-    ExitFullSafepoint(state, ignore_unwind_in_progress);
+    ExitFullSafepoint(state);
   } else {
-    // flag only makes sense if we are leaving safepoint
-    ASSERT(!ignore_unwind_in_progress);
 #if defined(DEBUG)
     // Ensure we've already left the safepoint.
     ASSERT(target::Thread::native_safepoint_state_acquired() != 0);
@@ -4495,8 +4492,7 @@ void Assembler::EnterFullSafepoint(Register state) {
   Bind(&done);
 }
 
-void Assembler::ExitFullSafepoint(Register state,
-                                  bool ignore_unwind_in_progress) {
+void Assembler::ExitFullSafepoint(Register state) {
   // We generate the same number of instructions whether or not the slow-path is
   // forced, for consistency with EnterFullSafepoint.
   // For TSAN, we always go to the runtime so TSAN is aware of the acquire
@@ -4524,14 +4520,7 @@ void Assembler::ExitFullSafepoint(Register state,
   }
 
   Bind(&slow_path);
-  if (ignore_unwind_in_progress) {
-    lx(addr,
-       Address(THR,
-               target::Thread::
-                   exit_safepoint_ignore_unwind_in_progress_stub_offset()));
-  } else {
-    lx(addr, Address(THR, target::Thread::exit_safepoint_stub_offset()));
-  }
+  lx(addr, Address(THR, target::Thread::exit_safepoint_stub_offset()));
   lx(addr, FieldAddress(addr, target::Code::entry_point_offset()));
   jalr(addr);
 

runtime/vm/compiler/assembler/assembler_riscv.h

@@ -1452,12 +1452,11 @@ class Assembler : public MicroAssembler {
                                    bool enter_safepoint);
   void TransitionNativeToGenerated(Register scratch,
                                    bool exit_safepoint,
-                                   bool ignore_unwind_in_progress = false,
                                    bool set_tag = true);
   void VerifyInGenerated(Register scratch);
   void VerifyNotInGenerated(Register scratch);
   void EnterFullSafepoint(Register scratch);
-  void ExitFullSafepoint(Register scratch, bool ignore_unwind_in_progress);
+  void ExitFullSafepoint(Register scratch);
 
   void CheckFpSpDist(intptr_t fp_sp_dist);
 

runtime/vm/compiler/assembler/assembler_x64.cc

@@ -216,7 +216,7 @@ void Assembler::TransitionGeneratedToNative(Register destination_address,
   }
 }
 
-void Assembler::ExitFullSafepoint(bool ignore_unwind_in_progress) {
+void Assembler::ExitFullSafepoint() {
   // We generate the same number of instructions whether or not the slow-path is
   // forced, for consistency with EnterFullSafepoint.
   // For TSAN, we always go to the runtime so TSAN is aware of the acquire
@@ -243,14 +243,7 @@ void Assembler::ExitFullSafepoint(bool ignore_unwind_in_progress) {
   }
 
   Bind(&slow_path);
-  if (ignore_unwind_in_progress) {
-    movq(TMP,
-         Address(THR,
-                 target::Thread::
-                     exit_safepoint_ignore_unwind_in_progress_stub_offset()));
-  } else {
-    movq(TMP, Address(THR, target::Thread::exit_safepoint_stub_offset()));
-  }
+  movq(TMP, Address(THR, target::Thread::exit_safepoint_stub_offset()));
   movq(TMP, FieldAddress(TMP, target::Code::entry_point_offset()));
 
   // Use call instead of CallCFunction to avoid having to clean up shadow space
@@ -261,14 +254,10 @@ void Assembler::ExitFullSafepoint(bool ignore_unwind_in_progress) {
   Bind(&done);
 }
 
-void Assembler::TransitionNativeToGenerated(bool exit_safepoint,
-                                            bool ignore_unwind_in_progress,
-                                            bool set_tag) {
+void Assembler::TransitionNativeToGenerated(bool exit_safepoint, bool set_tag) {
   if (exit_safepoint) {
-    ExitFullSafepoint(ignore_unwind_in_progress);
+    ExitFullSafepoint();
   } else {
-    // flag only makes sense if we are leaving safepoint
-    ASSERT(!ignore_unwind_in_progress);
 #if defined(DEBUG)
     // Ensure we've already left the safepoint.
     movq(TMP, Address(THR, target::Thread::safepoint_state_offset()));

runtime/vm/compiler/assembler/assembler_x64.h

@@ -319,14 +319,12 @@ class Assembler : public AssemblerBase {
   void setcc(Condition condition, ByteRegister dst);
 
   void EnterFullSafepoint();
-  void ExitFullSafepoint(bool ignore_unwind_in_progress);
+  void ExitFullSafepoint();
   void TransitionGeneratedToNative(Register destination_address,
                                    Register new_exit_frame,
                                    Register new_exit_through_ffi,
                                    bool enter_safepoint);
-  void TransitionNativeToGenerated(bool exit_safepoint,
-                                   bool ignore_unwind_in_progress = false,
-                                   bool set_tag = true);
+  void TransitionNativeToGenerated(bool exit_safepoint, bool set_tag = true);
   void VerifyInGenerated(Register scratch);
   void VerifyNotInGenerated(Register scratch);