8369238: Allow virtual thread preemption on some common class initialization paths

src/hotspot/cpu/ppc/continuationFreezeThaw_ppc.inline.hpp

@@ -72,9 +72,16 @@ void FreezeBase::adjust_interpreted_frame_unextended_sp(frame& f) {
 }
 
 inline void FreezeBase::prepare_freeze_interpreted_top_frame(frame& f) {
-  // nothing to do
-  DEBUG_ONLY( intptr_t* lspp = (intptr_t*) &(f.get_ijava_state()->top_frame_sp); )
-  assert(*lspp == f.unextended_sp() - f.fp(), "should be " INTPTR_FORMAT " usp:" INTPTR_FORMAT " fp:" INTPTR_FORMAT, *lspp, p2i(f.unextended_sp()), p2i(f.fp()));
+  // Nothing to do. We don't save a last sp since we cannot use sp as esp.
+  // Instead the top frame is trimmed when making an i2i call. The original
+  // top_frame_sp is set when the frame is pushed (see generate_fixed_frame()).
+  // An interpreter top frame that was just thawed is resized to top_frame_sp by the
+  // resume adapter (see generate_cont_resume_interpreter_adapter()). So the assertion is
+  // false, if we freeze again right after thawing as we do when redoing a vm call wasn't
+  // successful.
+  assert(_thread->interp_redoing_vm_call() ||
+         ((intptr_t*)f.at_relative(ijava_idx(top_frame_sp)) == f.unextended_sp()),
+         "top_frame_sp:" PTR_FORMAT " usp:" PTR_FORMAT, f.at_relative(ijava_idx(top_frame_sp)), p2i(f.unextended_sp()));
 }
 
 inline void FreezeBase::relativize_interpreted_frame_metadata(const frame& f, const frame& hf) {
@@ -338,6 +345,8 @@ inline void FreezeBase::patch_pd_unused(intptr_t* sp) {
 }
 
 inline intptr_t* AnchorMark::anchor_mark_set_pd() {
+  // Nothing to do on PPC because the interpreter does not use SP as expression stack pointer.
+  // Instead there is a dedicated register R15_esp which is not affected by VM calls.
   return _top_frame.sp();
 }
 
@@ -574,8 +583,16 @@ inline intptr_t* ThawBase::push_cleanup_continuation() {
 }
 
 inline intptr_t* ThawBase::push_preempt_adapter() {
-  Unimplemented();
-  return nullptr;
+  frame enterSpecial = new_entry_frame();
+  frame::common_abi* enterSpecial_abi = (frame::common_abi*)enterSpecial.sp();
+
+  enterSpecial_abi->lr = (intptr_t)StubRoutines::cont_preempt_stub();
+
+  log_develop_trace(continuations, preempt)("push_preempt_adapter enterSpecial sp: " INTPTR_FORMAT " adapter pc: " INTPTR_FORMAT,
+                                            p2i(enterSpecial_abi),
+                                            p2i(StubRoutines::cont_preempt_stub()));
+
+  return enterSpecial.sp();
 }
 
 inline void ThawBase::patch_pd(frame& f, const frame& caller) {

src/hotspot/cpu/ppc/continuationHelper_ppc.inline.hpp

@@ -37,6 +37,9 @@ static inline void patch_return_pc_with_preempt_stub(frame& f) {
     // The target will check for preemption once it returns to the interpreter
     // or the native wrapper code and will manually jump to the preempt stub.
     JavaThread *thread = JavaThread::current();
+    DEBUG_ONLY(Method* m = f.is_interpreted_frame() ? f.interpreter_frame_method() : f.cb()->as_nmethod()->method();)
+    assert(m->is_object_wait0() || thread->interp_at_preemptable_vmcall_cnt() > 0,
+           "preemptable VM call not using call_VM_preemptable");
     thread->set_preempt_alternate_return(StubRoutines::cont_preempt_stub());
   }
 }

src/hotspot/cpu/ppc/frame_ppc.hpp

@@ -215,7 +215,7 @@
     uint64_t bcp;
     uint64_t esp;
     uint64_t mdx;
-    uint64_t top_frame_sp; // Maybe define parent_frame_abi and move there.
+    uint64_t top_frame_sp; // Original sp to be restored when returning from an i2i call
     uint64_t sender_sp;
     // Slots only needed for native calls. Maybe better to move elsewhere.
     uint64_t oop_tmp;

src/hotspot/cpu/ppc/interp_masm_ppc.hpp

@@ -49,12 +49,14 @@ class InterpreterMacroAssembler: public MacroAssembler {
   virtual void check_and_handle_popframe(Register scratch_reg);
   virtual void check_and_handle_earlyret(Register scratch_reg);
 
+  // Use for vthread preemption
   void call_VM_preemptable(Register oop_result, address entry_point, Register arg_1, bool check_exceptions = true);
+  void call_VM_preemptable(Register oop_result, address entry_point, Register arg_1, Register arg_2, bool check_exceptions = true);
   void restore_after_resume(Register fp);
   // R22 and R31 are preserved when a vthread gets preempted in the interpreter.
   // The interpreter already assumes that these registers are nonvolatile across native calls.
   bool nonvolatile_accross_vthread_preemtion(Register r) const {
-    return r->is_nonvolatile() && ((r == R22) || (r == R31));
+    return r->is_nonvolatile() && ((r == R24) || (r == R31));
   }
 
   // Base routine for all dispatches.

src/hotspot/cpu/ppc/interp_masm_ppc_64.cpp

@@ -108,6 +108,8 @@ void InterpreterMacroAssembler::dispatch_prolog(TosState state, int bcp_incr) {
 // own dispatch. The dispatch address in R24_dispatch_addr is used for the
 // dispatch.
 void InterpreterMacroAssembler::dispatch_epilog(TosState state, int bcp_incr) {
+  assert(nonvolatile_accross_vthread_preemtion(R24_dispatch_addr),
+         "Requirement of field accesses (e.g. putstatic)");
   if (bcp_incr) { addi(R14_bcp, R14_bcp, bcp_incr); }
   mtctr(R24_dispatch_addr);
   bcctr(bcondAlways, 0, bhintbhBCCTRisNotPredictable);
@@ -862,6 +864,9 @@ void InterpreterMacroAssembler::remove_activation(TosState state,
                                                   bool install_monitor_exception) {
   BLOCK_COMMENT("remove_activation {");
 
+  asm_assert_mem8_is_zero(in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread,
+                          "remove_activation: should not have alternate return address set");
+
   unlock_if_synchronized_method(state, throw_monitor_exception, install_monitor_exception);
 
   // The below poll is for the stack watermark barrier. It allows fixing up frames lazily,
@@ -2014,57 +2019,89 @@ void InterpreterMacroAssembler::call_VM(Register oop_result, address entry_point
 }
 
 void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address entry_point,
-                                        Register arg_1, bool check_exceptions) {
+                                                    Register arg_1,
+                                                    bool check_exceptions) {
   if (!Continuations::enabled()) {
     call_VM(oop_result, entry_point, arg_1, check_exceptions);
     return;
   }
+  call_VM_preemptable(oop_result, entry_point, arg_1, noreg /* arg_2 */, check_exceptions);
+}
+
+void InterpreterMacroAssembler::call_VM_preemptable(Register oop_result, address entry_point,
+                                                    Register arg_1, Register arg_2,
+                                                    bool check_exceptions) {
+  if (!Continuations::enabled()) {
+    call_VM(oop_result, entry_point, arg_1, arg_2, check_exceptions);
+    return;
+  }
 
   Label resume_pc, not_preempted;
+  Register tmp = R11_scratch1;
+  assert_different_registers(arg_1, tmp);
+  assert_different_registers(arg_2, tmp);
 
-  DEBUG_ONLY(ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread));
-  DEBUG_ONLY(cmpdi(CR0, R0, 0));
-  asm_assert_eq("Should not have alternate return address set");
+#ifdef ASSERT
+  asm_assert_mem8_is_zero(in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread,
+                          "Should not have alternate return address set");
+  // We check this counter in patch_return_pc_with_preempt_stub() during freeze.
+  lwa(tmp, in_bytes(JavaThread::interp_at_preemptable_vmcall_cnt_offset()), R16_thread);
+  addi(tmp, tmp, 1);
+  cmpwi(CR0, tmp, 0);
+  stw(tmp, in_bytes(JavaThread::interp_at_preemptable_vmcall_cnt_offset()), R16_thread);
+  asm_assert(gt, "call_VM_preemptable: should be > 0");
+#endif // ASSERT
 
   // Preserve 2 registers
-  assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R22), "");
+  assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R24), "");
   ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP
   std(R31, _ijava_state_neg(lresult), R3_ARG1);
-  std(R22, _ijava_state_neg(fresult), R3_ARG1);
+  std(R24, _ijava_state_neg(fresult), R3_ARG1);
 
   // We set resume_pc as last java pc. It will be saved if the vthread gets preempted.
   // Later execution will continue right there.
   mr_if_needed(R4_ARG2, arg_1);
+  assert(arg_2 != R4_ARG2, "smashed argument");
+  mr_if_needed(R5_ARG3, arg_2, true /* allow_noreg */);
   push_cont_fastpath();
-  call_VM(oop_result, entry_point, false /*check_exceptions*/, &resume_pc /* last_java_pc */);
+  call_VM(noreg /* oop_result */, entry_point, false /*check_exceptions*/, &resume_pc /* last_java_pc */);
   pop_cont_fastpath();
 
+#ifdef ASSERT
+  lwa(tmp, in_bytes(JavaThread::interp_at_preemptable_vmcall_cnt_offset()), R16_thread);
+  addi(tmp, tmp, -1);
+  cmpwi(CR0, tmp, 0);
+  stw(tmp, in_bytes(JavaThread::interp_at_preemptable_vmcall_cnt_offset()), R16_thread);
+  asm_assert(ge, "call_VM_preemptable: should be >= 0");
+#endif // ASSERT
+
   // Jump to handler if the call was preempted
   ld(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread);
   cmpdi(CR0, R0, 0);
   beq(CR0, not_preempted);
+  // Preempted. Frames are already frozen on heap.
   mtlr(R0);
   li(R0, 0);
   std(R0, in_bytes(JavaThread::preempt_alternate_return_offset()), R16_thread);
   blr();
 
   bind(resume_pc); // Location to resume execution
   restore_after_resume(noreg /* fp */);
+
   bind(not_preempted);
+  if (check_exceptions) {
+    check_and_forward_exception(R11_scratch1, R12_scratch2);
+  }
+  if (oop_result->is_valid()) {
+    get_vm_result_oop(oop_result);
+  }
 }
 
 void InterpreterMacroAssembler::restore_after_resume(Register fp) {
-  if (!Continuations::enabled()) return;
-
   const address resume_adapter = TemplateInterpreter::cont_resume_interpreter_adapter();
   add_const_optimized(R31, R29_TOC, MacroAssembler::offset_to_global_toc(resume_adapter));
   mtctr(R31);
   bctrl();
-  // Restore registers that are preserved across vthread preemption
-  assert(nonvolatile_accross_vthread_preemtion(R31) && nonvolatile_accross_vthread_preemtion(R22), "");
-  ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP
-  ld(R31, _ijava_state_neg(lresult), R3_ARG1);
-  ld(R22, _ijava_state_neg(fresult), R3_ARG1);
 #ifdef ASSERT
   // Assert FP is in R11_scratch1 (see generate_cont_resume_interpreter_adapter())
   {

src/hotspot/cpu/ppc/macroAssembler_ppc.cpp

@@ -757,10 +757,11 @@ void MacroAssembler::clobber_nonvolatile_registers() {
       R31
   };
   Register bad = regs[0];
-  load_const_optimized(bad, 0xbad0101babe11111);
+  load_const_optimized(bad, 0xbad0101babe00000);
   for (uint32_t i = 1; i < (sizeof(regs) / sizeof(Register)); i++) {
-    mr(regs[i], bad);
+    addi(regs[i], regs[0], regs[i]->encoding());
   }
+  addi(regs[0], regs[0], regs[0]->encoding());
   BLOCK_COMMENT("} clobber nonvolatile registers");
 }
 #endif // ASSERT
@@ -4341,21 +4342,36 @@ void MacroAssembler::multiply_to_len(Register x, Register xlen,
   bind(L_done);
 }   // multiply_to_len
 
-void MacroAssembler::asm_assert(bool check_equal, const char *msg) {
 #ifdef ASSERT
+void MacroAssembler::asm_assert(AsmAssertCond cond, const char *msg) {
   Label ok;
-  if (check_equal) {
+  switch (cond) {
+  case eq:
     beq(CR0, ok);
-  } else {
+    break;
+  case ne:
     bne(CR0, ok);
+    break;
+  case ge:
+    bge(CR0, ok);
+    break;
+  case gt:
+    bgt(CR0, ok);
+    break;
+  case lt:
+    blt(CR0, ok);
+    break;
+  case le:
+    ble(CR0, ok);
+    break;
+  default:
+    assert(false, "unknown cond:%d", cond);
   }
   stop(msg);
   bind(ok);
-#endif
 }
 
-#ifdef ASSERT
-void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_offset,
+void MacroAssembler::asm_assert_mems_zero(AsmAssertCond cond, int size, int mem_offset,
                                           Register mem_base, const char* msg) {
   switch (size) {
     case 4:
@@ -4369,7 +4385,7 @@ void MacroAssembler::asm_assert_mems_zero(bool check_equal, int size, int mem_of
     default:
       ShouldNotReachHere();
   }
-  asm_assert(check_equal, msg);
+  asm_assert(cond, msg);
 }
 #endif // ASSERT
 

src/hotspot/cpu/ppc/macroAssembler_ppc.hpp

@@ -68,7 +68,7 @@ class MacroAssembler: public Assembler {
   void store_sized_value(Register dst, RegisterOrConstant offs, Register base, size_t size_in_bytes);
 
   // Move register if destination register and target register are different
-  inline void mr_if_needed(Register rd, Register rs);
+  inline void mr_if_needed(Register rd, Register rs, bool allow_invalid = false);
   inline void fmr_if_needed(FloatRegister rd, FloatRegister rs);
   // This is dedicated for emitting scheduled mach nodes. For better
   // readability of the ad file I put it here.
@@ -942,21 +942,29 @@ class MacroAssembler: public Assembler {
   //
 
   // assert on cr0
-  void asm_assert(bool check_equal, const char* msg);
-  void asm_assert_eq(const char* msg) { asm_assert(true, msg); }
-  void asm_assert_ne(const char* msg) { asm_assert(false, msg); }
+  enum AsmAssertCond {
+    eq,
+    ne,
+    ge,
+    gt,
+    lt,
+    le
+  };
+  void asm_assert(AsmAssertCond cond, const char* msg) PRODUCT_RETURN;
+  void asm_assert_eq(const char* msg) { asm_assert(eq, msg); }
+  void asm_assert_ne(const char* msg) { asm_assert(ne, msg); }
 
  private:
-  void asm_assert_mems_zero(bool check_equal, int size, int mem_offset, Register mem_base,
+  void asm_assert_mems_zero(AsmAssertCond cond, int size, int mem_offset, Register mem_base,
                             const char* msg) NOT_DEBUG_RETURN;
 
  public:
 
   void asm_assert_mem8_is_zero(int mem_offset, Register mem_base, const char* msg) {
-    asm_assert_mems_zero(true,  8, mem_offset, mem_base, msg);
+    asm_assert_mems_zero(eq,  8, mem_offset, mem_base, msg);
   }
   void asm_assert_mem8_isnot_zero(int mem_offset, Register mem_base, const char* msg) {
-    asm_assert_mems_zero(false, 8, mem_offset, mem_base, msg);
+    asm_assert_mems_zero(ne, 8, mem_offset, mem_base, msg);
   }
 
   // Calls verify_oop. If UseCompressedOops is on, decodes the oop.

src/hotspot/cpu/ppc/macroAssembler_ppc.inline.hpp

@@ -65,7 +65,8 @@ inline void MacroAssembler::round_to(Register r, int modulus) {
 }
 
 // Move register if destination register and target register are different.
-inline void MacroAssembler::mr_if_needed(Register rd, Register rs) {
+inline void MacroAssembler::mr_if_needed(Register rd, Register rs, bool allow_noreg) {
+  if (allow_noreg && (rs == noreg)) return;
   if (rs != rd) mr(rd, rs);
 }
 inline void MacroAssembler::fmr_if_needed(FloatRegister rd, FloatRegister rs) {

src/hotspot/cpu/ppc/smallRegisterMap_ppc.inline.hpp

@@ -61,22 +61,22 @@ class SmallRegisterMapType {
 
   JavaThread* thread() const {
   #ifndef ASSERT
-    guarantee (false, "");
+    guarantee (false, "unreachable");
   #endif
     return nullptr;
   }
 
   bool update_map()    const { return false; }
   bool walk_cont()     const { return false; }
-  bool include_argument_oops() const { return false; }
+  bool include_argument_oops() const { return IncludeArgs; }
   void set_include_argument_oops(bool f)  {}
   bool in_cont()       const { return false; }
   stackChunkHandle stack_chunk() const { return stackChunkHandle(); }
 
 #ifdef ASSERT
   bool should_skip_missing() const  { return false; }
   VMReg find_register_spilled_here(void* p, intptr_t* sp) {
-    Unimplemented();
+    assert(false, "Shouldn't reach here! p:" PTR_FORMAT " sp:" PTR_FORMAT, p2i(p), p2i(p));
     return nullptr;
   }
   void print() const { print_on(tty); }

src/hotspot/cpu/ppc/stackChunkFrameStream_ppc.inline.hpp

@@ -180,14 +180,10 @@ template <typename RegisterMapT>
 inline int StackChunkFrameStream<frame_kind>::interpreter_frame_num_oops(RegisterMapT* map) const {
   assert_is_interpreted_and_frame_type_mixed();
   ResourceMark rm;
-  InterpreterOopMap mask;
   frame f = to_frame();
-  f.interpreted_frame_oop_map(&mask);
-  return  mask.num_oops()
-          + 1 // for the mirror oop
-          + (f.interpreter_frame_method()->is_native() ? 1 : 0) // temp oop slot
-          + pointer_delta_as_int((intptr_t*)f.interpreter_frame_monitor_begin(),
-                                 (intptr_t*)f.interpreter_frame_monitor_end())/BasicObjectLock::size();
+  InterpreterOopCount closure;
+  f.oops_interpreted_do(&closure, map);
+  return closure.count();
 }
 
 template<>

src/hotspot/cpu/ppc/templateInterpreterGenerator_ppc.cpp

@@ -702,6 +702,11 @@ address TemplateInterpreterGenerator::generate_cont_resume_interpreter_adapter()
 
   __ load_const_optimized(R25_templateTableBase, (address)Interpreter::dispatch_table((TosState)0), R12_scratch2);
   __ restore_interpreter_state(R11_scratch1, false, true /*restore_top_frame_sp*/);
+  // Restore registers that are preserved across vthread preemption
+  assert(__ nonvolatile_accross_vthread_preemtion(R31) && __ nonvolatile_accross_vthread_preemtion(R24), "");
+  __ ld(R3_ARG1, _abi0(callers_sp), R1_SP); // load FP
+  __ ld(R31, _ijava_state_neg(lresult), R3_ARG1);
+  __ ld(R24, _ijava_state_neg(fresult), R3_ARG1);
   __ blr();
 
   return start;
@@ -1249,7 +1254,7 @@ address TemplateInterpreterGenerator::generate_native_entry(bool synchronized) {
   const Register pending_exception    = R0;
   const Register result_handler_addr  = R31;
   const Register native_method_fd     = R12_scratch2; // preferred in MacroAssembler::branch_to
-  const Register access_flags         = R22_tmp2;
+  const Register access_flags         = R24_tmp4;
   const Register active_handles       = R11_scratch1; // R26_monitor saved to state.
   const Register sync_state           = R12_scratch2;
   const Register sync_state_addr      = sync_state;   // Address is dead after use.