Deopt through patching post-call-nop into trap.

Signal handler returns to deopt stub.

src/hotspot/cpu/x86/nativeInst_x86.cpp

@@ -647,6 +647,17 @@ address NativeGeneralJump::jump_destination() const {
     return addr_at(0) + length + sbyte_at(offset);
 }
 
+void NativePostCallNop::make_deopt() {
+  NativeDeoptInstruction::insert(addr_at(0));
+  // makes the first 2 bytes into UD, the rest is thrash. Make NOPs to help debugging.
+  set_char_at(2, 0x90);
+  set_char_at(3, 0x90);
+  set_char_at(4, 0x90);
+  set_char_at(5, 0x90);
+  set_char_at(6, 0x90);
+  set_char_at(7, 0x90);
+}
+
 void NativePostCallNop::patch(jint diff) {
   assert(diff != 0, "must be");
   int32_t *code_pos = (int32_t *) addr_at(displacement_offset);
@@ -670,3 +681,13 @@ void NativePostCallNop::patch_int2(uint32_t int2) {
   wrote(12);
 }
 #endif
+
+void NativeDeoptInstruction::verify() {
+}
+
+// Inserts an undefined instruction at a given pc
+void NativeDeoptInstruction::insert(address code_pos) {
+  *code_pos = instruction_prefix;
+  *(code_pos+1) = instruction_code;
+  ICache::invalidate_range(code_pos, instruction_size);
+}

src/hotspot/cpu/x86/nativeInst_x86.hpp

@@ -753,6 +753,7 @@ class NativePostCallNop: public NativeInstruction {
   }
   int displacement() const { return (jint) int_at(displacement_offset); }
   void patch(jint diff);
+  void make_deopt();
 
 #ifdef CONT_DOUBLE_NOP
   bool check1() const { return (int_at(0) & 0xffffff) == 0x841f0f && (int_at(8) & 0xffffff) == 0x841f0f; }
@@ -794,4 +795,27 @@ inline NativePostCallNop* nativePostCallNop_unsafe_at(address address) {
   return nop;
 }
 
+class NativeDeoptInstruction: public NativeInstruction {
+ public:
+  enum Intel_specific_constants {
+    instruction_prefix          = 0x0F,
+    instruction_code            = 0xFF,
+    instruction_size            =    2,
+    instruction_offset          =    0,
+  };
+
+  address instruction_address() const       { return addr_at(instruction_offset); }
+  address next_instruction_address() const  { return addr_at(instruction_size); }
+
+  void  verify();
+
+  static bool is_deopt_at(address instr) {
+    return ((*instr) & 0xFF) == NativeDeoptInstruction::instruction_prefix && 
+      ((*(instr+1)) & 0xFF) == NativeDeoptInstruction::instruction_code;
+  }
+
+  // MT-safe patching
+  static void insert(address code_pos);
+};
+
 #endif // CPU_X86_NATIVEINST_X86_HPP

src/hotspot/cpu/zero/nativeInst_zero.hpp

@@ -41,6 +41,7 @@
 // - - NativeReturnX (return with argument)
 // - - NativePushConst
 // - - NativeTstRegMem
+// - - NativeDeoptInstruction
 
 // The base class for different kinds of native instruction abstractions.
 // Provides the primitive operations to manipulate code relative to this.

src/hotspot/os/posix/signals_posix.cpp

@@ -25,6 +25,9 @@
 #include "precompiled.hpp"
 
 #include "jvm.h"
+#include "code/codeCache.hpp"
+#include "code/compiledMethod.hpp"
+#include "code/nativeInst.hpp"
 #include "logging/log.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/globals.hpp"
@@ -641,6 +644,26 @@ int JVM_HANDLE_XXX_SIGNAL(int sig, siginfo_t* info,
     signal_was_handled = true; // unconditionally.
   }
 
+  // Check for UD trap caused by NOP patching.
+  // If it is, patch return address to be deopt handler.
+  if (!signal_was_handled) {
+    address pc = os::Posix::ucontext_get_pc(uc);
+    if (NativeDeoptInstruction::is_deopt_at(pc)) {
+      CodeBlob* cb = CodeCache::find_blob_unsafe(pc);
+      if (cb != NULL && cb->is_compiled()) {
+        CompiledMethod* cm = cb->as_compiled_method();
+        frame fr = os::fetch_frame_from_context(uc);
+        address deopt = cm->is_method_handle_return(pc) ?
+          cm->deopt_mh_handler_begin() :
+          cm->deopt_handler_begin();
+        assert(cm->insts_contains_inclusive(pc), "");
+        cm->set_original_pc(&fr, pc);
+        os::Posix::ucontext_set_pc(uc, deopt);
+        signal_was_handled = true;
+      }
+    }
+  }
+
   // Call platform dependent signal handler.
   if (!signal_was_handled) {
     JavaThread* const jt = (t != NULL && t->is_Java_thread()) ? (JavaThread*) t : NULL;

src/hotspot/share/code/codeCache.cpp

@@ -1212,13 +1212,23 @@ int CodeCache::mark_for_deoptimization(Method* dependee) {
   return number_of_marked_CodeBlobs;
 }
 
-void CodeCache::make_marked_nmethods_not_entrant() {
+void CodeCache::make_marked_nmethods_not_entrant(GrowableArray<CompiledMethod*>* marked) {
   assert_locked_or_safepoint(CodeCache_lock);
   CompiledMethodIterator iter(CompiledMethodIterator::only_alive_and_not_unloading);
   while(iter.next()) {
     CompiledMethod* nm = iter.method();
     if (nm->is_marked_for_deoptimization()) {
       nm->make_not_entrant();
+      marked->append(nm);
+    }
+  }
+}
+
+void CodeCache::make_marked_nmethods_deoptimized(GrowableArray<CompiledMethod*>* marked) {
+  for (int i = 0; i < marked->length(); i++) {
+    CompiledMethod* nm = marked->at(i);
+    if (nm->is_marked_for_deoptimization()) {
+      nm->make_deoptimized();
     }
   }
 }

src/hotspot/share/code/codeCache.hpp

@@ -276,7 +276,8 @@ class CodeCache : AllStatic {
  public:
   static void mark_all_nmethods_for_deoptimization();
   static int  mark_for_deoptimization(Method* dependee);
-  static void make_marked_nmethods_not_entrant();
+  static void make_marked_nmethods_not_entrant(GrowableArray<CompiledMethod*>* marked);
+  static void make_marked_nmethods_deoptimized(GrowableArray<CompiledMethod*>* marked);
 
   // Flushing and deoptimization
   static void flush_dependents_on(InstanceKlass* dependee);

src/hotspot/share/code/compiledIC.hpp

@@ -260,7 +260,7 @@ class CompiledIC: public ResourceObj {
 
   bool is_icholder_call() const;
 
-  address end_of_call() { return  _call->return_address(); }
+  address end_of_call() const { return  _call->return_address(); }
 
   // MT-safe patching of inline caches. Note: Only safe to call is_xxx when holding the CompiledIC_ock
   // so you are guaranteed that no patching takes place. The same goes for verify.
@@ -383,6 +383,7 @@ class CompiledStaticCall : public ResourceObj {
   virtual bool is_call_to_interpreted() const = 0;
 
   virtual address instruction_address() const = 0;
+  virtual address end_of_call() const = 0;
 protected:
   virtual address resolve_call_stub() const = 0;
   virtual void set_destination_mt_safe(address dest) = 0;
@@ -434,6 +435,7 @@ class CompiledDirectStaticCall : public CompiledStaticCall {
 
   // Delegation
   address destination() const { return _call->destination(); }
+  address end_of_call() const { return _call->return_address(); }
 
   // State
   virtual bool is_call_to_interpreted() const;

src/hotspot/share/code/compiledMethod.cpp

@@ -44,6 +44,7 @@
 #include "prims/methodHandles.hpp"
 #include "runtime/atomic.hpp"
 #include "runtime/deoptimization.hpp"
+#include "runtime/frame.inline.hpp"
 #include "runtime/jniHandles.inline.hpp"
 #include "runtime/handles.inline.hpp"
 #include "runtime/mutexLocker.hpp"
@@ -627,6 +628,10 @@ void CompiledMethod::cleanup_inline_caches(bool clean_all) {
   }
 }
 
+address* CompiledMethod::orig_pc_addr(const frame* fr) { 
+  return (address*) ((address)fr->unextended_sp() + orig_pc_offset()); 
+}
+
 // Called to clean up after class unloading for live nmethods and from the sweeper
 // for all methods.
 bool CompiledMethod::cleanup_inline_caches_impl(bool unloading_occurred, bool clean_all) {

src/hotspot/share/code/compiledMethod.hpp

@@ -248,6 +248,8 @@ class CompiledMethod : public CodeBlob {
   bool  is_marked_for_deoptimization() const { return _mark_for_deoptimization_status != not_marked; }
   void  mark_for_deoptimization(bool inc_recompile_counts = true);
 
+  virtual void  make_deoptimized() { assert(false, "not supported"); };
+
   bool update_recompile_counts() const {
     // Update recompile counts when either the update is explicitly requested (deoptimize)
     // or the nmethod is not marked for deoptimization at all (not_marked).
@@ -337,7 +339,7 @@ class CompiledMethod : public CodeBlob {
 
   virtual int orig_pc_offset() = 0;
 private:
-  address* orig_pc_addr(const frame* fr) { return (address*) ((address)fr->unextended_sp() + orig_pc_offset()); };
+  address* orig_pc_addr(const frame* fr);
 
 public:
   virtual bool can_convert_to_zombie() = 0;

src/hotspot/share/code/nmethod.cpp

@@ -1121,6 +1121,45 @@ void nmethod::fix_oop_relocations(address begin, address end, bool initialize_im
 }
 
 
+void nmethod::make_deoptimized() {
+  CompiledICLocker ml(this);
+  assert(CompiledICLocker::is_safe(this), "mt unsafe call");
+  ResourceMark rm;
+  RelocIterator iter(this, oops_reloc_begin());
+
+  while(iter.next()) {
+
+    switch(iter.type()) {
+      case relocInfo::virtual_call_type:
+      case relocInfo::opt_virtual_call_type: {
+        CompiledIC *ic = CompiledIC_at(&iter);
+        address pc = ic->end_of_call();
+        NativePostCallNop* nop = nativePostCallNop_at(pc);
+        if (nop != NULL) {
+          nop->make_deopt();
+        }
+        assert(NativeDeoptInstruction::is_deopt_at(pc), "check");
+        break;
+      }
+      case relocInfo::static_call_type: {
+        CompiledStaticCall *csc = compiledStaticCall_at(iter.reloc());
+        address pc = csc->end_of_call();
+        NativePostCallNop* nop = nativePostCallNop_at(pc);
+        //tty->print_cr(" - static pc %p", pc);
+        if (nop != NULL) {
+          nop->make_deopt();
+        }
+        // We can't assert here, there are some calls to stubs / runtime
+        // that have reloc data and doesn't have a post call NOP.
+        //assert(NativeDeoptInstruction::is_deopt_at(pc), "check");
+        break;
+      }
+      default:
+        break;
+    }
+  }
+}
+
 void nmethod::verify_clean_inline_caches() {
   assert(CompiledICLocker::is_safe(this), "mt unsafe call");
 

src/hotspot/share/code/nmethod.hpp

@@ -769,6 +769,8 @@ class nmethod : public CompiledMethod {
   virtual CompiledStaticCall* compiledStaticCall_at(Relocation* call_site) const;
   virtual CompiledStaticCall* compiledStaticCall_at(address addr) const;
   virtual CompiledStaticCall* compiledStaticCall_before(address addr) const;
+
+  virtual void  make_deoptimized();
 };
 
 // Locks an nmethod so its code will not get removed and it will not

src/hotspot/share/oops/instanceStackChunkKlass.inline.hpp

@@ -533,11 +533,15 @@ template <bool mixed>
 inline void StackChunkFrameStream<mixed>::get_oopmap(address pc, int oopmap_slot) const {
   assert (cb() != nullptr, "");
   assert (!is_compiled() || !cb()->as_compiled_method()->is_deopt_pc(pc), "oopmap_slot: %d", oopmap_slot);
-  assert (oopmap_slot >= 0, "");
-  assert (cb()->oop_map_for_slot(oopmap_slot, pc) != nullptr, "");
-  assert (cb()->oop_map_for_slot(oopmap_slot, pc) == cb()->oop_map_for_return_address(pc), "");
+  if (oopmap_slot >= 0) {
+    assert (oopmap_slot >= 0, "");
+    assert (cb()->oop_map_for_slot(oopmap_slot, pc) != nullptr, "");
+    assert (cb()->oop_map_for_slot(oopmap_slot, pc) == cb()->oop_map_for_return_address(pc), "");
 
-  _oopmap = cb()->oop_map_for_slot(oopmap_slot, pc);
+    _oopmap = cb()->oop_map_for_slot(oopmap_slot, pc);
+  } else {
+    _oopmap = cb()->oop_map_for_return_address(pc);
+  }
   assert (_oopmap != nullptr, "");
 }
 
@@ -619,8 +623,11 @@ void StackChunkFrameStream<mixed>::handle_deopted() const {
   address pc1 = pc();
   int oopmap_slot = CodeCache::find_oopmap_slot_fast(pc1);
   if (UNLIKELY(oopmap_slot < 0)) { // we could have marked frames for deoptimization in thaw_chunk
-    pc1 = orig_pc();
-    oopmap_slot = CodeCache::find_oopmap_slot_fast(pc1);
+    CompiledMethod* cm = cb()->as_compiled_method();
+    if (cm->is_deopt_pc(pc1)) {
+      pc1 = orig_pc();
+      oopmap_slot = CodeCache::find_oopmap_slot_fast(pc1);
+    }
   }
   get_oopmap(pc1, oopmap_slot);
 }

src/hotspot/share/runtime/continuation.cpp

@@ -178,6 +178,7 @@ extern "C" void find(intptr_t x);
 bool do_verify_after_thaw(JavaThread* thread);
 template<int x> NOINLINE static bool do_verify_after_thaw1(JavaThread* thread) { return do_verify_after_thaw(thread); }
 static void print_vframe(frame f, const RegisterMap* map = nullptr, outputStream* st = tty);
+void do_deopt_after_thaw(JavaThread* thread);
 
 #ifdef ASSERT
   template <bool relative>
@@ -2317,6 +2318,9 @@ inline bool can_thaw_fast() {
   intptr_t* sp0 = vsp;
   ContinuationHelper::set_anchor(_thread, sp0);
   print_frames(_thread, tty); // must be done after write(), as frame walking reads fields off the Java objects.
+  if (LoomDeoptAfterThaw) {
+    do_deopt_after_thaw(_thread);
+  }
   // if (LoomVerifyAfterThaw) {
   //   assert(do_verify_after_thaw(_thread), "partial: %d empty: %d is_last: %d fix: %d", partial, empty, is_last, fix);
   // }
@@ -2831,6 +2835,22 @@ class ThawVerifyOopsClosure: public OopClosure {
   }
 };
 
+void do_deopt_after_thaw(JavaThread* thread) {
+  int i = 0;
+  StackFrameStream fst(thread, true, false);
+  fst.register_map()->set_include_argument_oops(false);
+  ContinuationHelper::update_register_map_with_callee(fst.register_map(), *fst.current());
+  for (; !fst.is_done(); fst.next()) {
+    if (fst.current()->cb()->is_compiled()) {
+      CompiledMethod* cm = fst.current()->cb()->as_compiled_method();
+      if (!cm->method()->is_continuation_enter_intrinsic()) {
+        cm->make_deoptimized();
+      }
+    }
+  }
+}
+
+
 bool do_verify_after_thaw(JavaThread* thread) {
   assert(thread->has_last_Java_frame(), "");
 

src/hotspot/share/runtime/deoptimization.cpp

@@ -893,14 +893,17 @@ void Deoptimization::deoptimize_all_marked(nmethod* nmethod_only) {
   ResourceMark rm;
   DeoptimizationMarker dm;
 
+  GrowableArray<CompiledMethod*>* marked = new GrowableArray<CompiledMethod*>();
   // Make the dependent methods not entrant
   if (nmethod_only != NULL) {
     nmethod_only->mark_for_deoptimization();
     nmethod_only->make_not_entrant();
+    marked->append(nmethod_only);
   } else {
     MutexLocker mu(SafepointSynchronize::is_at_safepoint() ? NULL : CodeCache_lock, Mutex::_no_safepoint_check_flag);
-    CodeCache::make_marked_nmethods_not_entrant();
+    CodeCache::make_marked_nmethods_not_entrant(marked);
   }
+  CodeCache::make_marked_nmethods_deoptimized(marked);
 
   DeoptimizeMarkedClosure deopt;
   if (SafepointSynchronize::is_at_safepoint()) {

src/hotspot/share/runtime/frame.cpp

@@ -353,6 +353,8 @@ void frame::deoptimize(JavaThread* thread) {
                         cm->deopt_mh_handler_begin() :
                         cm->deopt_handler_begin();
 
+  NativePostCallNop* inst = nativePostCallNop_at(pc());
+
   // Save the original pc before we patch in the new one
   cm->set_original_pc(this, pc());
   patch_pc(thread, deopt);
@@ -1174,6 +1176,22 @@ void frame::oops_entry_do(OopClosure* f, const RegisterMap* map) const {
   entry_frame_call_wrapper()->oops_do(f);
 }
 
+bool frame::is_deoptimized_frame() const {
+  assert(_deopt_state != unknown, "not answerable");
+  if (_deopt_state == is_deoptimized) {
+    return true;
+  }
+
+  /* This method only checks if the frame is deoptimized
+   * as in return address being patched. 
+   * It doesn't care if the OP that we return to is a 
+   * deopt instruction */
+  /*if (_cb != NULL && _cb->is_nmethod()) {
+    return NativeDeoptInstruction::is_deopt_at(_pc);
+  }*/
+  return false;
+}
+
 void frame::oops_do_internal(OopClosure* f, CodeBlobClosure* cf, DerivedOopClosure* df, DerivedPointerIterationMode derived_mode, const RegisterMap* map, bool use_interpreter_oop_map_cache) const {
 #ifndef PRODUCT
   // simulate GC crash here to dump java thread in error report