Use marking cycle counter instead of barriers in old chunks

Author: pron

src/hotspot/cpu/x86/continuationChunk_x86.inline.hpp

@@ -179,6 +179,23 @@ static bool iterate_oops(OopClosureType* closure, const ImmutableOopMap* oopmap,
   return mutated;
 }
 
+static int get_chunk_sp(oop chunk) {
+  // we don't invoke write barriers on oops in thawed frames, so we use the gcSP field to traverse thawed frames
+  int chunk_sp = jdk_internal_misc_StackChunk::sp(chunk);
+  if (chunk_sp != jdk_internal_misc_StackChunk::gc_sp(chunk) && Universe::heap()->requires_barriers(chunk)) {
+    uint64_t marking_cycle = CodeCache::marking_cycle() >> 1;
+    uint64_t chunk_marking_cycle = jdk_internal_misc_StackChunk::mark_cycle(chunk) >> 1;
+    if (marking_cycle == chunk_marking_cycle) {
+      // Marking isn't finished, so we need to traverse thawed frames
+      chunk_sp = jdk_internal_misc_StackChunk::gc_sp(chunk);
+      assert (chunk_sp >= 0 && chunk_sp <= jdk_internal_misc_StackChunk::sp(chunk), "");
+    } else {
+      jdk_internal_misc_StackChunk::set_gc_sp(chunk, chunk_sp); // atomic; benign race
+    }
+  }
+  return chunk_sp;
+}
+
 template <class OopClosureType, bool concurrent_gc>
 void Continuation::stack_chunk_iterate_stack(oop chunk, OopClosureType* closure) {
   // see sender_for_compiled_frame
@@ -209,7 +226,7 @@ void Continuation::stack_chunk_iterate_stack(oop chunk, OopClosureType* closure)
   intptr_t* const start = (intptr_t*)InstanceStackChunkKlass::start_of_stack(chunk);
   intptr_t* const end = start + jdk_internal_misc_StackChunk::size(chunk) - argsize;
   CodeBlob* cb = NULL;
-  for (intptr_t* sp = start + jdk_internal_misc_StackChunk::sp(chunk); sp < end; sp += cb->frame_size()) {
+  for (intptr_t* sp = start + get_chunk_sp(chunk); sp < end; sp += cb->frame_size()) {
     address pc = *(address*)(sp - 1);
     log_develop_trace(jvmcont)("stack_chunk_iterate_stack sp: %ld pc: " INTPTR_FORMAT, sp - start, p2i(pc));
     assert (pc != NULL, "");
@@ -394,7 +411,7 @@ void Continuation::stack_chunk_iterate_stack_bounded(oop chunk, OopClosureType*
   intptr_t* end = start + jdk_internal_misc_StackChunk::size(chunk) - argsize;
   if (end > h) end = h;
   CodeBlob* cb = NULL;
-  for (intptr_t* sp = start + jdk_internal_misc_StackChunk::sp(chunk); sp < end; sp += cb->frame_size()) {
+  for (intptr_t* sp = start + get_chunk_sp(chunk); sp < end; sp += cb->frame_size()) {
     intptr_t* next_sp = sp + cb->frame_size();
     if (sp + cb->frame_size() >= l) {
       sp += cb->frame_size();

src/hotspot/share/classfile/javaClasses.cpp

@@ -4941,6 +4941,8 @@ int jdk_internal_misc_StackChunk::_sp_offset;
 int jdk_internal_misc_StackChunk::_pc_offset;
 int jdk_internal_misc_StackChunk::_argsize_offset;
 int jdk_internal_misc_StackChunk::_mode_offset;
+int jdk_internal_misc_StackChunk::_gcSP_offset;
+int jdk_internal_misc_StackChunk::_markCycle_offset;
 int jdk_internal_misc_StackChunk::_numFrames_offset;
 int jdk_internal_misc_StackChunk::_numOops_offset;
 int jdk_internal_misc_StackChunk::_cont_offset;
@@ -4952,6 +4954,8 @@ int jdk_internal_misc_StackChunk::_cont_offset;
   macro(_pc_offset,        k, vmSymbols::pc_name(),        long_signature,       false); \
   macro(_argsize_offset,   k, vmSymbols::argsize_name(),   int_signature,        false); \
   macro(_mode_offset,      k, vmSymbols::mode_name(),      bool_signature,       false); \
+  macro(_gcSP_offset,      k, "gcSP",                      int_signature,        false); \
+  macro(_markCycle_offset, k, "markCycle",                 long_signature,       false); \
   macro(_numFrames_offset, k, vmSymbols::numFrames_name(), int_signature,        false); \
   macro(_numOops_offset,   k, vmSymbols::numOops_name(),   int_signature,        false); \
   macro(_cont_offset,      k, "cont",                      continuation_signature, false);

src/hotspot/share/classfile/javaClasses.hpp

@@ -1118,6 +1118,8 @@ class jdk_internal_misc_StackChunk: AllStatic {
   static int _pc_offset;
   static int _argsize_offset;
   static int _mode_offset;
+  static int _gcSP_offset;
+  static int _markCycle_offset;
   static int _numFrames_offset;
   static int _numOops_offset;
   static int _cont_offset;
@@ -1143,6 +1145,10 @@ class jdk_internal_misc_StackChunk: AllStatic {
   static inline void set_argsize(oop ref, int value);
   static inline bool gc_mode(oop ref);
   static inline void set_gc_mode(oop ref, bool value);
+  static inline int gc_sp(oop ref);
+  static inline void set_gc_sp(oop ref, int value);
+  static inline uint64_t mark_cycle(oop ref);
+  static inline void set_mark_cycle(oop ref, uint64_t value);
   static inline int end(oop ref);
   static inline int numFrames(oop ref);
   static inline void set_numFrames(oop ref, int value);

src/hotspot/share/classfile/javaClasses.inline.hpp

@@ -336,6 +336,18 @@ inline bool jdk_internal_misc_StackChunk::gc_mode(oop ref) {
 inline void jdk_internal_misc_StackChunk::set_gc_mode(oop ref, bool value) {
   ref->bool_field_put(_mode_offset, (jboolean)value);
 }
+inline int jdk_internal_misc_StackChunk::gc_sp(oop ref) {
+  return ref->int_field(_gcSP_offset);
+}
+inline void jdk_internal_misc_StackChunk::set_gc_sp(oop ref, int value) {
+  ref->int_field_put(_gcSP_offset, value);
+}
+inline uint64_t jdk_internal_misc_StackChunk::mark_cycle(oop ref) {
+  return (uint64_t)ref->long_field(_markCycle_offset);
+}
+inline void jdk_internal_misc_StackChunk::set_mark_cycle(oop ref, uint64_t value) {
+  ref->long_field_put(_markCycle_offset, (jlong)value);
+}
 inline int jdk_internal_misc_StackChunk::end(oop ref) {
   return size(ref) - argsize(ref);
 }

src/hotspot/share/runtime/continuation.cpp

@@ -2518,6 +2518,7 @@ class Freeze {
     // We're always writing to a young chunk, so the GC can't see it until the next safepoint.
     jdk_internal_misc_StackChunk::set_sp(chunk, sp);
     jdk_internal_misc_StackChunk::set_pc(chunk, *(address*)(top - SENDER_SP_RET_ADDRESS_OFFSET));
+    jdk_internal_misc_StackChunk::set_gc_sp(chunk, sp);
 
     // we copy the top frame's return address and link, but not the bottom's
     intptr_t* chunk_top = (intptr_t*)InstanceStackChunkKlass::start_of_stack(chunk) + sp;
@@ -2570,6 +2571,8 @@ class Freeze {
     jdk_internal_misc_StackChunk::set_pc(chunk, NULL);
     jdk_internal_misc_StackChunk::set_argsize(chunk, 0); // TODO PERF unnecessary?
     jdk_internal_misc_StackChunk::set_gc_mode(chunk, false);
+    jdk_internal_misc_StackChunk::set_gc_sp(chunk, sp);
+    jdk_internal_misc_StackChunk::set_mark_cycle(chunk, 0);
     jdk_internal_misc_StackChunk::set_parent_raw<typename ConfigT::OopT>(chunk, chunk0); // field is uninitialized
     jdk_internal_misc_StackChunk::set_cont_raw<typename ConfigT::OopT>(chunk, _cont.mirror());
     ContMirror::reset_chunk_counters(chunk);
@@ -2627,6 +2630,7 @@ class Freeze {
       for (oop chunk = _cont.tail(); chunk != (oop)NULL; chunk = jdk_internal_misc_StackChunk::parent(chunk)) {
         // jdk_internal_misc_StackChunk::set_parent(chunk, (oop)NULL); // TODO: kills loop
         jdk_internal_misc_StackChunk::set_sp(chunk, jdk_internal_misc_StackChunk::size(chunk) + frame_metadata);
+        jdk_internal_misc_StackChunk::set_gc_sp(chunk, jdk_internal_misc_StackChunk::sp(chunk));
         ContMirror::reset_chunk_counters(chunk);
       }
       assert (!_cont.is_flag(FLAG_LAST_FRAME_INTERPRETED), "");
@@ -3960,7 +3964,7 @@ class Thaw {
     assert (verify_stack_chunk<1>(chunk), "");
     // assert (verify_continuation<99>(_cont.mirror()), "");
 
-    const bool barriers = requires_barriers(chunk); // TODO R PERF
+    const bool barriers = false; // requires_barriers(chunk); // TODO R PERF
 
     // if (!barriers) { // TODO ????
     //   ContMirror::reset_chunk_counters(chunk);
@@ -3976,8 +3980,11 @@ class Thaw {
       vsp = _cont.entrySP();
     }
 
+    // Instead of invoking barriers on oops in thawed frames, we use the gcSP field; see continuationChunk's get_chunk_sp
+    jdk_internal_misc_StackChunk::set_mark_cycle(chunk, CodeCache::marking_cycle());
+
     bool partial, empty;
-    if (!TEST_THAW_ONE_CHUNK_FRAME && !barriers && LIKELY(FULL_STACK || (size < threshold))) {
+    if (!TEST_THAW_ONE_CHUNK_FRAME && LIKELY((FULL_STACK || (size < threshold)) && !barriers)) {
       // prefetch with anticipation of memcpy starting at highest address
       prefetch_chunk_pd(InstanceStackChunkKlass::start_of_stack(chunk), size);
 

src/java.base/share/classes/jdk/internal/misc/StackChunk.java

@@ -32,6 +32,8 @@ public class StackChunk {
      public int sp; // in words
      private int argsize; // bottom stack-passed arguments, in words
      private long pc;
+     private int gcSP;
+     private long markCycle;
 
      private int numFrames;
      private int numOops;