8242054: Shenandoah: New incremental-update mode

Reviewed-by: shade

src/hotspot/cpu/aarch64/gc/shenandoah/shenandoahBarrierSetAssembler_aarch64.cpp

@@ -49,7 +49,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
                                                        Register src, Register dst, Register count, RegSet saved_regs) {
   if (is_oop) {
     bool dest_uninitialized = (decorators & IS_DEST_UNINITIALIZED) != 0;
-    if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
+    if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {
 
       Label done;
 
@@ -59,7 +59,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
       // Is GC active?
       Address gc_state(rthread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
       __ ldrb(rscratch1, gc_state);
-      if (dest_uninitialized) {
+      if (ShenandoahSATBBarrier && dest_uninitialized) {
         __ tbz(rscratch1, ShenandoahHeap::HAS_FORWARDED_BITPOS, done);
       } else {
         __ mov(rscratch2, ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING);

src/hotspot/cpu/x86/gc/shenandoah/shenandoahBarrierSetAssembler_x86.cpp

@@ -53,7 +53,7 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
 
   if (is_reference_type(type)) {
 
-    if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahLoadRefBarrier) {
+    if ((ShenandoahSATBBarrier && !dest_uninitialized) || ShenandoahStoreValEnqueueBarrier || ShenandoahLoadRefBarrier) {
 #ifdef _LP64
       Register thread = r15_thread;
 #else
@@ -79,9 +79,11 @@ void ShenandoahBarrierSetAssembler::arraycopy_prologue(MacroAssembler* masm, Dec
 
       // Avoid runtime call when not active.
       Address gc_state(thread, in_bytes(ShenandoahThreadLocalData::gc_state_offset()));
-      int flags = ShenandoahHeap::HAS_FORWARDED;
-      if (!dest_uninitialized) {
-        flags |= ShenandoahHeap::MARKING;
+      int flags;
+      if (ShenandoahSATBBarrier && dest_uninitialized) {
+        flags = ShenandoahHeap::HAS_FORWARDED;
+      } else {
+        flags = ShenandoahHeap::HAS_FORWARDED | ShenandoahHeap::MARKING;
       }
       __ testb(gc_state, flags);
       __ jcc(Assembler::zero, done);

src/hotspot/share/gc/shenandoah/c2/shenandoahBarrierSetC2.cpp

@@ -774,7 +774,7 @@ bool ShenandoahBarrierSetC2::array_copy_requires_gc_barriers(bool tightly_couple
   if (!is_oop) {
     return false;
   }
-  if (tightly_coupled_alloc) {
+  if (ShenandoahSATBBarrier && tightly_coupled_alloc) {
     if (phase == Optimization) {
       return false;
     }
@@ -841,7 +841,11 @@ void ShenandoahBarrierSetC2::clone_at_expansion(PhaseMacroExpand* phase, ArrayCo
     debug_only(gc_state_adr_type = phase->C->get_adr_type(gc_state_idx));
 
     Node* gc_state    = phase->transform_later(new LoadBNode(ctrl, mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered));
-    Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED)));
+    int flags = ShenandoahHeap::HAS_FORWARDED;
+    if (ShenandoahStoreValEnqueueBarrier) {
+      flags |= ShenandoahHeap::MARKING;
+    }
+    Node* stable_and  = phase->transform_later(new AndINode(gc_state, phase->igvn().intcon(flags)));
     Node* stable_cmp  = phase->transform_later(new CmpINode(stable_and, phase->igvn().zerocon(T_INT)));
     Node* stable_test = phase->transform_later(new BoolNode(stable_cmp, BoolTest::ne));
 

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.cpp

@@ -860,8 +860,8 @@ static void hide_strip_mined_loop(OuterStripMinedLoopNode* outer, CountedLoopNod
   inner->clear_strip_mined();
 }
 
-void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
-                                                  PhaseIdealLoop* phase) {
+void ShenandoahBarrierC2Support::test_heap_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
+                                                 PhaseIdealLoop* phase, int flags) {
   IdealLoopTree* loop = phase->get_loop(ctrl);
   Node* thread = new ThreadLocalNode();
   phase->register_new_node(thread, ctrl);
@@ -875,7 +875,7 @@ void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, No
 
   Node* gc_state = new LoadBNode(ctrl, raw_mem, gc_state_addr, gc_state_adr_type, TypeInt::BYTE, MemNode::unordered);
   phase->register_new_node(gc_state, ctrl);
-  Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(ShenandoahHeap::HAS_FORWARDED));
+  Node* heap_stable_and = new AndINode(gc_state, phase->igvn().intcon(flags));
   phase->register_new_node(heap_stable_and, ctrl);
   Node* heap_stable_cmp = new CmpINode(heap_stable_and, phase->igvn().zerocon(T_INT));
   phase->register_new_node(heap_stable_cmp, ctrl);
@@ -889,7 +889,7 @@ void ShenandoahBarrierC2Support::test_heap_stable(Node*& ctrl, Node* raw_mem, No
   ctrl = new IfTrueNode(heap_stable_iff);
   phase->register_control(ctrl, loop, heap_stable_iff);
 
-  assert(is_heap_stable_test(heap_stable_iff), "Should match the shape");
+  assert(is_heap_state_test(heap_stable_iff, flags), "Should match the shape");
 }
 
 void ShenandoahBarrierC2Support::test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase) {
@@ -1437,7 +1437,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
     Node* raw_mem_phi = PhiNode::make(region, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
 
     // Stable path.
-    test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
+    test_heap_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::HAS_FORWARDED);
     IfNode* heap_stable_iff = heap_stable_ctrl->in(0)->as_If();
 
     // Heap stable case
@@ -1608,7 +1608,7 @@ void ShenandoahBarrierC2Support::pin_and_expand(PhaseIdealLoop* phase) {
     Node* phi2 = PhiNode::make(region2, raw_mem, Type::MEMORY, TypeRawPtr::BOTTOM);
 
     // Stable path.
-    test_heap_stable(ctrl, raw_mem, heap_stable_ctrl, phase);
+    test_heap_state(ctrl, raw_mem, heap_stable_ctrl, phase, ShenandoahHeap::MARKING);
     region->init_req(_heap_stable, heap_stable_ctrl);
     phi->init_req(_heap_stable, raw_mem);
 

src/hotspot/share/gc/shenandoah/c2/shenandoahSupport.hpp

@@ -58,8 +58,8 @@ class ShenandoahBarrierC2Support : public AllStatic {
   static Node* find_bottom_mem(Node* ctrl, PhaseIdealLoop* phase);
   static void follow_barrier_uses(Node* n, Node* ctrl, Unique_Node_List& uses, PhaseIdealLoop* phase);
   static void test_null(Node*& ctrl, Node* val, Node*& null_ctrl, PhaseIdealLoop* phase);
-  static void test_heap_stable(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
-                               PhaseIdealLoop* phase);
+  static void test_heap_state(Node*& ctrl, Node* raw_mem, Node*& heap_stable_ctrl,
+                              PhaseIdealLoop* phase, int flags);
   static void call_lrb_stub(Node*& ctrl, Node*& val, Node* load_addr, Node*& result_mem, Node* raw_mem, bool is_native, PhaseIdealLoop* phase);
   static Node* clone_null_check(Node*& c, Node* val, Node* unc_ctrl, PhaseIdealLoop* phase);
   static void fix_null_check(Node* unc, Node* unc_ctrl, Node* new_unc_ctrl, Unique_Node_List& uses,

src/hotspot/share/gc/shenandoah/heuristics/shenandoahAggressiveHeuristics.cpp

@@ -46,7 +46,7 @@ ShenandoahAggressiveHeuristics::ShenandoahAggressiveHeuristics() : ShenandoahHeu
 
   // Final configuration checks
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
-  SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
+  SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier || ShenandoahStoreValEnqueueBarrier);
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
 }

src/hotspot/share/gc/shenandoah/heuristics/shenandoahCompactHeuristics.cpp

@@ -44,7 +44,7 @@ ShenandoahCompactHeuristics::ShenandoahCompactHeuristics() : ShenandoahHeuristic
 
   // Final configuration checks
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
-  SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
+  SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier || ShenandoahStoreValEnqueueBarrier);
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
 }

src/hotspot/share/gc/shenandoah/heuristics/shenandoahStaticHeuristics.cpp

@@ -37,7 +37,7 @@ ShenandoahStaticHeuristics::ShenandoahStaticHeuristics() : ShenandoahHeuristics(
 
   // Final configuration checks
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahLoadRefBarrier);
-  SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier);
+  SHENANDOAH_CHECK_FLAG_SET(ShenandoahSATBBarrier || ShenandoahStoreValEnqueueBarrier);
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahCASBarrier);
   SHENANDOAH_CHECK_FLAG_SET(ShenandoahCloneBarrier);
 }

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.cpp

@@ -241,7 +241,7 @@ oop ShenandoahBarrierSet::load_reference_barrier_native_impl(oop obj, T* load_ad
 }
 
 void ShenandoahBarrierSet::clone_barrier_runtime(oop src) {
-  if (_heap->has_forwarded_objects()) {
+  if (_heap->has_forwarded_objects() || (ShenandoahStoreValEnqueueBarrier && _heap->is_concurrent_mark_in_progress())) {
     clone_barrier(src);
   }
 }

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.hpp

@@ -107,12 +107,16 @@ class ShenandoahBarrierSet: public BarrierSet {
 
 private:
   template <class T>
-  inline void arraycopy_marking(T* ary, size_t count);
+  inline void arraycopy_marking(T* src, T* dst, size_t count);
   template <class T>
   inline void arraycopy_evacuation(T* src, size_t count);
   template <class T>
   inline void arraycopy_update(T* src, size_t count);
 
+  inline void clone_marking(oop src);
+  inline void clone_evacuation(oop src);
+  inline void clone_update(oop src);
+
   template <class T, bool HAS_FWD, bool EVAC, bool ENQUEUE>
   inline void arraycopy_work(T* src, size_t count);
 

src/hotspot/share/gc/shenandoah/shenandoahBarrierSet.inline.hpp

@@ -86,7 +86,7 @@ inline void ShenandoahBarrierSet::satb_enqueue(oop value) {
 }
 
 inline void ShenandoahBarrierSet::storeval_barrier(oop obj) {
-  if (obj != NULL && ShenandoahStoreValEnqueueBarrier) {
+  if (ShenandoahStoreValEnqueueBarrier && obj != NULL && _heap->is_concurrent_mark_in_progress()) {
     enqueue(obj);
   }
 }
@@ -291,7 +291,7 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
   }
   int gc_state = _heap->gc_state();
   if ((gc_state & ShenandoahHeap::MARKING) != 0) {
-    arraycopy_marking(dst, count);
+    arraycopy_marking(src, dst, count);
   } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
     arraycopy_evacuation(src, count);
   } else if ((gc_state & ShenandoahHeap::UPDATEREFS) != 0) {
@@ -300,8 +300,9 @@ void ShenandoahBarrierSet::arraycopy_barrier(T* src, T* dst, size_t count) {
 }
 
 template <class T>
-void ShenandoahBarrierSet::arraycopy_marking(T* array, size_t count) {
+void ShenandoahBarrierSet::arraycopy_marking(T* src, T* dst, size_t count) {
   assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
+  T* array = ShenandoahSATBBarrier ? dst : src;
   if (!_heap->marking_context()->allocated_after_mark_start(reinterpret_cast<HeapWord*>(array))) {
     arraycopy_work<T, false, false, true>(array, count);
   }

src/hotspot/share/gc/shenandoah/shenandoahBarrierSetClone.inline.hpp

@@ -33,7 +33,7 @@
 #include "oops/access.hpp"
 #include "oops/compressedOops.hpp"
 
-template <bool EVAC, bool ENQUEUE>
+template <bool HAS_FWD, bool EVAC, bool ENQUEUE>
 class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure {
 private:
   ShenandoahHeap* const _heap;
@@ -46,18 +46,18 @@ class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure {
     T o = RawAccess<>::oop_load(p);
     if (!CompressedOops::is_null(o)) {
       oop obj = CompressedOops::decode_not_null(o);
-      if (_cset->is_in(obj)) {
+      if (HAS_FWD && _cset->is_in(obj)) {
         oop fwd = _bs->resolve_forwarded_not_null(obj);
         if (EVAC && obj == fwd) {
           fwd = _heap->evacuate_object(obj, _thread);
         }
-        if (ENQUEUE) {
-          _bs->enqueue(fwd);
-        }
         assert(obj != fwd || _heap->cancelled_gc(), "must be forwarded");
         ShenandoahHeap::cas_oop(fwd, p, o);
+        obj = fwd;
+      }
+      if (ENQUEUE) {
+        _bs->enqueue(obj);
       }
-
     }
   }
 public:
@@ -72,24 +72,43 @@ class ShenandoahUpdateRefsForOopClosure: public BasicOopIterateClosure {
   virtual void do_oop(narrowOop* p) { do_oop_work(p); }
 };
 
-void ShenandoahBarrierSet::clone_barrier(oop obj) {
-  assert(ShenandoahCloneBarrier, "only get here with clone barriers enabled");
-  assert(_heap->has_forwarded_objects(), "only when heap is unstable");
+void ShenandoahBarrierSet::clone_marking(oop obj) {
+  assert(_heap->is_concurrent_mark_in_progress(), "only during marking");
+  assert(ShenandoahStoreValEnqueueBarrier, "only with incremental-update");
+  if (!_heap->marking_context()->allocated_after_mark_start(obj)) {
+    ShenandoahUpdateRefsForOopClosure</* has_fwd = */ false, /* evac = */ false, /* enqueue */ true> cl;
+    obj->oop_iterate(&cl);
+  }
+}
+
+void ShenandoahBarrierSet::clone_evacuation(oop obj) {
+  assert(_heap->is_evacuation_in_progress(), "only during evacuation");
+  if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
+    ShenandoahEvacOOMScope oom_evac_scope;
+    ShenandoahUpdateRefsForOopClosure</* has_fwd = */ true, /* evac = */ true, /* enqueue */ false> cl;
+    obj->oop_iterate(&cl);
+  }
+}
 
-  // This is called for cloning an object (see jvm.cpp) after the clone
-  // has been made. We are not interested in any 'previous value' because
-  // it would be NULL in any case. But we *are* interested in any oop*
-  // that potentially need to be updated.
+void ShenandoahBarrierSet::clone_update(oop obj) {
+  assert(_heap->is_update_refs_in_progress(), "only during update-refs");
+  if (need_bulk_update(cast_from_oop<HeapWord*>(obj))) {
+    ShenandoahUpdateRefsForOopClosure</* has_fwd = */ true, /* evac = */ false, /* enqueue */ false> cl;
+    obj->oop_iterate(&cl);
+  }
+}
 
+void ShenandoahBarrierSet::clone_barrier(oop obj) {
+  assert(ShenandoahCloneBarrier, "only get here with clone barriers enabled");
   shenandoah_assert_correct(NULL, obj);
-  if (!need_bulk_update(cast_from_oop<HeapWord*>(obj))) return;
-  if (_heap->is_evacuation_in_progress()) {
-    ShenandoahEvacOOMScope evac_scope;
-    ShenandoahUpdateRefsForOopClosure</* evac = */ true, /* enqueue */ false> cl;
-    obj->oop_iterate(&cl);
+
+  int gc_state = _heap->gc_state();
+  if ((gc_state & ShenandoahHeap::MARKING) != 0) {
+    clone_marking(obj);
+  } else if ((gc_state & ShenandoahHeap::EVACUATION) != 0) {
+    clone_evacuation(obj);
   } else {
-    ShenandoahUpdateRefsForOopClosure</* evac = */ false, /* enqueue */ false> cl;
-    obj->oop_iterate(&cl);
+    clone_update(obj);
   }
 }
 

src/hotspot/share/gc/shenandoah/shenandoahConcurrentMark.cpp

@@ -183,26 +183,32 @@ class ShenandoahConcurrentMarkingTask : public AbstractGangTask {
 class ShenandoahSATBAndRemarkCodeRootsThreadsClosure : public ThreadClosure {
 private:
   ShenandoahSATBBufferClosure* _satb_cl;
+  OopClosure*            const _cl;
   MarkingCodeBlobClosure*      _code_cl;
   uintx _claim_token;
 
 public:
-  ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, MarkingCodeBlobClosure* code_cl) :
-    _satb_cl(satb_cl), _code_cl(code_cl),
+  ShenandoahSATBAndRemarkCodeRootsThreadsClosure(ShenandoahSATBBufferClosure* satb_cl, OopClosure* cl, MarkingCodeBlobClosure* code_cl) :
+    _satb_cl(satb_cl), _cl(cl), _code_cl(code_cl),
     _claim_token(Threads::thread_claim_token()) {}
 
   void do_thread(Thread* thread) {
     if (thread->claim_threads_do(true, _claim_token)) {
       ShenandoahThreadLocalData::satb_mark_queue(thread).apply_closure_and_empty(_satb_cl);
-      if (_code_cl != NULL && thread->is_Java_thread()) {
-        // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
-        // however the liveness of oops reachable from nmethods have very complex lifecycles:
-        // * Alive if on the stack of an executing method
-        // * Weakly reachable otherwise
-        // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
-        // live by the SATB invariant but other oops recorded in nmethods may behave differently.
-        JavaThread* jt = (JavaThread*)thread;
-        jt->nmethods_do(_code_cl);
+      if (thread->is_Java_thread()) {
+        if (_cl != NULL) {
+          ResourceMark rm;
+          thread->oops_do(_cl, _code_cl);
+        } else if (_code_cl != NULL) {
+          // In theory it should not be neccessary to explicitly walk the nmethods to find roots for concurrent marking
+          // however the liveness of oops reachable from nmethods have very complex lifecycles:
+          // * Alive if on the stack of an executing method
+          // * Weakly reachable otherwise
+          // Some objects reachable from nmethods, such as the class loader (or klass_holder) of the receiver should be
+          // live by the SATB invariant but other oops recorded in nmethods may behave differently.
+          JavaThread* jt = (JavaThread*)thread;
+          jt->nmethods_do(_code_cl);
+        }
       }
     }
   }
@@ -242,21 +248,20 @@ class ShenandoahFinalMarkingTask : public AbstractGangTask {
       ShenandoahSATBBufferClosure cl(q);
       SATBMarkQueueSet& satb_mq_set = ShenandoahBarrierSet::satb_mark_queue_set();
       while (satb_mq_set.apply_closure_to_completed_buffer(&cl));
-
-      if (heap->unload_classes() && !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading()) {
-        if (heap->has_forwarded_objects()) {
-          ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp);
-          MarkingCodeBlobClosure blobsCl(&resolve_mark_cl, !CodeBlobToOopClosure::FixRelocations);
-          ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, &blobsCl);
-          Threads::threads_do(&tc);
-        } else {
-          ShenandoahMarkRefsClosure mark_cl(q, rp);
-          MarkingCodeBlobClosure blobsCl(&mark_cl, !CodeBlobToOopClosure::FixRelocations);
-          ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, &blobsCl);
-          Threads::threads_do(&tc);
-        }
+      bool do_nmethods = heap->unload_classes() && !ShenandoahConcurrentRoots::can_do_concurrent_class_unloading();
+      if (heap->has_forwarded_objects()) {
+        ShenandoahMarkResolveRefsClosure resolve_mark_cl(q, rp);
+        MarkingCodeBlobClosure blobsCl(&resolve_mark_cl, !CodeBlobToOopClosure::FixRelocations);
+        ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
+                                                          ShenandoahStoreValEnqueueBarrier ? &resolve_mark_cl : NULL,
+                                                          do_nmethods ? &blobsCl : NULL);
+        Threads::threads_do(&tc);
       } else {
-        ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl, NULL);
+        ShenandoahMarkRefsClosure mark_cl(q, rp);
+        MarkingCodeBlobClosure blobsCl(&mark_cl, !CodeBlobToOopClosure::FixRelocations);
+        ShenandoahSATBAndRemarkCodeRootsThreadsClosure tc(&cl,
+                                                          ShenandoahStoreValEnqueueBarrier ? &mark_cl : NULL,
+                                                          do_nmethods ? &blobsCl : NULL);
         Threads::threads_do(&tc);
       }
     }

src/hotspot/share/gc/shenandoah/shenandoahHeap.cpp

@@ -46,6 +46,7 @@
 #include "gc/shenandoah/shenandoahHeap.inline.hpp"
 #include "gc/shenandoah/shenandoahHeapRegion.hpp"
 #include "gc/shenandoah/shenandoahHeapRegionSet.hpp"
+#include "gc/shenandoah/shenandoahIUMode.hpp"
 #include "gc/shenandoah/shenandoahMarkCompact.hpp"
 #include "gc/shenandoah/shenandoahMarkingContext.inline.hpp"
 #include "gc/shenandoah/shenandoahMemoryPool.hpp"
@@ -397,6 +398,8 @@ void ShenandoahHeap::initialize_heuristics() {
   if (ShenandoahGCMode != NULL) {
     if (strcmp(ShenandoahGCMode, "normal") == 0) {
       _gc_mode = new ShenandoahNormalMode();
+    } else if (strcmp(ShenandoahGCMode, "iu") == 0) {
+      _gc_mode = new ShenandoahIUMode();
     } else if (strcmp(ShenandoahGCMode, "passive") == 0) {
       _gc_mode = new ShenandoahPassiveMode();
     } else {