8276098: Do precise BOT updates in G1 evacuation phase

Reviewed-by: tschatzl, ayang

src/hotspot/share/gc/g1/g1AllocRegion.cpp

@@ -43,10 +43,7 @@ void G1AllocRegion::setup(G1CollectedHeap* g1h, HeapRegion* dummy_region) {
 
   // Make sure that any allocation attempt on this region will fail
   // and will not trigger any asserts.
-  assert(dummy_region->allocate_no_bot_updates(1) == NULL, "should fail");
-  assert(dummy_region->allocate(1) == NULL, "should fail");
   DEBUG_ONLY(size_t assert_tmp);
-  assert(dummy_region->par_allocate_no_bot_updates(1, 1, &assert_tmp) == NULL, "should fail");
   assert(dummy_region->par_allocate(1, 1, &assert_tmp) == NULL, "should fail");
 
   _g1h = g1h;
@@ -77,8 +74,9 @@ size_t G1AllocRegion::fill_up_remaining_space(HeapRegion* alloc_region) {
   while (free_word_size >= min_word_size_to_fill) {
     HeapWord* dummy = par_allocate(alloc_region, free_word_size);
     if (dummy != NULL) {
-      // If the allocation was successful we should fill in the space.
-      CollectedHeap::fill_with_object(dummy, free_word_size);
+      // If the allocation was successful we should fill in the space. If the
+      // allocation was in old any necessary BOT updates will be done.
+      alloc_region->fill_with_dummy_object(dummy, free_word_size);
       alloc_region->set_pre_dummy_top(dummy);
       result += free_word_size * HeapWordSize;
       break;
@@ -256,7 +254,6 @@ G1AllocRegion::G1AllocRegion(const char* name,
   : _alloc_region(NULL),
     _count(0),
     _used_bytes_before(0),
-    _bot_updates(bot_updates),
     _name(name),
     _node_index(node_index)
  { }
@@ -389,7 +386,7 @@ HeapRegion* OldGCAllocRegion::release() {
       // original problem cannot occur.
       if (to_allocate_words >= G1CollectedHeap::min_fill_size()) {
         HeapWord* dummy = attempt_allocation(to_allocate_words);
-        CollectedHeap::fill_with_object(dummy, to_allocate_words);
+        cur->fill_with_dummy_object(dummy, to_allocate_words);
       }
     }
   }

src/hotspot/share/gc/g1/g1AllocRegion.hpp

@@ -68,9 +68,6 @@ class G1AllocRegion : public CHeapObj<mtGC> {
   // we allocated in it.
   size_t _used_bytes_before;
 
-  // When true, indicates that allocate calls should do BOT updates.
-  const bool _bot_updates;
-
   // Useful for debugging and tracing.
   const char* _name;
 

src/hotspot/share/gc/g1/g1AllocRegion.inline.hpp

@@ -31,9 +31,9 @@
 
 #define assert_alloc_region(p, message)                                  \
   do {                                                                   \
-    assert((p), "[%s] %s c: %u b: %s r: " PTR_FORMAT " u: " SIZE_FORMAT, \
-           _name, (message), _count, BOOL_TO_STR(_bot_updates),          \
-           p2i(_alloc_region), _used_bytes_before);                      \
+    assert((p), "[%s] %s c: %u r: " PTR_FORMAT " u: " SIZE_FORMAT,       \
+           _name, (message), _count, p2i(_alloc_region),                 \
+           _used_bytes_before);                                          \
   } while (0)
 
 
@@ -45,11 +45,7 @@ inline HeapWord* G1AllocRegion::allocate(HeapRegion* alloc_region,
                                          size_t word_size) {
   assert(alloc_region != NULL, "pre-condition");
 
-  if (!_bot_updates) {
-    return alloc_region->allocate_no_bot_updates(word_size);
-  } else {
-    return alloc_region->allocate(word_size);
-  }
+  return alloc_region->allocate(word_size);
 }
 
 inline HeapWord* G1AllocRegion::par_allocate(HeapRegion* alloc_region, size_t word_size) {
@@ -64,11 +60,7 @@ inline HeapWord* G1AllocRegion::par_allocate(HeapRegion* alloc_region,
   assert(alloc_region != NULL, "pre-condition");
   assert(!alloc_region->is_empty(), "pre-condition");
 
-  if (!_bot_updates) {
-    return alloc_region->par_allocate_no_bot_updates(min_word_size, desired_word_size, actual_word_size);
-  } else {
-    return alloc_region->par_allocate(min_word_size, desired_word_size, actual_word_size);
-  }
+  return alloc_region->par_allocate(min_word_size, desired_word_size, actual_word_size);
 }
 
 inline HeapWord* G1AllocRegion::attempt_allocation(size_t word_size) {

src/hotspot/share/gc/g1/g1Allocator.cpp

@@ -299,8 +299,12 @@ G1PLABAllocator::G1PLABAllocator(G1Allocator* allocator) :
     _direct_allocated[state] = 0;
     uint length = alloc_buffers_length(state);
     _alloc_buffers[state] = NEW_C_HEAP_ARRAY(PLAB*, length, mtGC);
+    size_t word_sz = _g1h->desired_plab_sz(state);
     for (uint node_index = 0; node_index < length; node_index++) {
-      _alloc_buffers[state][node_index] = new PLAB(_g1h->desired_plab_sz(state));
+      // Specialized PLABs for old that handle BOT updates for object allocations.
+      _alloc_buffers[state][node_index] = (state == G1HeapRegionAttr::Old)
+                                        ? new G1BotUpdatingPLAB(word_sz)
+                                        : new PLAB(word_sz);
     }
   }
 }
@@ -360,6 +364,7 @@ HeapWord* G1PLABAllocator::allocate_direct_or_new_plab(G1HeapRegionAttr dest,
   // Try direct allocation.
   HeapWord* result = _allocator->par_allocate_during_gc(dest, word_sz, node_index);
   if (result != NULL) {
+    update_bot_for_direct_allocation(dest, result, word_sz);
     _direct_allocated[dest.type()] += word_sz;
   }
   return result;

src/hotspot/share/gc/g1/g1Allocator.hpp

@@ -145,6 +145,22 @@ class G1Allocator : public CHeapObj<mtGC> {
                                    uint node_index);
 };
 
+// Specialized PLAB for old generation promotions. For old regions the
+// BOT needs to be updated and the relevant data to do this efficiently
+// is stored in the PLAB.
+class G1BotUpdatingPLAB : public PLAB {
+  // An object spanning this threshold will cause a BOT update.
+  HeapWord* _next_bot_threshold;
+  // The region in which the PLAB resides.
+  HeapRegion* _region;
+public:
+  G1BotUpdatingPLAB(size_t word_sz) : PLAB(word_sz) { }
+  // Sets the new PLAB buffer as well as updates the threshold and region.
+  void set_buf(HeapWord* buf, size_t word_sz) override;
+  // Updates the BOT if the last allocation crossed the threshold.
+  inline void update_bot(size_t word_sz);
+};
+
 // Manages the PLABs used during garbage collection. Interface for allocation from PLABs.
 // Needs to handle multiple contexts, extra alignment in any "survivor" area and some
 // statistics.
@@ -165,11 +181,18 @@ class G1PLABAllocator : public CHeapObj<mtGC> {
   inline PLAB* alloc_buffer(G1HeapRegionAttr dest, uint node_index) const;
   inline PLAB* alloc_buffer(region_type_t dest, uint node_index) const;
 
+  // Helpers to do explicit BOT updates for allocations in old generation regions.
+  void update_bot_for_direct_allocation(G1HeapRegionAttr attr, HeapWord* addr, size_t size);
+
   // Returns the number of allocation buffers for the given dest.
   // There is only 1 buffer for Old while Young may have multiple buffers depending on
   // active NUMA nodes.
   inline uint alloc_buffers_length(region_type_t dest) const;
 
+  // Returns if BOT updates are needed for the given destinaion. Currently we only have
+  // two destinations and BOT updates are only needed for the old generation.
+  inline bool needs_bot_update(G1HeapRegionAttr dest) const;
+
   bool may_throw_away_buffer(size_t const allocation_word_sz, size_t const buffer_size) const;
 public:
   G1PLABAllocator(G1Allocator* allocator);
@@ -198,6 +221,9 @@ class G1PLABAllocator : public CHeapObj<mtGC> {
                             bool* refill_failed,
                             uint node_index);
 
+  // Update the BOT for the last PLAB allocation.
+  inline void update_bot_for_plab_allocation(G1HeapRegionAttr dest, size_t word_sz, uint node_index);
+
   void undo_allocation(G1HeapRegionAttr dest, HeapWord* obj, size_t word_sz, uint node_index);
 };
 

src/hotspot/share/gc/g1/g1Allocator.inline.hpp

@@ -133,4 +133,49 @@ inline HeapWord* G1PLABAllocator::allocate(G1HeapRegionAttr dest,
   return allocate_direct_or_new_plab(dest, word_sz, refill_failed, node_index);
 }
 
+inline bool G1PLABAllocator::needs_bot_update(G1HeapRegionAttr dest) const {
+  return dest.is_old();
+}
+
+inline void G1PLABAllocator::update_bot_for_direct_allocation(G1HeapRegionAttr attr, HeapWord* addr, size_t size) {
+  if (!needs_bot_update(attr)) {
+    return;
+  }
+
+  // Out of PLAB allocations in an old generation region. Update BOT.
+  HeapRegion* region = _g1h->heap_region_containing(addr);
+  region->update_bot_at(addr, size);
+}
+
+inline void G1PLABAllocator::update_bot_for_plab_allocation(G1HeapRegionAttr dest, size_t word_sz, uint node_index) {
+  assert(needs_bot_update(dest), "Wrong destination: %s", dest.get_type_str());
+  G1BotUpdatingPLAB* plab = static_cast<G1BotUpdatingPLAB*>(alloc_buffer(dest, node_index));
+  plab->update_bot(word_sz);
+}
+
+inline void G1BotUpdatingPLAB::set_buf(HeapWord* buf, size_t word_sz) {
+  PLAB::set_buf(buf, word_sz);
+  // Update the region and threshold to allow efficient BOT updates.
+  _region = G1CollectedHeap::heap()->heap_region_containing(buf);
+  _next_bot_threshold = _region->bot_threshold_for_addr(buf);
+}
+
+inline void G1BotUpdatingPLAB::update_bot(size_t word_sz) {
+  // The last object end is at _top, if it did not cross the
+  // threshold, there is nothing to do.
+  if (_top <= _next_bot_threshold) {
+    return;
+  }
+
+  HeapWord* obj_start = _top - word_sz;
+  assert(contains(obj_start),
+         "Object start outside PLAB. bottom: " PTR_FORMAT " object: " PTR_FORMAT,
+         p2i(_bottom), p2i(obj_start));
+  assert(obj_start <= _next_bot_threshold,
+         "Object start not below or at threshold. threshold: " PTR_FORMAT " object: " PTR_FORMAT,
+         p2i(_next_bot_threshold), p2i(obj_start));
+
+  _region->update_bot_crossing_threshold(&_next_bot_threshold, obj_start, _top);
+}
+
 #endif // SHARE_GC_G1_G1ALLOCATOR_INLINE_HPP

src/hotspot/share/gc/g1/g1BlockOffsetTable.hpp

@@ -108,6 +108,7 @@ class G1BlockOffsetTable: public CHeapObj<mtGC> {
 
 class G1BlockOffsetTablePart {
   friend class G1BlockOffsetTable;
+  friend class HeapRegion;
   friend class VMStructs;
 private:
   // allocation boundary at which offset array must be updated
@@ -181,6 +182,9 @@ class G1BlockOffsetTablePart {
 
   void verify() const;
 
+  // Given an address calculate where the next threshold needing an update is.
+  inline HeapWord* threshold_for_addr(const void* addr);
+
   // Returns the address of the start of the block containing "addr", or
   // else "null" if it is covered by no block.  (May have side effects,
   // namely updating of shared array entries that "point" too far
@@ -204,6 +208,11 @@ class G1BlockOffsetTablePart {
   // updated.
   HeapWord* threshold() const { return _next_offset_threshold; }
 
+  // Sets the threshold explicitly to keep it consistent with what has been
+  // updated. This needs to be done when the threshold is not used for updating
+  // the bot, for example when promoting to old in young collections.
+  void set_threshold(HeapWord* threshold) { _next_offset_threshold = threshold; }
+
   // These must be guaranteed to work properly (i.e., do nothing)
   // when "blk_start" ("blk" for second version) is "NULL".  In this
   // implementation, that's true because NULL is represented as 0, and thus

src/hotspot/share/gc/g1/g1BlockOffsetTable.inline.hpp

@@ -31,6 +31,20 @@
 #include "gc/shared/memset_with_concurrent_readers.hpp"
 #include "runtime/atomic.hpp"
 
+inline HeapWord* G1BlockOffsetTablePart::threshold_for_addr(const void* addr) {
+  assert(addr >= _hr->bottom() && addr < _hr->top(), "invalid address");
+  size_t index = _bot->index_for(addr);
+  HeapWord* card_boundary = _bot->address_for_index(index);
+  // Address at card boundary, use as threshold.
+  if (card_boundary == addr) {
+    return card_boundary;
+  }
+
+  // Calculate next threshold.
+  HeapWord* threshold = card_boundary + BOTConstants::N_words;
+  return threshold;
+}
+
 inline HeapWord* G1BlockOffsetTablePart::block_start(const void* addr) {
   assert(addr >= _hr->bottom() && addr < _hr->top(), "invalid address");
   HeapWord* q = block_at_or_preceding(addr);
@@ -109,9 +123,6 @@ inline HeapWord* G1BlockOffsetTablePart::block_at_or_preceding(const void* addr)
          "Object crossed region boundary, found offset %u instead of 0",
          (uint) _bot->offset_array(_bot->index_for(_hr->bottom())));
 
-  // We must make sure that the offset table entry we use is valid.
-  assert(addr < _next_offset_threshold, "Precondition");
-
   size_t index = _bot->index_for(addr);
 
   HeapWord* q = _bot->address_for_index(index);

src/hotspot/share/gc/g1/g1CollectedHeap.cpp

@@ -3293,6 +3293,7 @@ void G1CollectedHeap::retire_gc_alloc_region(HeapRegion* alloc_region,
   _bytes_used_during_gc += allocated_bytes;
   if (dest.is_old()) {
     old_set_add(alloc_region);
+    alloc_region->update_bot_threshold();
   } else {
     assert(dest.is_young(), "Retiring alloc region should be young (%d)", dest.type());
     _survivor.add_used_bytes(allocated_bytes);
@@ -3445,3 +3446,8 @@ GrowableArray<GCMemoryManager*> G1CollectedHeap::memory_managers() {
 GrowableArray<MemoryPool*> G1CollectedHeap::memory_pools() {
   return _monitoring_support->memory_pools();
 }
+
+void G1CollectedHeap::fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap) {
+  HeapRegion* region = heap_region_containing(start);
+  region->fill_with_dummy_object(start, pointer_delta(end, start), zap);
+}

src/hotspot/share/gc/g1/g1CollectedHeap.hpp

@@ -934,6 +934,8 @@ class G1CollectedHeap : public CollectedHeap {
   virtual GrowableArray<GCMemoryManager*> memory_managers();
   virtual GrowableArray<MemoryPool*> memory_pools();
 
+  virtual void fill_with_dummy_object(HeapWord* start, HeapWord* end, bool zap);
+
   // Try to minimize the remembered set.
   void scrub_rem_set();
 

src/hotspot/share/gc/g1/g1ParScanThreadState.cpp

@@ -498,6 +498,11 @@ oop G1ParScanThreadState::do_copy_to_survivor_space(G1HeapRegionAttr const regio
         obj->incr_age();
       }
       _age_table.add(age, word_sz);
+    } else {
+      // Currently we only have two destinations and we only need BOT updates for
+      // old. If the current allocation was done outside the PLAB this call will
+      // have no effect since the _top of the PLAB has not changed.
+      _plab_allocator->update_bot_for_plab_allocation(dest_attr, word_sz, node_index);
     }
 
     // Most objects are not arrays, so do one array check rather than

src/hotspot/share/gc/g1/heapRegion.cpp

@@ -235,7 +235,6 @@ HeapRegion::HeapRegion(uint hrm_index,
   _top(NULL),
   _compaction_top(NULL),
   _bot_part(bot, this),
-  _par_alloc_lock(Mutex::service-2, "HeapRegionParAlloc_lock"),
   _pre_dummy_top(NULL),
   _rem_set(NULL),
   _hrm_index(hrm_index),
@@ -818,3 +817,12 @@ void HeapRegion::object_iterate(ObjectClosure* blk) {
     p += block_size(p);
   }
 }
+
+void HeapRegion::fill_with_dummy_object(HeapWord* address, size_t word_size, bool zap) {
+  // Keep the BOT in sync for old generation regions.
+  if (is_old()) {
+    update_bot_at(address, word_size);
+  }
+  // Fill in the object.
+  CollectedHeap::fill_with_object(address, word_size, zap);
+}

src/hotspot/share/gc/g1/heapRegion.hpp

@@ -78,7 +78,7 @@ class HeapRegion : public CHeapObj<mtGC> {
   HeapWord* _compaction_top;
 
   G1BlockOffsetTablePart _bot_part;
-  Mutex _par_alloc_lock;
+
   // When we need to retire an allocation region, while other threads
   // are also concurrently trying to allocate into it, we typically
   // allocate a dummy object at the end of the region to ensure that
@@ -153,18 +153,24 @@ class HeapRegion : public CHeapObj<mtGC> {
 
   void object_iterate(ObjectClosure* blk);
 
-  // Allocation (return NULL if full).  Assumes the caller has established
-  // mutually exclusive access to the HeapRegion.
-  HeapWord* allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size);
-  // Allocation (return NULL if full).  Enforces mutual exclusion internally.
-  HeapWord* par_allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_word_size);
-
-  HeapWord* allocate(size_t word_size);
-  HeapWord* par_allocate(size_t word_size);
-
-  inline HeapWord* par_allocate_no_bot_updates(size_t min_word_size, size_t desired_word_size, size_t* word_size);
-  inline HeapWord* allocate_no_bot_updates(size_t word_size);
-  inline HeapWord* allocate_no_bot_updates(size_t min_word_size, size_t desired_word_size, size_t* actual_size);
+  // At the given address create an object with the given size. If the region
+  // is old the BOT will be updated if the object spans a threshold.
+  void fill_with_dummy_object(HeapWord* address, size_t word_size, bool zap = true);
+
+  // All allocations are done without updating the BOT. The BOT
+  // needs to be kept in sync for old generation regions and
+  // this is done by explicit updates when crossing thresholds.
+  inline HeapWord* par_allocate(size_t min_word_size, size_t desired_word_size, size_t* word_size);
+  inline HeapWord* allocate(size_t word_size);
+  inline HeapWord* allocate(size_t min_word_size, size_t desired_word_size, size_t* actual_size);
+
+  // Update the BOT for the given address if it crosses the next
+  // BOT threshold at or after obj_start.
+  inline void update_bot_at(HeapWord* obj_start, size_t obj_size);
+  // Update BOT at the given threshold for the given object. The
+  // given object must cross the threshold.
+  inline void update_bot_crossing_threshold(HeapWord** threshold, HeapWord* obj_start, HeapWord* obj_end);
+  inline HeapWord* bot_threshold_for_addr(const void* addr);
 
   // Full GC support methods.
 
@@ -201,6 +207,10 @@ class HeapRegion : public CHeapObj<mtGC> {
     _bot_part.update();
   }
 
+  void update_bot_threshold() {
+    _bot_part.set_threshold(top());
+  }
+
 private:
   // The remembered set for this region.
   HeapRegionRemSet* _rem_set;