8259776: Remove ParallelGC non-CAS oldgen allocation

Reviewed-by: tschatzl, sjohanss

Author: Kim Barrett

src/hotspot/share/gc/parallel/mutableNUMASpace.cpp

@@ -792,51 +792,6 @@ void MutableNUMASpace::clear(bool mangle_space) {
    objects.
  */
 
-HeapWord* MutableNUMASpace::allocate(size_t size) {
-  Thread* thr = Thread::current();
-  int lgrp_id = thr->lgrp_id();
-  if (lgrp_id == -1 || !os::numa_has_group_homing()) {
-    lgrp_id = os::numa_get_group_id();
-    thr->set_lgrp_id(lgrp_id);
-  }
-
-  int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);
-
-  // It is possible that a new CPU has been hotplugged and
-  // we haven't reshaped the space accordingly.
-  if (i == -1) {
-    i = os::random() % lgrp_spaces()->length();
-  }
-
-  LGRPSpace* ls = lgrp_spaces()->at(i);
-  MutableSpace *s = ls->space();
-  HeapWord *p = s->allocate(size);
-
-  if (p != NULL) {
-    size_t remainder = s->free_in_words();
-    if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
-      s->set_top(s->top() - size);
-      p = NULL;
-    }
-  }
-  if (p != NULL) {
-    if (top() < s->top()) { // Keep _top updated.
-      MutableSpace::set_top(s->top());
-    }
-  }
-  // Make the page allocation happen here if there is no static binding..
-  if (p != NULL && !os::numa_has_static_binding()) {
-    for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
-      *(int*)i = 0;
-    }
-  }
-  if (p == NULL) {
-    ls->set_allocation_failed();
-  }
-  return p;
-}
-
-// This version is lock-free.
 HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
   Thread* thr = Thread::current();
   int lgrp_id = thr->lgrp_id();

src/hotspot/share/gc/parallel/mutableNUMASpace.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2006, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -227,7 +227,6 @@ class MutableNUMASpace : public MutableSpace {
   virtual size_t unsafe_max_tlab_alloc(Thread* thr) const;
 
   // Allocation (return NULL if full)
-  virtual HeapWord* allocate(size_t word_size);
   virtual HeapWord* cas_allocate(size_t word_size);
 
   // Debugging

src/hotspot/share/gc/parallel/mutableSpace.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -173,25 +173,6 @@ void MutableSpace::set_top_for_allocations() {
 }
 #endif
 
-// This version requires locking. */
-HeapWord* MutableSpace::allocate(size_t size) {
-  assert(Heap_lock->owned_by_self() ||
-         (SafepointSynchronize::is_at_safepoint() &&
-          Thread::current()->is_VM_thread()),
-         "not locked");
-  HeapWord* obj = top();
-  if (pointer_delta(end(), obj) >= size) {
-    HeapWord* new_top = obj + size;
-    set_top(new_top);
-    assert(is_object_aligned(obj) && is_object_aligned(new_top),
-           "checking alignment");
-    return obj;
-  } else {
-    return NULL;
-  }
-}
-
-// This version is lock-free.
 HeapWord* MutableSpace::cas_allocate(size_t size) {
   do {
     // Read top before end, else the range check may pass when it shouldn't.

src/hotspot/share/gc/parallel/mutableSpace.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -130,7 +130,6 @@ class MutableSpace: public ImmutableSpace {
   virtual size_t unsafe_max_tlab_alloc(Thread* thr) const { return free_in_bytes();                }
 
   // Allocation (return NULL if full)
-  virtual HeapWord* allocate(size_t word_size);
   virtual HeapWord* cas_allocate(size_t word_size);
   // Optional deallocation. Used in NUMA-allocator.
   bool cas_deallocate(HeapWord *obj, size_t size);

src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -398,18 +398,27 @@ ParallelScavengeHeap::death_march_check(HeapWord* const addr, size_t size) {
   }
 }
 
+HeapWord* ParallelScavengeHeap::allocate_old_gen_and_record(size_t size) {
+  assert_locked_or_safepoint(Heap_lock);
+  HeapWord* res = old_gen()->allocate(size);
+  if (res != NULL) {
+    _size_policy->tenured_allocation(size * HeapWordSize);
+  }
+  return res;
+}
+
 HeapWord* ParallelScavengeHeap::mem_allocate_old_gen(size_t size) {
   if (!should_alloc_in_eden(size) || GCLocker::is_active_and_needs_gc()) {
     // Size is too big for eden, or gc is locked out.
-    return old_gen()->allocate(size);
+    return allocate_old_gen_and_record(size);
   }
 
   // If a "death march" is in progress, allocate from the old gen a limited
   // number of times before doing a GC.
   if (_death_march_count > 0) {
     if (_death_march_count < 64) {
       ++_death_march_count;
-      return old_gen()->allocate(size);
+      return allocate_old_gen_and_record(size);
     } else {
       _death_march_count = 0;
     }
@@ -457,7 +466,7 @@ HeapWord* ParallelScavengeHeap::failed_mem_allocate(size_t size) {
   //   After mark sweep and young generation allocation failure,
   //   allocate in old generation.
   if (result == NULL) {
-    result = old_gen()->allocate(size);
+    result = allocate_old_gen_and_record(size);
   }
 
   // Fourth level allocation failure. We're running out of memory.
@@ -470,7 +479,7 @@ HeapWord* ParallelScavengeHeap::failed_mem_allocate(size_t size) {
   // Fifth level allocation failure.
   //   After more complete mark sweep, allocate in old generation.
   if (result == NULL) {
-    result = old_gen()->allocate(size);
+    result = allocate_old_gen_and_record(size);
   }
 
   return result;

src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -43,7 +43,6 @@
 #include "utilities/growableArray.hpp"
 #include "utilities/ostream.hpp"
 
-class AdjoiningGenerations;
 class GCHeapSummary;
 class HeapBlockClaimer;
 class MemoryManager;
@@ -80,6 +79,9 @@ class ParallelScavengeHeap : public CollectedHeap {
   void trace_actual_reserved_page_size(const size_t reserved_heap_size, const ReservedSpace rs);
   void trace_heap(GCWhen::Type when, const GCTracer* tracer);
 
+  // Allocate in oldgen and record the allocation with the size_policy.
+  HeapWord* allocate_old_gen_and_record(size_t word_size);
+
  protected:
   static inline size_t total_invocations();
   HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size);

src/hotspot/share/gc/parallel/psOldGen.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -147,25 +147,6 @@ bool  PSOldGen::is_allocated() {
   return virtual_space()->reserved_size() != 0;
 }
 
-// Allocation. We report all successful allocations to the size policy
-// Note that the perm gen does not use this method, and should not!
-HeapWord* PSOldGen::allocate(size_t word_size) {
-  assert_locked_or_safepoint(Heap_lock);
-  HeapWord* res = allocate_noexpand(word_size);
-
-  if (res == NULL) {
-    res = expand_and_allocate(word_size);
-  }
-
-  // Allocations in the old generation need to be reported
-  if (res != NULL) {
-    ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
-    heap->size_policy()->tenured_allocation(word_size * HeapWordSize);
-  }
-
-  return res;
-}
-
 size_t PSOldGen::num_iterable_blocks() const {
   return (object_space()->used_in_bytes() + IterateBlockSize - 1) / IterateBlockSize;
 }
@@ -198,14 +179,6 @@ void PSOldGen::object_iterate_block(ObjectClosure* cl, size_t block_index) {
   }
 }
 
-HeapWord* PSOldGen::expand_and_allocate(size_t word_size) {
-  expand(word_size*HeapWordSize);
-  if (GCExpandToAllocateDelayMillis > 0) {
-    os::naked_sleep(GCExpandToAllocateDelayMillis);
-  }
-  return allocate_noexpand(word_size);
-}
-
 HeapWord* PSOldGen::expand_and_cas_allocate(size_t word_size) {
   expand(word_size*HeapWordSize);
   if (GCExpandToAllocateDelayMillis > 0) {

src/hotspot/share/gc/parallel/psOldGen.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,9 +34,6 @@
 
 class PSOldGen : public CHeapObj<mtGC> {
   friend class VMStructs;
-  friend class PSPromotionManager; // Uses the cas_allocate methods
-  friend class ParallelScavengeHeap;
-  friend class AdjoiningGenerations;
 
  private:
   MemRegion                _reserved;          // Used for simple containment tests
@@ -72,22 +69,8 @@ class PSOldGen : public CHeapObj<mtGC> {
   }
 #endif
 
-  HeapWord* allocate_noexpand(size_t word_size) {
-    // We assume the heap lock is held here.
-    assert_locked_or_safepoint(Heap_lock);
-    HeapWord* res = object_space()->allocate(word_size);
-    if (res != NULL) {
-      DEBUG_ONLY(assert_block_in_covered_region(MemRegion(res, word_size)));
-      _start_array.allocate_block(res);
-    }
-    return res;
-  }
-
-  // Support for MT garbage collection. CAS allocation is lower overhead than grabbing
-  // and releasing the heap lock, which is held during gc's anyway. This method is not
-  // safe for use at the same time as allocate_noexpand()!
   HeapWord* cas_allocate_noexpand(size_t word_size) {
-    assert(SafepointSynchronize::is_at_safepoint(), "Must only be called at safepoint");
+    assert_locked_or_safepoint(Heap_lock);
     HeapWord* res = object_space()->cas_allocate(word_size);
     if (res != NULL) {
       DEBUG_ONLY(assert_block_in_covered_region(MemRegion(res, word_size)));
@@ -96,13 +79,6 @@ class PSOldGen : public CHeapObj<mtGC> {
     return res;
   }
 
-  // Support for MT garbage collection. See above comment.
-  HeapWord* cas_allocate(size_t word_size) {
-    HeapWord* res = cas_allocate_noexpand(word_size);
-    return (res == NULL) ? expand_and_cas_allocate(word_size) : res;
-  }
-
-  HeapWord* expand_and_allocate(size_t word_size);
   HeapWord* expand_and_cas_allocate(size_t word_size);
   void expand(size_t bytes);
   bool expand_by(size_t bytes);
@@ -158,9 +134,10 @@ class PSOldGen : public CHeapObj<mtGC> {
   // Calculating new sizes
   void resize(size_t desired_free_space);
 
-  // Allocation. We report all successful allocations to the size policy
-  // Note that the perm gen does not use this method, and should not!
-  HeapWord* allocate(size_t word_size);
+  HeapWord* allocate(size_t word_size) {
+    HeapWord* res = cas_allocate_noexpand(word_size);
+    return (res == NULL) ? expand_and_cas_allocate(word_size) : res;
+  }
 
   // Iteration.
   void oop_iterate(OopIterateClosure* cl) { object_space()->oop_iterate(cl); }

src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp

@@ -194,13 +194,13 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
           // Do we allocate directly, or flush and refill?
           if (new_obj_size > (OldPLABSize / 2)) {
             // Allocate this object directly
-            new_obj = (oop)old_gen()->cas_allocate(new_obj_size);
+            new_obj = (oop)old_gen()->allocate(new_obj_size);
             promotion_trace_event(new_obj, o, new_obj_size, age, true, NULL);
           } else {
             // Flush and fill
             _old_lab.flush();
 
-            HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize);
+            HeapWord* lab_base = old_gen()->allocate(OldPLABSize);
             if(lab_base != NULL) {
 #ifdef ASSERT
               // Delay the initialization of the promotion lab (plab).

src/hotspot/share/gc/parallel/psYoungGen.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -34,7 +34,6 @@
 class PSYoungGen : public CHeapObj<mtGC> {
   friend class VMStructs;
   friend class ParallelScavengeHeap;
-  friend class AdjoiningGenerations;
 
  private:
   MemRegion       _reserved;