Skip to content

Commit

Permalink
8259776: Remove ParallelGC non-CAS oldgen allocation
Browse files Browse the repository at this point in the history
Reviewed-by: tschatzl, sjohanss
  • Loading branch information
Kim Barrett committed Jan 23, 2021
1 parent 6c4c96f commit 06348df
Show file tree
Hide file tree
Showing 10 changed files with 31 additions and 137 deletions.
45 changes: 0 additions & 45 deletions src/hotspot/share/gc/parallel/mutableNUMASpace.cpp
Expand Up @@ -792,51 +792,6 @@ void MutableNUMASpace::clear(bool mangle_space) {
objects.
*/

HeapWord* MutableNUMASpace::allocate(size_t size) {
Thread* thr = Thread::current();
int lgrp_id = thr->lgrp_id();
if (lgrp_id == -1 || !os::numa_has_group_homing()) {
lgrp_id = os::numa_get_group_id();
thr->set_lgrp_id(lgrp_id);
}

int i = lgrp_spaces()->find(&lgrp_id, LGRPSpace::equals);

// It is possible that a new CPU has been hotplugged and
// we haven't reshaped the space accordingly.
if (i == -1) {
i = os::random() % lgrp_spaces()->length();
}

LGRPSpace* ls = lgrp_spaces()->at(i);
MutableSpace *s = ls->space();
HeapWord *p = s->allocate(size);

if (p != NULL) {
size_t remainder = s->free_in_words();
if (remainder < CollectedHeap::min_fill_size() && remainder > 0) {
s->set_top(s->top() - size);
p = NULL;
}
}
if (p != NULL) {
if (top() < s->top()) { // Keep _top updated.
MutableSpace::set_top(s->top());
}
}
// Make the page allocation happen here if there is no static binding..
if (p != NULL && !os::numa_has_static_binding()) {
for (HeapWord *i = p; i < p + size; i += os::vm_page_size() >> LogHeapWordSize) {
*(int*)i = 0;
}
}
if (p == NULL) {
ls->set_allocation_failed();
}
return p;
}

// This version is lock-free.
HeapWord* MutableNUMASpace::cas_allocate(size_t size) {
Thread* thr = Thread::current();
int lgrp_id = thr->lgrp_id();
Expand Down
3 changes: 1 addition & 2 deletions src/hotspot/share/gc/parallel/mutableNUMASpace.hpp
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2006, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2006, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -227,7 +227,6 @@ class MutableNUMASpace : public MutableSpace {
virtual size_t unsafe_max_tlab_alloc(Thread* thr) const;

// Allocation (return NULL if full)
virtual HeapWord* allocate(size_t word_size);
virtual HeapWord* cas_allocate(size_t word_size);

// Debugging
Expand Down
21 changes: 1 addition & 20 deletions src/hotspot/share/gc/parallel/mutableSpace.cpp
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -173,25 +173,6 @@ void MutableSpace::set_top_for_allocations() {
}
#endif

// This version requires locking. */
HeapWord* MutableSpace::allocate(size_t size) {
assert(Heap_lock->owned_by_self() ||
(SafepointSynchronize::is_at_safepoint() &&
Thread::current()->is_VM_thread()),
"not locked");
HeapWord* obj = top();
if (pointer_delta(end(), obj) >= size) {
HeapWord* new_top = obj + size;
set_top(new_top);
assert(is_object_aligned(obj) && is_object_aligned(new_top),
"checking alignment");
return obj;
} else {
return NULL;
}
}

// This version is lock-free.
HeapWord* MutableSpace::cas_allocate(size_t size) {
do {
// Read top before end, else the range check may pass when it shouldn't.
Expand Down
3 changes: 1 addition & 2 deletions src/hotspot/share/gc/parallel/mutableSpace.hpp
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -130,7 +130,6 @@ class MutableSpace: public ImmutableSpace {
virtual size_t unsafe_max_tlab_alloc(Thread* thr) const { return free_in_bytes(); }

// Allocation (return NULL if full)
virtual HeapWord* allocate(size_t word_size);
virtual HeapWord* cas_allocate(size_t word_size);
// Optional deallocation. Used in NUMA-allocator.
bool cas_deallocate(HeapWord *obj, size_t size);
Expand Down
19 changes: 14 additions & 5 deletions src/hotspot/share/gc/parallel/parallelScavengeHeap.cpp
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -398,18 +398,27 @@ ParallelScavengeHeap::death_march_check(HeapWord* const addr, size_t size) {
}
}

HeapWord* ParallelScavengeHeap::allocate_old_gen_and_record(size_t size) {
assert_locked_or_safepoint(Heap_lock);
HeapWord* res = old_gen()->allocate(size);
if (res != NULL) {
_size_policy->tenured_allocation(size * HeapWordSize);
}
return res;
}

HeapWord* ParallelScavengeHeap::mem_allocate_old_gen(size_t size) {
if (!should_alloc_in_eden(size) || GCLocker::is_active_and_needs_gc()) {
// Size is too big for eden, or gc is locked out.
return old_gen()->allocate(size);
return allocate_old_gen_and_record(size);
}

// If a "death march" is in progress, allocate from the old gen a limited
// number of times before doing a GC.
if (_death_march_count > 0) {
if (_death_march_count < 64) {
++_death_march_count;
return old_gen()->allocate(size);
return allocate_old_gen_and_record(size);
} else {
_death_march_count = 0;
}
Expand Down Expand Up @@ -457,7 +466,7 @@ HeapWord* ParallelScavengeHeap::failed_mem_allocate(size_t size) {
// After mark sweep and young generation allocation failure,
// allocate in old generation.
if (result == NULL) {
result = old_gen()->allocate(size);
result = allocate_old_gen_and_record(size);
}

// Fourth level allocation failure. We're running out of memory.
Expand All @@ -470,7 +479,7 @@ HeapWord* ParallelScavengeHeap::failed_mem_allocate(size_t size) {
// Fifth level allocation failure.
// After more complete mark sweep, allocate in old generation.
if (result == NULL) {
result = old_gen()->allocate(size);
result = allocate_old_gen_and_record(size);
}

return result;
Expand Down
6 changes: 4 additions & 2 deletions src/hotspot/share/gc/parallel/parallelScavengeHeap.hpp
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -43,7 +43,6 @@
#include "utilities/growableArray.hpp"
#include "utilities/ostream.hpp"

class AdjoiningGenerations;
class GCHeapSummary;
class HeapBlockClaimer;
class MemoryManager;
Expand Down Expand Up @@ -80,6 +79,9 @@ class ParallelScavengeHeap : public CollectedHeap {
void trace_actual_reserved_page_size(const size_t reserved_heap_size, const ReservedSpace rs);
void trace_heap(GCWhen::Type when, const GCTracer* tracer);

// Allocate in oldgen and record the allocation with the size_policy.
HeapWord* allocate_old_gen_and_record(size_t word_size);

protected:
static inline size_t total_invocations();
HeapWord* allocate_new_tlab(size_t min_size, size_t requested_size, size_t* actual_size);
Expand Down
29 changes: 1 addition & 28 deletions src/hotspot/share/gc/parallel/psOldGen.cpp
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -147,25 +147,6 @@ bool PSOldGen::is_allocated() {
return virtual_space()->reserved_size() != 0;
}

// Allocation. We report all successful allocations to the size policy
// Note that the perm gen does not use this method, and should not!
HeapWord* PSOldGen::allocate(size_t word_size) {
assert_locked_or_safepoint(Heap_lock);
HeapWord* res = allocate_noexpand(word_size);

if (res == NULL) {
res = expand_and_allocate(word_size);
}

// Allocations in the old generation need to be reported
if (res != NULL) {
ParallelScavengeHeap* heap = ParallelScavengeHeap::heap();
heap->size_policy()->tenured_allocation(word_size * HeapWordSize);
}

return res;
}

size_t PSOldGen::num_iterable_blocks() const {
return (object_space()->used_in_bytes() + IterateBlockSize - 1) / IterateBlockSize;
}
Expand Down Expand Up @@ -198,14 +179,6 @@ void PSOldGen::object_iterate_block(ObjectClosure* cl, size_t block_index) {
}
}

HeapWord* PSOldGen::expand_and_allocate(size_t word_size) {
expand(word_size*HeapWordSize);
if (GCExpandToAllocateDelayMillis > 0) {
os::naked_sleep(GCExpandToAllocateDelayMillis);
}
return allocate_noexpand(word_size);
}

HeapWord* PSOldGen::expand_and_cas_allocate(size_t word_size) {
expand(word_size*HeapWordSize);
if (GCExpandToAllocateDelayMillis > 0) {
Expand Down
35 changes: 6 additions & 29 deletions src/hotspot/share/gc/parallel/psOldGen.hpp
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -34,9 +34,6 @@

class PSOldGen : public CHeapObj<mtGC> {
friend class VMStructs;
friend class PSPromotionManager; // Uses the cas_allocate methods
friend class ParallelScavengeHeap;
friend class AdjoiningGenerations;

private:
MemRegion _reserved; // Used for simple containment tests
Expand Down Expand Up @@ -72,22 +69,8 @@ class PSOldGen : public CHeapObj<mtGC> {
}
#endif

HeapWord* allocate_noexpand(size_t word_size) {
// We assume the heap lock is held here.
assert_locked_or_safepoint(Heap_lock);
HeapWord* res = object_space()->allocate(word_size);
if (res != NULL) {
DEBUG_ONLY(assert_block_in_covered_region(MemRegion(res, word_size)));
_start_array.allocate_block(res);
}
return res;
}

// Support for MT garbage collection. CAS allocation is lower overhead than grabbing
// and releasing the heap lock, which is held during gc's anyway. This method is not
// safe for use at the same time as allocate_noexpand()!
HeapWord* cas_allocate_noexpand(size_t word_size) {
assert(SafepointSynchronize::is_at_safepoint(), "Must only be called at safepoint");
assert_locked_or_safepoint(Heap_lock);
HeapWord* res = object_space()->cas_allocate(word_size);
if (res != NULL) {
DEBUG_ONLY(assert_block_in_covered_region(MemRegion(res, word_size)));
Expand All @@ -96,13 +79,6 @@ class PSOldGen : public CHeapObj<mtGC> {
return res;
}

// Support for MT garbage collection. See above comment.
HeapWord* cas_allocate(size_t word_size) {
HeapWord* res = cas_allocate_noexpand(word_size);
return (res == NULL) ? expand_and_cas_allocate(word_size) : res;
}

HeapWord* expand_and_allocate(size_t word_size);
HeapWord* expand_and_cas_allocate(size_t word_size);
void expand(size_t bytes);
bool expand_by(size_t bytes);
Expand Down Expand Up @@ -158,9 +134,10 @@ class PSOldGen : public CHeapObj<mtGC> {
// Calculating new sizes
void resize(size_t desired_free_space);

// Allocation. We report all successful allocations to the size policy
// Note that the perm gen does not use this method, and should not!
HeapWord* allocate(size_t word_size);
HeapWord* allocate(size_t word_size) {
HeapWord* res = cas_allocate_noexpand(word_size);
return (res == NULL) ? expand_and_cas_allocate(word_size) : res;
}

// Iteration.
void oop_iterate(OopIterateClosure* cl) { object_space()->oop_iterate(cl); }
Expand Down
4 changes: 2 additions & 2 deletions src/hotspot/share/gc/parallel/psPromotionManager.inline.hpp
Expand Up @@ -194,13 +194,13 @@ inline oop PSPromotionManager::copy_to_survivor_space(oop o) {
// Do we allocate directly, or flush and refill?
if (new_obj_size > (OldPLABSize / 2)) {
// Allocate this object directly
new_obj = (oop)old_gen()->cas_allocate(new_obj_size);
new_obj = (oop)old_gen()->allocate(new_obj_size);
promotion_trace_event(new_obj, o, new_obj_size, age, true, NULL);
} else {
// Flush and fill
_old_lab.flush();

HeapWord* lab_base = old_gen()->cas_allocate(OldPLABSize);
HeapWord* lab_base = old_gen()->allocate(OldPLABSize);
if(lab_base != NULL) {
#ifdef ASSERT
// Delay the initialization of the promotion lab (plab).
Expand Down
3 changes: 1 addition & 2 deletions src/hotspot/share/gc/parallel/psYoungGen.hpp
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2001, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2001, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
Expand Down Expand Up @@ -34,7 +34,6 @@
class PSYoungGen : public CHeapObj<mtGC> {
friend class VMStructs;
friend class ParallelScavengeHeap;
friend class AdjoiningGenerations;

private:
MemRegion _reserved;
Expand Down

1 comment on commit 06348df

@openjdk-notifier
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please sign in to comment.